From 406bfcc2d4c8c748ba8cba6d3fd78f21296b2390 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 22 Oct 2026 02:41:30 +0000 Subject: [PATCH 1/8] Pin ty 0.0.90 and ruff 0.17.1, drop unused mypy and pyright config --- pyproject.toml | 12 ++----- uv.lock | 91 +++++++++++++++++++++++++------------------------- 2 files changed, 47 insertions(+), 56 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4ca8cba..e9fba20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,8 +52,8 @@ postgres = [ dev = [ "oa-configurator[postgres]>=2.0.0,<3", "pytest>=9.0.3", - "ty>=0.0.59", - "ruff>=0.14.11", + "ty==0.0.90", + "ruff==0.17.1", "mkdocs-material>=9.7.7", "mkdocstrings-python>=2.0.1", "requests>=2.33.0", @@ -71,11 +71,6 @@ cache-keys = [{ file = "pyproject.toml" }, { git = { commit = true, tags = true [tool.ruff] line-length = 100 -target-version = "py311" - -[tool.mypy] -python_version = "3.11" -strict = true [tool.pytest.ini_options] testpaths = ["tests"] @@ -83,6 +78,3 @@ python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] addopts = "-ra -m 'not db_dialect'" - -[tool.pyright] -reportMissingTypeStubs = false diff --git a/uv.lock b/uv.lock index 439e964..40909c7 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12" [[package]] @@ -625,9 +625,9 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.3" }, { name = "python-dotenv", marker = "extra == 'dev'" }, { name = "requests", marker = "extra == 'dev'", specifier = ">=2.33.0" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.14.11" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.17.1" }, { name = "sqlalchemy", specifier = ">=2.0.45" }, - { name = "ty", marker = "extra == 'dev'", specifier = ">=0.0.59" }, + { name = "ty", marker = "extra == 'dev'", specifier = "==0.0.90" }, ] provides-extras = ["dev", "postgres"] @@ -1070,28 +1070,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.14.11" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d4/77/9a7fe084d268f8855d493e5031ea03fa0af8cc05887f638bf1c4e3363eb8/ruff-0.14.11.tar.gz", hash = "sha256:f6dc463bfa5c07a59b1ff2c3b9767373e541346ea105503b4c0369c520a66958", size = 5993417, upload-time = "2026-01-08T19:11:58.322Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/a6/a4c40a5aaa7e331f245d2dc1ac8ece306681f52b636b40ef87c88b9f7afd/ruff-0.14.11-py3-none-linux_armv6l.whl", hash = "sha256:f6ff2d95cbd335841a7217bdfd9c1d2e44eac2c584197ab1385579d55ff8830e", size = 12951208, upload-time = "2026-01-08T19:12:09.218Z" }, - { url = "https://files.pythonhosted.org/packages/5c/5c/360a35cb7204b328b685d3129c08aca24765ff92b5a7efedbdd6c150d555/ruff-0.14.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6f6eb5c1c8033680f4172ea9c8d3706c156223010b8b97b05e82c59bdc774ee6", size = 13330075, upload-time = "2026-01-08T19:12:02.549Z" }, - { url = "https://files.pythonhosted.org/packages/1b/9e/0cc2f1be7a7d33cae541824cf3f95b4ff40d03557b575912b5b70273c9ec/ruff-0.14.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f2fc34cc896f90080fca01259f96c566f74069a04b25b6205d55379d12a6855e", size = 12257809, upload-time = "2026-01-08T19:12:00.366Z" }, - { url = "https://files.pythonhosted.org/packages/a7/e5/5faab97c15bb75228d9f74637e775d26ac703cc2b4898564c01ab3637c02/ruff-0.14.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:53386375001773ae812b43205d6064dae49ff0968774e6befe16a994fc233caa", size = 12678447, upload-time = "2026-01-08T19:12:13.899Z" }, - { url = "https://files.pythonhosted.org/packages/1b/33/e9767f60a2bef779fb5855cab0af76c488e0ce90f7bb7b8a45c8a2ba4178/ruff-0.14.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a697737dce1ca97a0a55b5ff0434ee7205943d4874d638fe3ae66166ff46edbe", size = 12758560, upload-time = "2026-01-08T19:11:42.55Z" }, - { url = "https://files.pythonhosted.org/packages/eb/84/4c6cf627a21462bb5102f7be2a320b084228ff26e105510cd2255ea868e5/ruff-0.14.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6845ca1da8ab81ab1dce755a32ad13f1db72e7fba27c486d5d90d65e04d17b8f", size = 13599296, upload-time = "2026-01-08T19:11:30.371Z" }, - { url = "https://files.pythonhosted.org/packages/88/e1/92b5ed7ea66d849f6157e695dc23d5d6d982bd6aa8d077895652c38a7cae/ruff-0.14.11-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:e36ce2fd31b54065ec6f76cb08d60159e1b32bdf08507862e32f47e6dde8bcbf", size = 15048981, upload-time = "2026-01-08T19:12:04.742Z" }, - { url = "https://files.pythonhosted.org/packages/61/df/c1bd30992615ac17c2fb64b8a7376ca22c04a70555b5d05b8f717163cf9f/ruff-0.14.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:590bcc0e2097ecf74e62a5c10a6b71f008ad82eb97b0a0079e85defe19fe74d9", size = 14633183, upload-time = "2026-01-08T19:11:40.069Z" }, - { url = "https://files.pythonhosted.org/packages/04/e9/fe552902f25013dd28a5428a42347d9ad20c4b534834a325a28305747d64/ruff-0.14.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:53fe71125fc158210d57fe4da26e622c9c294022988d08d9347ec1cf782adafe", size = 14050453, upload-time = "2026-01-08T19:11:37.555Z" }, - { url = "https://files.pythonhosted.org/packages/ae/93/f36d89fa021543187f98991609ce6e47e24f35f008dfe1af01379d248a41/ruff-0.14.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a35c9da08562f1598ded8470fcfef2afb5cf881996e6c0a502ceb61f4bc9c8a3", size = 13757889, upload-time = "2026-01-08T19:12:07.094Z" }, - { url = "https://files.pythonhosted.org/packages/b7/9f/c7fb6ecf554f28709a6a1f2a7f74750d400979e8cd47ed29feeaa1bd4db8/ruff-0.14.11-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0f3727189a52179393ecf92ec7057c2210203e6af2676f08d92140d3e1ee72c1", size = 13955832, upload-time = "2026-01-08T19:11:55.064Z" }, - { url = "https://files.pythonhosted.org/packages/db/a0/153315310f250f76900a98278cf878c64dfb6d044e184491dd3289796734/ruff-0.14.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:eb09f849bd37147a789b85995ff734a6c4a095bed5fd1608c4f56afc3634cde2", size = 12586522, upload-time = "2026-01-08T19:11:35.356Z" }, - { url = "https://files.pythonhosted.org/packages/2f/2b/a73a2b6e6d2df1d74bf2b78098be1572191e54bec0e59e29382d13c3adc5/ruff-0.14.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:c61782543c1231bf71041461c1f28c64b961d457d0f238ac388e2ab173d7ecb7", size = 12724637, upload-time = "2026-01-08T19:11:47.796Z" }, - { url = "https://files.pythonhosted.org/packages/f0/41/09100590320394401cd3c48fc718a8ba71c7ddb1ffd07e0ad6576b3a3df2/ruff-0.14.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:82ff352ea68fb6766140381748e1f67f83c39860b6446966cff48a315c3e2491", size = 13145837, upload-time = "2026-01-08T19:11:32.87Z" }, - { url = "https://files.pythonhosted.org/packages/3b/d8/e035db859d1d3edf909381eb8ff3e89a672d6572e9454093538fe6f164b0/ruff-0.14.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:728e56879df4ca5b62a9dde2dd0eb0edda2a55160c0ea28c4025f18c03f86984", size = 13850469, upload-time = "2026-01-08T19:12:11.694Z" }, - { url = "https://files.pythonhosted.org/packages/4e/02/bb3ff8b6e6d02ce9e3740f4c17dfbbfb55f34c789c139e9cd91985f356c7/ruff-0.14.11-py3-none-win32.whl", hash = "sha256:337c5dd11f16ee52ae217757d9b82a26400be7efac883e9e852646f1557ed841", size = 12851094, upload-time = "2026-01-08T19:11:45.163Z" }, - { url = "https://files.pythonhosted.org/packages/58/f1/90ddc533918d3a2ad628bc3044cdfc094949e6d4b929220c3f0eb8a1c998/ruff-0.14.11-py3-none-win_amd64.whl", hash = "sha256:f981cea63d08456b2c070e64b79cb62f951aa1305282974d4d5216e6e0178ae6", size = 14001379, upload-time = "2026-01-08T19:11:52.591Z" }, - { url = "https://files.pythonhosted.org/packages/c4/1c/1dbe51782c0e1e9cfce1d1004752672d2d4629ea46945d19d731ad772b3b/ruff-0.14.11-py3-none-win_arm64.whl", hash = "sha256:649fb6c9edd7f751db276ef42df1f3df41c38d67d199570ae2a7bd6cbc3590f0", size = 12938644, upload-time = "2026-01-08T19:11:50.027Z" }, +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e7/e6d5749983b020e6cb1055abec0a6f361a8e94f0184cdcf56d4760995f02/ruff-0.17.1.tar.gz", hash = "sha256:5bb796c5112e9fb9527f2ef2faa0a130d6312d19de92bcfaddd70dbb2d41020f", size = 5323349, upload-time = "2026-10-15T15:23:18.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/78/90c46ca6250709f7c37e397b60d06061adee8d0f7d13959ed6a8e49b14fd/ruff-0.17.1-py3-none-linux_armv6l.whl", hash = "sha256:1796c41eafec66f4635125dd1e074a7aa27b8d20c187d484360acf11d247fc4a", size = 10147528, upload-time = "2026-10-15T15:22:33.667Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d2/a2e28bc35b083ae51fa07dfba5280a667c8b01ecaee7802fc8849dcd5c46/ruff-0.17.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:8eadb09a788010be29c895418bdfa513e4e1c65e64b94e5b19e4cd381f6cc7fb", size = 10225974, upload-time = "2026-10-15T15:22:36.328Z" }, + { url = "https://files.pythonhosted.org/packages/8e/71/7a9029a81c8206ee01a98f6cfeeb56e6c095af86857581a37045c46d7e94/ruff-0.17.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:29b454f18ca3eeaf37fbf140841dcf27d36109e29dee9aae8acecb68c26c6f74", size = 9270361, upload-time = "2026-10-15T15:22:38.651Z" }, + { url = "https://files.pythonhosted.org/packages/c0/8c/e075e9c917cb0575297880c96768304e881240171c870cbd8d9b7b54bec2/ruff-0.17.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872e03dd231744d6c5ec5c8af6be52b71323fc6ee096e4c8b34d9f844ee3b978", size = 9938300, upload-time = "2026-10-15T15:22:41.284Z" }, + { url = "https://files.pythonhosted.org/packages/87/27/0b4bb926a49de2383f161272b4d863571a4c7ecdbf595a6bc58813a13367/ruff-0.17.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4c0d6cb5f8fbac77eec9af49f4fc0eaf856b05545e372b0694d24b4e76b1c640", size = 9948464, upload-time = "2026-10-15T15:22:43.681Z" }, + { url = "https://files.pythonhosted.org/packages/14/27/518d57e0e02aecb8ea4acac6b57b51a53173d25fa734952bfdd7d531e88e/ruff-0.17.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78beced4a3c94dfedbee4b999b95b6a27b35820fdc0afbacc6d37a8da26c1cd1", size = 10638639, upload-time = "2026-10-15T15:22:46.081Z" }, + { url = "https://files.pythonhosted.org/packages/01/82/d89a4e498ade59e6cee52d67f8da569019372212b87468c1fdeae884601c/ruff-0.17.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7feefc63921202894f0e158b0e32f45a5950d70a7fb693ccb9264816c25b1c05", size = 11337238, upload-time = "2026-10-15T15:22:48.894Z" }, + { url = "https://files.pythonhosted.org/packages/a2/7c/3b0fdef9352656317c977086fe1d802902406e4f2a0d8578da90c0e6d1c5/ruff-0.17.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684154861a819bf1f433669d2a0ce9c3e54677c65b8ebb866cc2a7ec86345785", size = 10896327, upload-time = "2026-10-15T15:22:51.45Z" }, + { url = "https://files.pythonhosted.org/packages/62/86/5566290219708ce2928f3a29c4f843a1185b341ec6111c4631a33c5ceced/ruff-0.17.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6ef98e51de8ef38082f90653ceeb84944bd7673783eea13e1647c5b9b064189", size = 10350811, upload-time = "2026-10-15T15:22:54.401Z" }, + { url = "https://files.pythonhosted.org/packages/ad/d2/ca8462dabb3c413990d49703ba306c30c862195a75448d080fa8fcce9775/ruff-0.17.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:8f9aa5b5bd5fa1937803ea609c8f153b8de39bf3e0e149897ae34b0c5b52f467", size = 10569994, upload-time = "2026-10-15T15:22:57.142Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ad/dd1864c1f1db17c8cfb297ce70b352388e6977620f66f7182c52855407df/ruff-0.17.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:80e53b0e5faa3c5667f035daaa032893ae42b7cfe708c886d39ed131e54688ab", size = 10059672, upload-time = "2026-10-15T15:23:00.084Z" }, + { url = "https://files.pythonhosted.org/packages/65/45/3f56cf1d2168c2165cdd5ac6f9668aee66a4cfdd4caf2b2cc5ccb2b1d568/ruff-0.17.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:056c5d5fa0ac790051ae165d7035c7bf956d02de2db557f2e9b31d877a505fca", size = 9947803, upload-time = "2026-10-15T15:23:03.132Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4b/5457cb2be4c1d1ea5e919558131a0f197a4a454a0530d9e30820e8206e4c/ruff-0.17.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bd8f5a55b10f371642eacd0e8665a42a8b97da15c9a7187c79e697b77b5e0ce4", size = 10351807, upload-time = "2026-10-15T15:23:05.628Z" }, + { url = "https://files.pythonhosted.org/packages/49/fb/db15b4f8fa24a1e36b99e1401f42ef6df16ee299a48670d5d262b771e7d7/ruff-0.17.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ce05e6f3973611d9550a600e04bb531061e368baba31610eb57c1509a8f21a0a", size = 10765716, upload-time = "2026-10-15T15:23:08.092Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e8/1e186578076aff7a80735f625005fde678412ab609cd323ed182eb7d6b87/ruff-0.17.1-py3-none-win32.whl", hash = "sha256:1de0030a8aa78c65c5fecb84b2e5c390ac684c0452875a02afc7c083d49a496a", size = 9895093, upload-time = "2026-10-15T15:23:10.582Z" }, + { url = "https://files.pythonhosted.org/packages/6a/2e/da3d9589bae8e8b58f3a48bd5bc0b980232fb14a83bf02c3be7660f87c1b/ruff-0.17.1-py3-none-win_amd64.whl", hash = "sha256:5df76f16580fab58d0f6c98a3ac254a4abceb7b2489b8b678335869cff77b3d7", size = 10097955, upload-time = "2026-10-15T15:23:13.309Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/0f2a4dea06a5e12c1191ee3d96b1e1233797d0d5e18f97a2b909a95a9836/ruff-0.17.1-py3-none-win_arm64.whl", hash = "sha256:42400b9a6ff8515ff27796f07f92e0195be011c86e81e381606f58ea5502fef4", size = 9853542, upload-time = "2026-10-15T15:23:16.05Z" }, ] [[package]] @@ -1176,27 +1175,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.61" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/92/63/6944925d0fe9a4bb9cc744e6c045a42bbd2ee4654c103190674577a36c3f/ty-0.0.61.tar.gz", hash = "sha256:acbf0d914cc7e2e57ccc440036af36114819e2a604a5ffb554e72e4ca7dd65a2", size = 6234957, upload-time = "2026-07-18T01:39:54.696Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/cf/044f31523e2768e3e64b0ca2ec32f70b3a731d4a2caa6ea110baf26e251c/ty-0.0.61-py3-none-linux_armv6l.whl", hash = "sha256:148779b8675eac93f40ec58bd70037fe67537117f20a23272264f8f136d41336", size = 11891448, upload-time = "2026-07-18T01:39:18.449Z" }, - { url = "https://files.pythonhosted.org/packages/d2/55/558cfe76b65d91d1854bbfac336020bd42fd887caa632d845d13c0c539eb/ty-0.0.61-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:08217382b3385808ee7288501ea3214b32631b08d1fd091ece6799b0c95264c5", size = 11602442, upload-time = "2026-07-18T01:39:20.914Z" }, - { url = "https://files.pythonhosted.org/packages/27/be/78c0ae6634cd606a68e5b46b338db427a48a1800c96a749b2d2f7a702e03/ty-0.0.61-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6d99c729011b47dec20e78a32ac9c8f6defd4cf62f7bb851bbccf70dde6cee50", size = 11125286, upload-time = "2026-07-18T01:39:22.893Z" }, - { url = "https://files.pythonhosted.org/packages/a4/18/a40793962f1b6337938ddb0bca7496b54e70879e23b4d2cc8dfd7e5d1af3/ty-0.0.61-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cda607978ae271b77e51c947663218bce635c3507e256865444b10c37cdb60d", size = 11663403, upload-time = "2026-07-18T01:39:25.017Z" }, - { url = "https://files.pythonhosted.org/packages/98/c1/7879244da5b30407dc368946d36be5024380073408b079f144ffe034030e/ty-0.0.61-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0d78f160a0f9434d570cdcdbc4dafba1f6aac3c47a32f9f63995b3cb55ffe4b6", size = 11715250, upload-time = "2026-07-18T01:39:27.045Z" }, - { url = "https://files.pythonhosted.org/packages/35/c4/8a4637cd58abd37f315dd515e24c582986cb1bfdf2edc4786882f5a4f69a/ty-0.0.61-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09aeab4800b36e93e4ce918699004da642d74988cac920b7592a6a2b9be6611c", size = 12393876, upload-time = "2026-07-18T01:39:29.197Z" }, - { url = "https://files.pythonhosted.org/packages/27/4b/27e7c640b1272743503229aa17ae2167a538040c4716a2fa1777c2b34fea/ty-0.0.61-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dccc8136df44142a109953a168be17b4915c99876b047d0b6672c31dae939bdf", size = 12958187, upload-time = "2026-07-18T01:39:31.308Z" }, - { url = "https://files.pythonhosted.org/packages/3a/f5/70eaaefb6081fb0a8115cff66fbfaa20dafac8c646df2477adad95a59de2/ty-0.0.61-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:220760c2d13a887d027ee1093172c24ac35b6e634805329c93a30908ae4d3f5c", size = 12560101, upload-time = "2026-07-18T01:39:33.35Z" }, - { url = "https://files.pythonhosted.org/packages/e3/5a/17bae3b6429b5c479dc6c1e344d34e1f79efbc27531f15f3ee5b5da63745/ty-0.0.61-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:effefbb89da7128d18059529d1c2ea390fe7f1f3882690d257ca2143d49a0c34", size = 12225389, upload-time = "2026-07-18T01:39:35.436Z" }, - { url = "https://files.pythonhosted.org/packages/d2/0e/2ac380ba20d6395542c8df1d6fa4f00e2aead784c2e6aaefa1e02ed0610c/ty-0.0.61-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ba8b28a5ef811d5bb6461e37d76110c06fd20487474865c323d3d18b08b972b2", size = 12548403, upload-time = "2026-07-18T01:39:37.556Z" }, - { url = "https://files.pythonhosted.org/packages/b0/e5/7da4b73e825e1a9808c26d68b0156e9a37aede1846191210dfffb8c64042/ty-0.0.61-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:88ecd6d9b05e8174b1860dac9bd3e188d6cef5702b0d3239fd9f94f6ac73a29d", size = 11621813, upload-time = "2026-07-18T01:39:39.919Z" }, - { url = "https://files.pythonhosted.org/packages/9b/3c/5b58015e998cd0d89b17a463b6321421457d86d987574e8dac65ddfceba3/ty-0.0.61-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb0cdfe4c48542ffb9a1139825dfa3d4aae49e96e966682ef7da762ab97831ff", size = 11734101, upload-time = "2026-07-18T01:39:42.097Z" }, - { url = "https://files.pythonhosted.org/packages/a6/21/294f4cc819b7b12ed659fd860e5cdfbd592d4c768c8f23596685dbc43e6b/ty-0.0.61-py3-none-musllinux_1_2_i686.whl", hash = "sha256:dff03873c0c3d0b44738f8b6d403b0756a31cf54c65136397df7624c6159b1f0", size = 11988401, upload-time = "2026-07-18T01:39:44.183Z" }, - { url = "https://files.pythonhosted.org/packages/2e/26/0f96f79fdac118521a9771e9eef3f9b3f447d647b2c77953e80a1715c7e8/ty-0.0.61-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a9210e80e3d41c1dfc751e9e8e0980272f475031fafd0fb0f48aee233c78da03", size = 12330624, upload-time = "2026-07-18T01:39:46.662Z" }, - { url = "https://files.pythonhosted.org/packages/e3/08/1e62d1bca5c0cebdc7a34db1f4b61557aab85961cedd56953dd2c32d3e66/ty-0.0.61-py3-none-win32.whl", hash = "sha256:e3e1fe06f49a5492a922a5df2739834aa5ee978c7dd10414119dc8755cc40c9c", size = 11313991, upload-time = "2026-07-18T01:39:48.761Z" }, - { url = "https://files.pythonhosted.org/packages/26/f1/d8e33b3aeb36b73d81ae34d10e46ec4abf506d68f4e0a1491a76a593dd42/ty-0.0.61-py3-none-win_amd64.whl", hash = "sha256:25f2291169e0298fcdbba1b1fea64f8207a6c1908dddef32346fd5e3e6ac9221", size = 12311717, upload-time = "2026-07-18T01:39:50.881Z" }, - { url = "https://files.pythonhosted.org/packages/e1/14/7caec26d93a943c0e7d15eb7374644508d08cbd387d112b722b12d14e044/ty-0.0.61-py3-none-win_arm64.whl", hash = "sha256:3e496f7698bc4b5bbb1eb66d8b5799ba87596d88d36604ca359083893fa2fc49", size = 11693485, upload-time = "2026-07-18T01:39:52.73Z" }, +version = "0.0.90" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/68/b1/613b5534624442a9118828d7a9e54d50b16f0f5c8b67c80eb6623f33b921/ty-0.0.90.tar.gz", hash = "sha256:25cef63ec07a0254c2f5d885a63b001d0bfa437b20caa0a408f4f52e66b133e7", size = 7996206, upload-time = "2026-10-20T14:25:19.477Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/33/0e3395afa2c31c0e8e91547629858d2a5de213c69af237e6e50f37545565/ty-0.0.90-py3-none-linux_armv6l.whl", hash = "sha256:0dedb54977b40bc86638fec2c90eefd593f3a6683003a86b7e6cb67c17bcc476", size = 14281945, upload-time = "2026-10-20T14:24:21.018Z" }, + { url = "https://files.pythonhosted.org/packages/d7/fd/5d632b2f88d5c18991a4936da8643c7f204cfbc4980f8c7cd9080af988ad/ty-0.0.90-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3d7c30ceaac76dfbfcf36b72d36272d21c3898e98fc4de296aa9cbae956dec9d", size = 13786355, upload-time = "2026-10-20T14:24:24.392Z" }, + { url = "https://files.pythonhosted.org/packages/79/08/0d2f37bb4f766ab9630f01b07f4d11f1f7a6a6188f920d5ca9fa8a4676b1/ty-0.0.90-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c0b7e4d9ae0939294f0a18652de9ad7d81da2d7f567cf93f1edda1263008c124", size = 12856535, upload-time = "2026-10-20T14:24:27.386Z" }, + { url = "https://files.pythonhosted.org/packages/e4/77/70658f6b80c345ae0c12f2192a5cc89b8ca13eccfe06616b0fac3806c261/ty-0.0.90-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76c4a095a86a132df374e652894fe08392aeb1601d6347e1eb4b58808094a302", size = 13696342, upload-time = "2026-10-20T14:24:30.413Z" }, + { url = "https://files.pythonhosted.org/packages/b9/3e/693049f93653d2f6412bb0c2c0af1d01675e31643af29bef7177d3ff7e79/ty-0.0.90-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4e24a8aa07c05d9d14648289e79a88b7c7c2c522c58b53bb5f466a6ba4704e22", size = 14014180, upload-time = "2026-10-20T14:24:33.783Z" }, + { url = "https://files.pythonhosted.org/packages/27/08/19e373f97501eaadc8fc42e8c07cd54a64e375fbcf1470797be13590d5ba/ty-0.0.90-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40c3ea8fc1dfcf7502f71b93b12e80da1f7a8a5fd7e55ec9feabcf4ac39f72ea", size = 14935189, upload-time = "2026-10-20T14:24:37.459Z" }, + { url = "https://files.pythonhosted.org/packages/33/54/3331984f2c896a7383b6b18c86e968a63dd046baf503119a8e8e7682c752/ty-0.0.90-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1af75532bbe3e305089c30806fa5e804f8b01de231c1fa07918f7f3b335626da", size = 15311482, upload-time = "2026-10-20T14:24:40.87Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f3/179acf8d730c2c864767ffcc9081f5ea6fdb1b34ef711f1bd226200e9cff/ty-0.0.90-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f09449b34d837a044dab043aeb2b4b6ff2d2cef5eaf8116c7c19d34cc2a16a21", size = 15278844, upload-time = "2026-10-20T14:24:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/71/7e/661bc2fa20e0ac84c4d5f92e4c098ec667a0e920851a6d10c10a0251f0dc/ty-0.0.90-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc868ffa7dfcb3169eea3ba7d634e5a3283c7133fb932c6e47a2e16620d26ca5", size = 14558774, upload-time = "2026-10-20T14:24:47.814Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2e/3f65f61f4545f75255373be7646a606e004bb4cdc851e236bf17ca94bdc2/ty-0.0.90-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e2405206533dc896a2b711e781f2955d5883d85536f4df0ac262fca6517103c2", size = 14788158, upload-time = "2026-10-20T14:24:51.223Z" }, + { url = "https://files.pythonhosted.org/packages/1e/10/3c91a529591667b89d63c86806bfe2ac56d6989c7e0a09ee0964b7512deb/ty-0.0.90-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0dca4a29e8778285d2fc204af19254571486106fd14aed60769bfd272c795f6f", size = 13687177, upload-time = "2026-10-20T14:24:54.611Z" }, + { url = "https://files.pythonhosted.org/packages/21/b2/1b6d48606c67614bfb0c8e61a5d67a053ca1e3c5956ddb584f8569b8dc3d/ty-0.0.90-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d004b8e4e898fead929eceea669b73936444671748e0dd44e624cc109346c0dd", size = 14027780, upload-time = "2026-10-20T14:24:59.045Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c4/a0771debf725312524ca1434f8a146d37e573d45257ca4bebb9cc8e35bbc/ty-0.0.90-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a6be182da4281b4581c4a1b27c6a0caebeed7212e4bdc98c6b8fe0d9bc36dea0", size = 14355890, upload-time = "2026-10-20T14:25:03.034Z" }, + { url = "https://files.pythonhosted.org/packages/a7/4c/036724b6936ffc38e28ff5e20ab7972d609c65a3dd9645f3d10f57cfd16f/ty-0.0.90-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:0ba6fe0a24ff4cbc5034f3aa1c88b3ad1397315dd3369357e694fd157a11864e", size = 14692306, upload-time = "2026-10-20T14:25:07.461Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2a/c74fd5fc3fb5b046fe06a9067cabf5417b73e28249dc9af90ef72700f8fd/ty-0.0.90-py3-none-win32.whl", hash = "sha256:4c46a79caf84c4da0f7293e16bd769bd467c7cbc38015cac7608ab4e5ec8ba9c", size = 13235434, upload-time = "2026-10-20T14:25:10.545Z" }, + { url = "https://files.pythonhosted.org/packages/d2/54/be5cd6b17d8087a5e4c26892311c07e39d42b66c925b0f06cfc51a9ce83b/ty-0.0.90-py3-none-win_amd64.whl", hash = "sha256:8d77c16d20201b7a07ccaafb610eaf60488783dbba9865c4f9182e584f1426f8", size = 13855668, upload-time = "2026-10-20T14:25:13.648Z" }, + { url = "https://files.pythonhosted.org/packages/33/68/6dea089a897e8b314504eecc2633de3c8fe14f70e71a35606d14a2ba169e/ty-0.0.90-py3-none-win_arm64.whl", hash = "sha256:bfb63e74eeedb0a29d49cf58d611c540bfdfc0975dfce26efc6575bcf439208d", size = 13330853, upload-time = "2026-10-20T14:25:16.642Z" }, ] [[package]] From b84a31eee8373113ba20d9e89f4ff5b3d98cc58b Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 22 Oct 2026 02:42:49 +0000 Subject: [PATCH 2/8] Format with ruff --- README.md | 3 +- docs/registry/index.md | 20 ++-- docs/tables/mat_view.md | 20 ++-- src/orm_loader/backends/base.py | 23 ++-- src/orm_loader/backends/postgres.py | 26 +++-- src/orm_loader/backends/resolve.py | 4 +- src/orm_loader/backends/sqlite.py | 36 +++--- src/orm_loader/helpers/bootstrap.py | 8 +- src/orm_loader/helpers/bulk.py | 9 ++ src/orm_loader/helpers/discovery.py | 1 + src/orm_loader/helpers/metadata.py | 3 +- src/orm_loader/helpers/null_handlers.py | 1 + src/orm_loader/helpers/sqlite.py | 6 +- src/orm_loader/loaders/__init__.py | 6 +- src/orm_loader/loaders/data/__init__.py | 2 +- src/orm_loader/loaders/data/converters.py | 99 +++++++++------- src/orm_loader/loaders/data_classes.py | 29 +++-- src/orm_loader/loaders/loader_interface.py | 76 ++++++++----- src/orm_loader/loaders/loading_helpers.py | 78 +++++++------ src/orm_loader/mappers/__init__.py | 1 - .../mappers/materialised_view_mixin.py | 42 +++---- src/orm_loader/registry/__init__.py | 2 +- src/orm_loader/registry/registry.py | 26 ++--- src/orm_loader/registry/validation.py | 21 ++-- src/orm_loader/registry/validation_presets.py | 7 +- src/orm_loader/registry/validation_report.py | 21 ++-- src/orm_loader/registry/validation_runner.py | 12 +- src/orm_loader/tables/__init__.py | 2 +- src/orm_loader/tables/loadable_table.py | 64 ++++++----- src/orm_loader/tables/orm_table.py | 37 ++---- src/orm_loader/tables/serialisable_table.py | 9 +- src/orm_loader/tables/typing.py | 48 ++++++-- tests/backends/test_base_backend.py | 10 +- tests/backends/test_postgres_backend.py | 106 ++++++++++-------- tests/backends/test_shared_backend.py | 61 ++++++---- tests/backends/test_sqlite_backend.py | 9 +- tests/conftest.py | 15 ++- tests/helpers/test_bootstrap.py | 10 +- tests/loaders/test_casting.py | 70 +++++++++--- tests/loaders/test_dedupe.py | 13 ++- tests/loaders/test_helpers.py | 6 + tests/loaders/test_loader_e2e.py | 25 +++-- tests/loaders/test_pandas_loader.py | 52 +++------ tests/loaders/test_pg_loader.py | 101 ++++++++++------- tests/loaders/test_schema_translate_map.py | 26 ++++- tests/loaders/test_split_connection.py | 32 +++--- .../test_materialised_view_contracts.py | 7 +- tests/mappers/test_materialised_view_mixin.py | 37 +++--- tests/models.py | 2 +- tests/registry/test_model_descriptor.py | 6 +- tests/registry/test_registry.py | 9 +- tests/registry/test_validation_e2e.py | 16 +-- tests/tables/test_id_allocator.py | 2 + tests/tables/test_serialisable_table.py | 33 +++--- tests/tables/test_typing_protocols.py | 7 +- 55 files changed, 802 insertions(+), 595 deletions(-) diff --git a/README.md b/README.md index 6c0b0b2..5128227 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ It supports: ```python from orm_loader.tables import ORMTableBase + class MyTable(ORMTableBase, Base): __tablename__ = "my_table" - ``` You can inherit from it directly or pick it up through one of the higher-level mixins. @@ -55,7 +55,6 @@ Features include: ```python class MyTable(CSVLoadableTableInterface, ORMTableBase, Base): __tablename__ = "my_table" - ``` The main extension points here are loader choice, column mapping, and the normal SQLAlchemy model definitions themselves. Most downstream projects do not need to override much beyond `csv_columns()` and the model schema. diff --git a/docs/registry/index.md b/docs/registry/index.md index 6c41fa1..474fde0 100644 --- a/docs/registry/index.md +++ b/docs/registry/index.md @@ -66,14 +66,15 @@ from orm_loader.tables import ORMTableBase Base = declarative_base() + class Person(ORMTableBase, Base): __tablename__ = "person" # columns omitted for brevity + class VisitOccurrence(ORMTableBase, Base): __tablename__ = "visit_occurrence" # columns omitted for brevity - ``` ## 2. Create a ModelRegistry @@ -81,14 +82,9 @@ class VisitOccurrence(ORMTableBase, Base): The registry is the coordination object that holds models and specs. ```python - from orm_loader.registry import ModelRegistry -registry = ModelRegistry( - model_version="5.4", - model_name="OMOP" -) - +registry = ModelRegistry(model_version="5.4", model_name="OMOP") ``` ## 3. Load Specifications @@ -116,10 +112,12 @@ At this point, the registry knows: Models can be registered explicitly: ```python -registry.register_models([ - Person, - VisitOccurrence, -]) +registry.register_models( + [ + Person, + VisitOccurrence, + ] +) ``` Or discovered automatically from a package: diff --git a/docs/tables/mat_view.md b/docs/tables/mat_view.md index f03a7be..8ba8ae5 100644 --- a/docs/tables/mat_view.md +++ b/docs/tables/mat_view.md @@ -24,18 +24,12 @@ The class declaration is the integration point that downstream code can import a ```python class RecentObservationMV(Base, MaterializedViewMixin): __mv_name__ = "mv_recent_observation" - __mv_select__ = ( - sa.select( - Observation.observation_id, - Observation.person_id, - Observation.observation_date, - Observation.value_as_number, - ) - .where( - Observation.observation_date - >= sa.func.current_date() - sa.text("INTERVAL '30 days'") - ) - ) + __mv_select__ = sa.select( + Observation.observation_id, + Observation.person_id, + Observation.observation_date, + Observation.value_as_number, + ).where(Observation.observation_date >= sa.func.current_date() - sa.text("INTERVAL '30 days'")) __mv_dependencies__ = {"observation"} __mv_indexes__ = ( MaterializedViewIndex( @@ -120,12 +114,14 @@ class VocabSummaryMV(MaterializedViewMixin): __mv_select__ = ... __mv_schema_tag__ = Role.VOCAB.value # resolves via the connection's vocab schema + # Uses __mv_schema_tag__ ("primary" by default) via the connection's own schema_translate_map. RecentObservationMV.create_mv(engine) # Override for one call. RecentObservationMV.create_mv(engine, schema_tag=Role.VOCAB.value) + # Declaratively mapped: the mapped table's own schema is authoritative, no # __mv_schema_tag__ needed (or consulted) at all. class VocabPatientSummaryMV(Base, MaterializedViewMixin): diff --git a/src/orm_loader/backends/base.py b/src/orm_loader/backends/base.py index 016bb2c..7224256 100644 --- a/src/orm_loader/backends/base.py +++ b/src/orm_loader/backends/base.py @@ -38,10 +38,12 @@ class BackendCapabilities: STAGING_SCHEMA: str = "staging" -def staging_schema_claim(physical_schema: str | None = None, *, reserved: bool = True) -> SchemaClaim: +def staging_schema_claim( + physical_schema: str | None = None, *, reserved: bool = True +) -> SchemaClaim: """SchemaClaim reserving orm-loader's staging schema. Since orm-loader doesn't have an engine creation step of its own, - this claim needs to be registered by the caller ingesting tables into + this claim needs to be registered by the caller ingesting tables into the database through orm-loader. Add to your own ``create_engines(schema_claims=[...])`` call before @@ -99,7 +101,9 @@ class DatabaseBackend(ABC): without changing existing loader orchestration yet. """ - def __init__(self, *, staging_schema_tag: str | None = None, staging_schema: str | None = None) -> None: + def __init__( + self, *, staging_schema_tag: str | None = None, staging_schema: str | None = None + ) -> None: """ Parameters ---------- @@ -156,7 +160,9 @@ def qualified_staging_name(self, tablename: str) -> str: e.g. '"staging"."_staging_concept"' or '"_staging_concept"'. """ return qualified( - self.identifier_preparer, self.staging_name_for_table(tablename), physical_schema=self.staging_schema + self.identifier_preparer, + self.staging_name_for_table(tablename), + physical_schema=self.staging_schema, ) @property @@ -210,10 +216,8 @@ def _require_capability(self, capability_name: str, feature_name: str) -> None: f"Unknown backend capability {capability_name!r} on {type(self.capabilities).__name__}" ) if not getattr(self.capabilities, capability_name): - raise NotImplementedError( - f"Backend '{self.name}' does not support {feature_name}" - ) - + raise NotImplementedError(f"Backend '{self.name}' does not support {feature_name}") + @contextmanager def _as_connection( self, @@ -490,6 +494,5 @@ def create_materialized_view_index( the same compatibility reason as :meth:`drop_materialized_view`. """ raise NotImplementedError( - f"Backend '{self.name}' has not implemented " - "create_materialized_view_index()" + f"Backend '{self.name}' has not implemented create_materialized_view_index()" ) diff --git a/src/orm_loader/backends/postgres.py b/src/orm_loader/backends/postgres.py index 94eb39a..7c49533 100644 --- a/src/orm_loader/backends/postgres.py +++ b/src/orm_loader/backends/postgres.py @@ -35,7 +35,9 @@ class PostgresBackend(DatabaseBackend): - def __init__(self, *, staging_schema_tag: str | None = None, staging_schema: str | None = None) -> None: + def __init__( + self, *, staging_schema_tag: str | None = None, staging_schema: str | None = None + ) -> None: super().__init__(staging_schema_tag=staging_schema_tag, staging_schema=staging_schema) @staticmethod @@ -76,13 +78,15 @@ def create_staging_table( bind, table.name, physical_schema=physical_schema_of(bind, schema_tag=table.schema) ) bind_arguments = {"mapper": table_cls} - session.execute(sa.text(f'DROP TABLE IF EXISTS {staging_ref};'), bind_arguments=bind_arguments) + session.execute( + sa.text(f"DROP TABLE IF EXISTS {staging_ref};"), bind_arguments=bind_arguments + ) session.execute( sa.text( - f''' + f""" CREATE UNLOGGED TABLE {staging_ref} (LIKE {source_ref} INCLUDING DEFAULTS INCLUDING CONSTRAINTS); - ''' + """ ), bind_arguments=bind_arguments, ) @@ -90,14 +94,14 @@ def create_staging_table( computed_cols = [c.name for c in table.columns if c.computed is not None] for col in computed_cols: session.execute( - sa.text(f'ALTER TABLE {staging_ref} DROP COLUMN {preparer.quote_identifier(col)};'), + sa.text(f"ALTER TABLE {staging_ref} DROP COLUMN {preparer.quote_identifier(col)};"), bind_arguments=bind_arguments, ) # allows pagination in O(N log N) time for large tables in merge_insert without needing to add an index on every staging table session.execute( sa.text( - f'ALTER TABLE {staging_ref} ADD COLUMN _rownum BIGINT' + f"ALTER TABLE {staging_ref} ADD COLUMN _rownum BIGINT" f" GENERATED ALWAYS AS IDENTITY (CACHE 1000);" ), bind_arguments=bind_arguments, @@ -111,7 +115,7 @@ def drop_staging_table( session: so.Session, ) -> None: session.execute( - sa.text(f'DROP TABLE IF EXISTS {self.qualified_staging_name(table_cls.__tablename__)}'), + sa.text(f"DROP TABLE IF EXISTS {self.qualified_staging_name(table_cls.__tablename__)}"), bind_arguments={"mapper": table_cls}, ) @@ -440,7 +444,9 @@ def drop_materialized_view( try: conn.execute( DropMaterializedView( - qualified(conn, name, physical_schema=schema), if_exists=if_exists, cascade=cascade + qualified(conn, name, physical_schema=schema), + if_exists=if_exists, + cascade=cascade, ) ) except Exception as error: @@ -470,7 +476,9 @@ def create_materialized_view_index( try: conn.execute( CreateMaterializedViewIndex( - qualified(conn, name, physical_schema=schema), index, if_not_exists=if_not_exists + qualified(conn, name, physical_schema=schema), + index, + if_not_exists=if_not_exists, ) ) except Exception as error: diff --git a/src/orm_loader/backends/resolve.py b/src/orm_loader/backends/resolve.py index c2fee8e..d33e826 100644 --- a/src/orm_loader/backends/resolve.py +++ b/src/orm_loader/backends/resolve.py @@ -24,9 +24,7 @@ def _dialect(bindable: Bindable) -> Dialect: try: return Dialect(dialect_name) except ValueError as exc: - raise NotImplementedError( - f"Unsupported SQLAlchemy dialect '{dialect_name}'" - ) from exc + raise NotImplementedError(f"Unsupported SQLAlchemy dialect '{dialect_name}'") from exc def resolve_backend( diff --git a/src/orm_loader/backends/sqlite.py b/src/orm_loader/backends/sqlite.py index 6cc2753..a421100 100644 --- a/src/orm_loader/backends/sqlite.py +++ b/src/orm_loader/backends/sqlite.py @@ -21,9 +21,7 @@ logger = logging.getLogger(__name__) -VALID_SQLITE_JOURNAL_MODES = frozenset( - {"DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"} -) +VALID_SQLITE_JOURNAL_MODES = frozenset({"DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"}) class SQLiteBackend(DatabaseBackend): @@ -74,8 +72,7 @@ def _normalize_fk_check_state(previous_state: str | int) -> str: if normalised in {"0", "OFF"}: return "OFF" raise ValueError( - f"Invalid SQLite foreign_keys state {previous_state!r}. " - "Expected 0, 1, 'OFF', or 'ON'." + f"Invalid SQLite foreign_keys state {previous_state!r}. Expected 0, 1, 'OFF', or 'ON'." ) @property @@ -110,14 +107,15 @@ def create_staging_table( ) -> None: staging_name = self.staging_name_for_table(table_cls.__tablename__) session.execute( - sa.text(f'DROP TABLE IF EXISTS {self.identifier_preparer.quote_identifier(staging_name)};'), + sa.text( + f"DROP TABLE IF EXISTS {self.identifier_preparer.quote_identifier(staging_name)};" + ), bind_arguments={"mapper": table_cls}, ) metadata = sa.MetaData() staging_columns = [ - sa.Column(col.name, col.type, nullable=True) - for col in table_cls.__table__.columns + sa.Column(col.name, col.type, nullable=True) for col in table_cls.__table__.columns ] staging_table = sa.Table( staging_name, metadata, *staging_columns, schema=table_cls.__table__.schema @@ -132,9 +130,11 @@ def drop_staging_table( table_cls: type["CSVTableProtocol"], session: so.Session, ) -> None: - staging_ref = self.identifier_preparer.quote_identifier(self.staging_name_for_table(table_cls.__tablename__)) + staging_ref = self.identifier_preparer.quote_identifier( + self.staging_name_for_table(table_cls.__tablename__) + ) session.execute( - sa.text(f'DROP TABLE IF EXISTS {staging_ref}'), + sa.text(f"DROP TABLE IF EXISTS {staging_ref}"), bind_arguments={"mapper": table_cls}, ) @@ -202,9 +202,7 @@ def _delete(extra: sa.ColumnElement[bool] | None = None) -> sa.Delete: keys = keys.where(extra) if len(pk_cols) == 1: return sa.delete(target).where(target.c[pk_cols[0]].in_(keys)) - return sa.delete(target).where( - sa.tuple_(*(target.c[c] for c in pk_cols)).in_(keys) - ) + return sa.delete(target).where(sa.tuple_(*(target.c[c] for c in pk_cols)).in_(keys)) if merge_batch_size is None: session.execute(_delete()) @@ -296,7 +294,9 @@ def _insert(select_: Any) -> sa.Insert: session.execute(_insert(batch_select)) start = end - def configure_dbapi_connection(self, dbapi_connection: sa.engine.interfaces.DBAPIConnection) -> None: + def configure_dbapi_connection( + self, dbapi_connection: sa.engine.interfaces.DBAPIConnection + ) -> None: if dbapi_connection.__class__.__module__.startswith("sqlite3"): cursor = dbapi_connection.cursor() cursor.execute(f"PRAGMA busy_timeout = {self.busy_timeout_ms}") @@ -309,8 +309,7 @@ def configure_dbapi_connection(self, dbapi_connection: sa.engine.interfaces.DBA def install_engine_hooks(self, engine: "Engine") -> None: @event.listens_for(engine, "connect") def _enable_sqlite_foreign_keys( - dbapi_connection: sa.engine.interfaces.DBAPIConnection, - _connection_record: Any + dbapi_connection: sa.engine.interfaces.DBAPIConnection, _connection_record: Any ) -> None: self.configure_dbapi_connection(dbapi_connection) @@ -333,7 +332,10 @@ def explain_fk_error( for row in rows: logger.error( "FK violation: table=%s rowid=%s references=%s fk_index=%s", - row[0], row[1], row[2], row[3] + row[0], + row[1], + row[2], + row[3], ) if raise_error: diff --git a/src/orm_loader/helpers/bootstrap.py b/src/orm_loader/helpers/bootstrap.py index 77ccf14..0b44d00 100644 --- a/src/orm_loader/helpers/bootstrap.py +++ b/src/orm_loader/helpers/bootstrap.py @@ -21,7 +21,9 @@ logger = logging.getLogger(__name__) -def _resolve_binds(resolved: ResolvedDatabase, bindable: Bindable | None) -> tuple[Bindable, Bindable]: +def _resolve_binds( + resolved: ResolvedDatabase, bindable: Bindable | None +) -> tuple[Bindable, Bindable]: """Return (primary_bind, vocab_bind) to run DDL against. Built from resolved.create_engines()/create_engine() when bindable is @@ -115,9 +117,7 @@ def create_tables( candidate for candidate in metadata.tables.values() if candidate.schema is not None - and resolved.tags_share_a_transaction( - candidate.schema, cast(str, table.schema) - ) + and resolved.tags_share_a_transaction(candidate.schema, cast(str, table.schema)) ) for metadata, related in related_by_metadata.items(): diff --git a/src/orm_loader/helpers/bulk.py b/src/orm_loader/helpers/bulk.py index 459ccf5..533f7e5 100644 --- a/src/orm_loader/helpers/bulk.py +++ b/src/orm_loader/helpers/bulk.py @@ -6,9 +6,11 @@ logger = logging.getLogger(__name__) + def disable_fk_check(session: Session) -> str | int: """Disable foreign-key checks for the current session and return the previous state.""" from ..backends.resolve import resolve_backend + previous_state = resolve_backend(session).disable_fk_check(session) logger.info("Disabled foreign key checks for bulk load.") if not isinstance(previous_state, (str, int)): @@ -16,9 +18,11 @@ def disable_fk_check(session: Session) -> str | int: raise TypeError(f"Expected previous FK state to be str or int, got {type(previous_state)}") return previous_state + def enable_fk_check(session: Session) -> str | int: """Enable foreign-key checks for the current session and return the previous state.""" from ..backends.resolve import resolve_backend + previous_state = resolve_backend(session).enable_fk_check(session) logger.info("Explicitly re-enabled foreign key checks.") if not isinstance(previous_state, (str, int)): @@ -26,12 +30,15 @@ def enable_fk_check(session: Session) -> str | int: raise TypeError(f"Expected previous FK state to be str or int, got {type(previous_state)}") return previous_state + def restore_fk_check(session: Session, previous_state: str | int): """Restore foreign-key checks to a previously captured backend-specific state.""" from ..backends.resolve import resolve_backend + resolve_backend(session).restore_fk_check(session, previous_state) logger.info(f"Restored foreign key checks to state: {previous_state}") + @contextmanager def bulk_load_context( session: Session, @@ -46,6 +53,7 @@ def bulk_load_context( It exists so older call sites can keep using the helper import path. """ from ..backends.resolve import resolve_backend + backend = resolve_backend(session) with backend.bulk_load_context( session, @@ -64,6 +72,7 @@ def engine_with_replica_role(engine: Engine) -> Generator[Engine, None, None]: on backends that explicitly implement the behaviour. """ from ..backends.resolve import resolve_backend + backend = resolve_backend(engine) method = getattr(backend, "engine_with_replica_role", None) if method is None: diff --git a/src/orm_loader/helpers/discovery.py b/src/orm_loader/helpers/discovery.py index 6ee411f..5f8ae5b 100644 --- a/src/orm_loader/helpers/discovery.py +++ b/src/orm_loader/helpers/discovery.py @@ -3,6 +3,7 @@ ModelT = TypeVar("ModelT", bound=Base) + def get_model_by_tablename( tablename: str, base: type[ModelT] = Base, # ty: ignore[invalid-parameter-default] diff --git a/src/orm_loader/helpers/metadata.py b/src/orm_loader/helpers/metadata.py index 3ad0a84..cebf426 100644 --- a/src/orm_loader/helpers/metadata.py +++ b/src/orm_loader/helpers/metadata.py @@ -11,5 +11,6 @@ metadata = MetaData(naming_convention=NAMING_CONVENTIONS) + class Base(DeclarativeBase): - metadata = metadata \ No newline at end of file + metadata = metadata diff --git a/src/orm_loader/helpers/null_handlers.py b/src/orm_loader/helpers/null_handlers.py index 09a5464..fb3face 100644 --- a/src/orm_loader/helpers/null_handlers.py +++ b/src/orm_loader/helpers/null_handlers.py @@ -3,6 +3,7 @@ _NULL_STRINGS = {"", "nan", "null", "none", "na", "n/a"} + def normalise_null(value: Any) -> Any | None: if value is None: return None diff --git a/src/orm_loader/helpers/sqlite.py b/src/orm_loader/helpers/sqlite.py index f3e2f24..1cab657 100644 --- a/src/orm_loader/helpers/sqlite.py +++ b/src/orm_loader/helpers/sqlite.py @@ -20,7 +20,7 @@ def attach_sqlite_bulk_load_pragmas( enforcement, and can also enable deferred foreign-key checking for the connection. - Note that this is the replacement for old ``enable_sqlite_foreign_keys()`` + Note that this is the replacement for old ``enable_sqlite_foreign_keys()`` workaround - this should be no longer needed. """ SQLiteBackend( @@ -30,9 +30,7 @@ def attach_sqlite_bulk_load_pragmas( ).install_engine_hooks(engine) -def explain_sqlite_fk_error( - session, exc: IntegrityError, raise_error: bool = True, *, mapper=None -): +def explain_sqlite_fk_error(session, exc: IntegrityError, raise_error: bool = True, *, mapper=None): """Log SQLite foreign-key check details before re-raising an error.""" SQLiteBackend().explain_fk_error(session, exc, mapper=mapper, raise_error=raise_error) diff --git a/src/orm_loader/loaders/__init__.py b/src/orm_loader/loaders/__init__.py index 2be4aa2..b1a84eb 100644 --- a/src/orm_loader/loaders/__init__.py +++ b/src/orm_loader/loaders/__init__.py @@ -3,12 +3,12 @@ from .loading_helpers import infer_delim, infer_encoding, quick_load_pg __all__ = [ - "LoaderInterface", - "LoaderContext", + "LoaderInterface", + "LoaderContext", "PandasLoader", "TableCastingStats", "infer_delim", "infer_encoding", "quick_load_pg", "ParquetLoader", -] \ No newline at end of file +] diff --git a/src/orm_loader/loaders/data/__init__.py b/src/orm_loader/loaders/data/__init__.py index 782e1cc..8fbe877 100644 --- a/src/orm_loader/loaders/data/__init__.py +++ b/src/orm_loader/loaders/data/__init__.py @@ -3,4 +3,4 @@ __all__ = [ "perform_cast", "cast_arrow_column", -] \ No newline at end of file +] diff --git a/src/orm_loader/loaders/data/converters.py b/src/orm_loader/loaders/data/converters.py index c162a0e..6b5643e 100644 --- a/src/orm_loader/loaders/data/converters.py +++ b/src/orm_loader/loaders/data/converters.py @@ -16,10 +16,10 @@ _NUMERIC_RE = re.compile(r"^[+-]?\d+(\.\d+)?$") _AVAILABLE_DATE_FORMATS = ( - "%Y%m%d", # 20170824 (athena standard) - "%d-%b-%Y", # 24-AUG-2017 (oncology-branch vocab) - "%Y-%m-%d", # 2017-08-24 (ISO) - "%d/%m/%Y", # 24/08/2017 + "%Y%m%d", # 20170824 (athena standard) + "%d-%b-%Y", # 24-AUG-2017 (oncology-branch vocab) + "%Y-%m-%d", # 2017-08-24 (ISO) + "%d/%m/%Y", # 24/08/2017 ) @@ -31,11 +31,14 @@ DateTime: pa.timestamp("us"), } + @dataclass(frozen=True) class CastRule: sa_type: type scalar: Callable[[Any, Any], Any] - arrow: Callable[..., Any] | None = None # optional vectorised impl, never actually set/read today + arrow: Callable[..., Any] | None = ( + None # optional vectorised impl, never actually set/read today + ) _NULL_STRINGS = { @@ -46,10 +49,11 @@ class CastRule: "nan", } + def _normalise_null(value: Any) -> Any | None: if value is None: return None - + if value is pd.NA or value is pd.NaT: return None @@ -73,16 +77,16 @@ def _normalise_null_arrow(arr: pa.Array | pa.ChunkedArray) -> pa.Array | pa.Chun arrow_type = arr.type if pa.types.is_string(arrow_type) or pa.types.is_large_string(arrow_type): - probe = pc.utf8_lower(pc.utf8_trim_whitespace(arr)) # type: ignore - return pc.if_else( # type: ignore - pc.is_in(probe, value_set=_ARROW_NULL_STRINGS), # type: ignore + probe = pc.utf8_lower(pc.utf8_trim_whitespace(arr)) # type: ignore + return pc.if_else( # type: ignore + pc.is_in(probe, value_set=_ARROW_NULL_STRINGS), # type: ignore pa.nulls(len(arr), arrow_type), arr, ) if pa.types.is_floating(arrow_type): - return pc.if_else( # type: ignore - pc.is_nan(arr), # type: ignore + return pc.if_else( # type: ignore + pc.is_nan(arr), # type: ignore pa.nulls(len(arr), arrow_type), arr, ) @@ -95,7 +99,7 @@ def _to_numeric_string(value: str | None) -> str | None: return None if not _NUMERIC_RE.match(value): - return value + return value if "." in value: f = float(value) @@ -105,6 +109,7 @@ def _to_numeric_string(value: str | None) -> str | None: return str(int(value)) + def _to_number(value: Any) -> int | float | None: if value is None: return None @@ -127,7 +132,7 @@ def _to_number(value: Any) -> int | float | None: s = _to_numeric_string(s) if s is None: return None - + return int(s) @@ -141,6 +146,7 @@ def _to_int(value: Any) -> int | None: raise ValueError(f"Non-integer numeric value: {value}") + def _cast_string(value: Any, sa_type: TypeEngine[Any]) -> str | None: if value is None: return None @@ -160,6 +166,7 @@ def _cast_string(value: Any, sa_type: TypeEngine[Any]) -> str | None: return s + def _to_float(value: Any) -> float | None: if value is None: return None @@ -186,12 +193,12 @@ def _to_float(value: Any) -> float | None: CAST_RULES: list[CastRule] = [ CastRule(Integer, lambda v, _: _to_int(v) if v is not None else None), - CastRule(Float, lambda v, _: _to_float(v) if v is not None else None), + CastRule(Float, lambda v, _: _to_float(v) if v is not None else None), CastRule(Boolean, lambda v, _: _to_bool(v)), - CastRule(Date, lambda v, _: _parse_date(v)), - CastRule(DateTime,lambda v, _: _parse_datetime(v)), - CastRule(String, _cast_string), - CastRule(Text, _cast_string), + CastRule(Date, lambda v, _: _parse_date(v)), + CastRule(DateTime, lambda v, _: _parse_datetime(v)), + CastRule(String, _cast_string), + CastRule(Text, _cast_string), ] # Per-column overrides, keyed by (table_name, column_name). Checked ahead of @@ -199,7 +206,7 @@ def _to_float(value: Any) -> float | None: # target one specific column (e.g. a plain String(1) column with no # type-level signal of the enum it's meant to hold) or safely disambiguate # sa.Enum from its own String supertype without a caller telling it which -# column actually means "enum" +# column actually means "enum" _COLUMN_CAST_RULES: dict[tuple[str, str], Callable[[Any], Any]] = {} @@ -247,6 +254,7 @@ def _scalar(value: Any) -> Any: # Raise rather than return None, so an unmatched value goes through # the on_error/TableCastingStats path like every other cast failure. raise ValueError(f"{value!r} matches no {enum_type.__name__} member value") + return _scalar @@ -264,7 +272,7 @@ def register_column_cast_rule( by `.value` and coerced to the matching member's `.name`. Member values need not be strings -- an `IntEnum` matches both `1` (a parquet int64 column) and `"1"` (any CSV column, since those are read with dtype=str). - + This is `sa.Enum`'s default column-storage convention, and covers genuine `sa.Enum(enum_type)` columns with no further setup. An unmatched value goes through the existing on_error/TableCastingStats path and the @@ -275,33 +283,38 @@ def register_column_cast_rule( convention at all (e.g. OMOP CDM's `concept.standard_concept`: `String(1)`, values 'S'/'C'/NULL, where the column expects `.value` directly, not `.name`), or a genuine `sa.Enum` column configured with a - custom `values_callable` such as in HemOnc) use `scalar` instead. - Resolution logic for the scalar callable is then tailored to what + custom `values_callable` such as in HemOnc) use `scalar` instead. + Resolution logic for the scalar callable is then tailored to what that specific column actually expects to store (e.g. `lambda v: SomeEnum(v).value` for the plain-String case above). - + Registering the same (table_name, column_name) again replaces the previous rule rather than stacking, so repeated registration is safe/idempotent. """ if enum_type is not None: if scalar is not None: - raise ValueError("register_column_cast_rule requires exactly one of `scalar` or `enum_type`.") + raise ValueError( + "register_column_cast_rule requires exactly one of `scalar` or `enum_type`." + ) resolved_scalar = _enum_member_scalar(enum_type) elif scalar is not None: resolved_scalar = scalar else: - raise ValueError("register_column_cast_rule requires exactly one of `scalar` or `enum_type`.") + raise ValueError( + "register_column_cast_rule requires exactly one of `scalar` or `enum_type`." + ) _COLUMN_CAST_RULES[(table_name, column_name)] = resolved_scalar + def _dateutil_fallback(value: str) -> datetime | None: try: dt = parser.parse( value, dayfirst=True, yearfirst=False, - fuzzy=False, + fuzzy=False, ) except (ValueError, OverflowError): return None @@ -312,6 +325,7 @@ def _dateutil_fallback(value: str) -> datetime | None: return dt + def _parse_date(value: Any) -> date | None: if isinstance(value, date) and not isinstance(value, datetime): return value @@ -325,6 +339,7 @@ def _parse_date(value: Any) -> date | None: continue return None + def _parse_datetime(value: Any) -> datetime | None: if isinstance(value, datetime): return value @@ -343,6 +358,7 @@ def _parse_datetime(value: Any) -> datetime | None: return _dateutil_fallback(value) + def _to_bool(value: Any) -> bool | None: if value is None or (isinstance(value, float) and math.isnan(value)): return None @@ -355,6 +371,7 @@ def _to_bool(value: Any) -> bool | None: return False return None + def cast_scalar( value: Any, sa_type: TypeEngine[Any], @@ -367,7 +384,9 @@ def cast_scalar( if value is None: return None - column_rule = _COLUMN_CAST_RULES.get((table_name, column_name)) if table_name and column_name else None + column_rule = ( + _COLUMN_CAST_RULES.get((table_name, column_name)) if table_name and column_name else None + ) if column_rule is not None: try: return column_rule(value) @@ -396,10 +415,14 @@ def perform_cast( table_name: str | None = None, column_name: str | None = None, ) -> Any: - return cast_scalar(value, sa_type, on_error=on_error, table_name=table_name, column_name=column_name) + return cast_scalar( + value, sa_type, on_error=on_error, table_name=table_name, column_name=column_name + ) -def cast_arrow_column(arr: pa.Array, sa_col: ColumnElement[Any], stats: TableCastingStats | None = None) -> pa.Array: +def cast_arrow_column( + arr: pa.Array, sa_col: ColumnElement[Any], stats: TableCastingStats | None = None +) -> pa.Array: arr = _normalise_null_arrow(arr) column_rule = _COLUMN_CAST_RULES.get((sa_col.table.name, sa_col.name)) @@ -421,26 +444,22 @@ def cast_arrow_column(arr: pa.Array, sa_col: ColumnElement[Any], stats: TableCas arrow_type = _ARROW_TYPE_MAP.get(rule.sa_type) if arrow_type: try: - return pc.cast(arr, arrow_type) + return pc.cast(arr, arrow_type) except pa.ArrowInvalid: - validity = pc.is_valid(arr) # type: ignore - invalid_mask = pc.invert(validity) # type: ignore - invalid_count = pc.sum(invalid_mask).as_py() # type: ignore + validity = pc.is_valid(arr) # type: ignore + invalid_mask = pc.invert(validity) # type: ignore + invalid_count = pc.sum(invalid_mask).as_py() # type: ignore if invalid_count == 0: return arr - bad_values = [ - v.as_py() - for v, bad in zip(arr, invalid_mask) - if bad - ][:3] + bad_values = [v.as_py() for v, bad in zip(arr, invalid_mask) if bad][:3] if stats: stats.record( column=sa_col.name, value={ "count": invalid_count, "examples": bad_values, - "reason": f"Arrow cast to {arrow_type} failed" + "reason": f"Arrow cast to {arrow_type} failed", }, ) return arr @@ -449,4 +468,4 @@ def cast_arrow_column(arr: pa.Array, sa_col: ColumnElement[Any], stats: TableCas [rule.scalar(v.as_py(), sa_col) for v in arr], type=arr.type, ) - return arr \ No newline at end of file + return arr diff --git a/src/orm_loader/loaders/data_classes.py b/src/orm_loader/loaders/data_classes.py index e36e9b9..9596ff1 100644 --- a/src/orm_loader/loaders/data_classes.py +++ b/src/orm_loader/loaders/data_classes.py @@ -12,6 +12,7 @@ if TYPE_CHECKING: from ..tables.typing import CSVTableProtocol + def _clean_nulls(v): if v is None: return None @@ -20,6 +21,8 @@ def _clean_nulls(v): if isinstance(v, float) and pd.isna(v): return None return v + + """ Loader Data Structures ====================== @@ -38,7 +41,6 @@ def _clean_nulls(v): @dataclass(frozen=True) class LoaderContext: - """ Immutable context object passed through loader operations. @@ -69,6 +71,7 @@ class LoaderContext: resolve_backend() so every backend resolution within this load resolves the same schema. """ + tableclass: Type["CSVTableProtocol"] session: so.Session path: Path @@ -80,8 +83,8 @@ class LoaderContext: quote_mode: str = "auto" staging_schema_tag: str | None = None -class LoaderInterface: +class LoaderInterface: """ Abstract interface for file loaders. @@ -141,24 +144,18 @@ def _load_chunk( return 0 raw_records = dataframe.to_dict(orient="records") - records = [ - { - f'{k}': _clean_nulls(v) - for k, v in record.items() - } - for record in raw_records - ] + records = [{f"{k}": _clean_nulls(v) for k, v in record.items()} for record in raw_records] - session.execute( + session.execute( staging_cls.insert(), - records, + records, bind_arguments={"mapper": mapper}, ) session.flush() session.expunge_all() return len(dataframe) - + @classmethod def dedupe(cls, data: pd.DataFrame | pa.Table, ctx: LoaderContext) -> Any: """ @@ -179,13 +176,14 @@ def dedupe(cls, data: pd.DataFrame | pa.Table, ctx: LoaderContext) -> Any: Deduplicated data. """ raise NotImplementedError - + @dataclass class ColumnCastingStats: """ Casting statistics for a single column. """ + count: int = 0 examples: List[Any] = field(default_factory=list) @@ -194,11 +192,13 @@ def record(self, value: Any, example_limit: int = 3): if len(self.examples) < example_limit: self.examples.append(value) + @dataclass class TableCastingStats: """ Aggregated casting statistics for a table, keyed by column. """ + table_name: str columns: Dict[str, ColumnCastingStats] = field(default_factory=dict) @@ -228,7 +228,7 @@ def has_failures(self) -> bool: Whether any casting failures occurred. """ return self.total_failures > 0 - + def to_dict(self) -> dict[str, dict[str, Any]]: """ Return a dictionary representation of casting statistics. @@ -240,4 +240,3 @@ def to_dict(self) -> dict[str, dict[str, Any]]: } for col, stats in self.columns.items() } - diff --git a/src/orm_loader/loaders/loader_interface.py b/src/orm_loader/loaders/loader_interface.py index 2b02fee..2ba5213 100644 --- a/src/orm_loader/loaders/loader_interface.py +++ b/src/orm_loader/loaders/loader_interface.py @@ -8,7 +8,13 @@ import pyarrow.compute as pc from functools import reduce from .data_classes import LoaderContext, TableCastingStats, LoaderInterface -from .loading_helpers import infer_delim, infer_encoding, conservative_load_parquet, arrow_drop_duplicates, resolve_quote_mode +from .loading_helpers import ( + infer_delim, + infer_encoding, + conservative_load_parquet, + arrow_drop_duplicates, + resolve_quote_mode, +) from .data import perform_cast, cast_arrow_column from ..helpers import IngestError @@ -39,7 +45,10 @@ """ -def _require_columns_present(tableclass: Type["CSVTableProtocol"], present_columns: Iterable[str]) -> None: + +def _require_columns_present( + tableclass: Type["CSVTableProtocol"], present_columns: Iterable[str] +) -> None: """ Raise before staging if a genuinely required column is entirely absent from the incoming data. @@ -54,14 +63,15 @@ def _require_columns_present(tableclass: Type["CSVTableProtocol"], present_colum f"{tableclass.__tablename__}: source data is missing required column(s) {missing} (non-nullable, no default)" ) + @staticmethod def _normalise_columns(df: pd.DataFrame) -> pd.DataFrame: df = df.copy() - df.columns = [c.lower().replace('_hash', '').strip() for c in df.columns] + df.columns = [c.lower().replace("_hash", "").strip() for c in df.columns] return df -class PandasLoader(LoaderInterface): +class PandasLoader(LoaderInterface): """ For initial dataloads, pandasloader may not be sufficiently performant for very large files. However, it provides a very flexible and easy-to-debug pathway for data ingestion, especially @@ -79,10 +89,12 @@ def dedupe(cls, data: pd.DataFrame | pa.Table, ctx: LoaderContext) -> Any: return df pk_names = ctx.tableclass.pk_names() before = len(df) - df = df.drop_duplicates(subset=pk_names, keep='first') + df = df.drop_duplicates(subset=pk_names, keep="first") dropped_internal = before - len(df) if dropped_internal > 0: - logger.info(f"Dropped {dropped_internal} duplicate rows internally in staging for {ctx.tableclass.__tablename__}") + logger.info( + f"Dropped {dropped_internal} duplicate rows internally in staging for {ctx.tableclass.__tablename__}" + ) return df.copy() @classmethod @@ -93,7 +105,7 @@ def cast_to_model(cls, data: pd.DataFrame | pa.Table, ctx: LoaderContext) -> Any df = data if df.empty: return df - + table_name = ctx.tableclass.__tablename__ stats = TableCastingStats(table_name=table_name) @@ -107,7 +119,11 @@ def _on_cast_error(value, *, _col=col_name): df[col_name] = df[col_name].map( lambda v: perform_cast( - v, sa_col.type, on_error=_on_cast_error, table_name=table_name, column_name=col_name + v, + sa_col.type, + on_error=_on_cast_error, + table_name=table_name, + column_name=col_name, ) ) @@ -129,10 +145,12 @@ def _on_cast_error(value, *, _col=col_name): df = df.loc[~null_mask.any(axis=1)] if stats.has_failures(): for col, col_stats in stats.columns.items(): - logger.warning(f"CAST {table_name}.{col}: {col_stats.count} row(s) failed. Examples: {col_stats.examples}") + logger.warning( + f"CAST {table_name}.{col}: {col_stats.count} row(s) failed. Examples: {col_stats.examples}" + ) return df - + @classmethod def orm_file_load(cls, ctx: LoaderContext) -> int: """ @@ -143,7 +161,7 @@ def orm_file_load(cls, ctx: LoaderContext) -> int: """ delimiter = infer_delim(ctx.path) - encoding = infer_encoding(ctx.path)['encoding'] + encoding = infer_encoding(ctx.path)["encoding"] # Resolve to the same concrete mode the COPY fast-path would use, so a # file that falls back here is parsed identically. Without this the @@ -163,17 +181,21 @@ def orm_file_load(cls, ctx: LoaderContext) -> int: quoting=quoting, ) except pd.errors.EmptyDataError: - logger.info(f"File {ctx.path.name} is empty — skipping load for {ctx.tableclass.__tablename__}") + logger.info( + f"File {ctx.path.name} is empty — skipping load for {ctx.tableclass.__tablename__}" + ) return 0 - + logger.info(f"Detected encoding {encoding} for file {ctx.path.name}") - logger.info(f"Detected delimiter '{delimiter}' for file {ctx.path.name}") - logger.info(f"Loading with chunksize '{ctx.chunksize}' for file {ctx.path.name}") + logger.info(f"Detected delimiter '{delimiter}' for file {ctx.path.name}") + logger.info(f"Loading with chunksize '{ctx.chunksize}' for file {ctx.path.name}") chunks = (reader,) if isinstance(reader, pd.DataFrame) else reader total = 0 for i, chunk in enumerate(chunks): - logger.debug(f"Processing chunk {i} with {len(chunk)} rows for {ctx.tableclass.__tablename__}") + logger.debug( + f"Processing chunk {i} with {len(chunk)} rows for {ctx.tableclass.__tablename__}" + ) chunk = _normalise_columns(chunk) if ctx.dedupe: chunk = cls.dedupe(chunk, ctx) @@ -187,8 +209,8 @@ def orm_file_load(cls, ctx: LoaderContext) -> int: ) return total -class ParquetLoader(LoaderInterface): +class ParquetLoader(LoaderInterface): """ Overhead from this loader is worthwhile with processing and cleaning of very large files. """ @@ -197,7 +219,7 @@ class ParquetLoader(LoaderInterface): def cast_to_model(cls, data: pa.Table, ctx: LoaderContext) -> pa.Table: if data.num_rows == 0: return data - + table_name = ctx.tableclass.__tablename__ stats = TableCastingStats(table_name=table_name) model_columns = ctx.tableclass.model_columns() @@ -218,10 +240,10 @@ def cast_to_model(cls, data: pa.Table, ctx: LoaderContext) -> pa.Table: required_cols = list(ctx.tableclass.required_columns()) if required_cols: - masks = [pc.is_valid(out[c]) for c in required_cols] # type: ignore - valid_mask = reduce(pc.and_, masks) # type: ignore + masks = [pc.is_valid(out[c]) for c in required_cols] # type: ignore + valid_mask = reduce(pc.and_, masks) # type: ignore - dropped = out.num_rows - pc.sum(valid_mask).as_py() # type: ignore + dropped = out.num_rows - pc.sum(valid_mask).as_py() # type: ignore if dropped > 0: logger.warning( "Dropped %d rows with unexpected nulls in %s", @@ -233,7 +255,9 @@ def cast_to_model(cls, data: pa.Table, ctx: LoaderContext) -> pa.Table: if stats.has_failures(): for col, col_stats in stats.columns.items(): - logger.warning(f"CAST {table_name}.{col}: {col_stats.count} failures. Examples: {col_stats.examples}") + logger.warning( + f"CAST {table_name}.{col}: {col_stats.count} failures. Examples: {col_stats.examples}" + ) return out @@ -264,11 +288,12 @@ def _scan_batches(cls, ctx: LoaderContext): yield from dataset.to_batches(batch_size=ctx.chunksize or 64_000) elif suffix in {".csv", ".tsv"}: - yield from conservative_load_parquet(ctx.path, wanted_cols=wanted_cols, chunksize=ctx.chunksize) + yield from conservative_load_parquet( + ctx.path, wanted_cols=wanted_cols, chunksize=ctx.chunksize + ) else: raise ValueError(f"Unsupported file type: {ctx.path}") - @classmethod def orm_file_load(cls, ctx: LoaderContext) -> int: total = 0 @@ -284,9 +309,8 @@ def orm_file_load(cls, ctx: LoaderContext) -> int: if isinstance(data, pa.RecordBatch) or isinstance(data, pa.Table): df = data.to_pandas() else: - df = data + df = data if df.empty: - continue total += cls._load_chunk( diff --git a/src/orm_loader/loaders/loading_helpers.py b/src/orm_loader/loaders/loading_helpers.py index dbf5bbf..20169b0 100644 --- a/src/orm_loader/loaders/loading_helpers.py +++ b/src/orm_loader/loaders/loading_helpers.py @@ -15,7 +15,7 @@ import sqlalchemy.orm as so from oa_configurator import qualified -_SAFE_ENCODING = re.compile(r'^[A-Za-z][A-Za-z0-9_-]*$') +_SAFE_ENCODING = re.compile(r"^[A-Za-z][A-Za-z0-9_-]*$") logger = logging.getLogger(__name__) COPY_BLOCK_SIZE = 8192 @@ -35,6 +35,7 @@ These helpers are intentionally low-level and stateless. """ + class NormalisedCSVStream(io.RawIOBase): def __init__(self, f, encoding: str, delimiter: str): self._f = f @@ -58,7 +59,7 @@ def read(self, size=-1): header = self._f.readline().decode(self._encoding) newline = check_line_ending(header) cols = header.rstrip(newline).split(self._delimiter) - lowered = [c.strip().strip('"').lower().replace('_hash', '') for c in cols] + lowered = [c.strip().strip('"').lower().replace("_hash", "") for c in cols] new_header = (self._delimiter.join(lowered) + "\n").encode(self._encoding) out.extend(new_header) self._sent_header = True @@ -78,21 +79,25 @@ def read(self, size=-1): return bytes(out) + def infer_encoding(file): - with open(file, 'rb') as infile: + with open(file, "rb") as infile: encoding = chardet.detect(infile.read(10000)) - if encoding['encoding'] == 'ascii': - encoding['encoding'] = 'utf-8' # utf-8 valid superset of ascii, so being more conservative here just because it flakes occasionally + if encoding["encoding"] == "ascii": + encoding["encoding"] = ( + "utf-8" # utf-8 valid superset of ascii, so being more conservative here just because it flakes occasionally + ) return encoding + def infer_delim(file): - with open(file, 'r') as infile: + with open(file, "r") as infile: line = infile.readline() - tabs = line.count('\t') - commas = line.count(',') + tabs = line.count("\t") + commas = line.count(",") if tabs > commas: - return '\t' - return ',' + return "\t" + return "," def infer_quote_mode( @@ -125,9 +130,7 @@ def infer_quote_mode( return "literal" try: - rows_lit = list( - _csv.reader(io.StringIO(raw), delimiter=delimiter, quoting=_csv.QUOTE_NONE) - ) + rows_lit = list(_csv.reader(io.StringIO(raw), delimiter=delimiter, quoting=_csv.QUOTE_NONE)) except _csv.Error: return "csv" @@ -194,7 +197,7 @@ def arrow_drop_duplicates( return table sort_keys = [(name, "ascending") for name in pk_names] - sorted_idx = pc.sort_indices(table, sort_keys=sort_keys) # type: ignore + sorted_idx = pc.sort_indices(table, sort_keys=sort_keys) # type: ignore sorted_table = table.take(sorted_idx) diffs = [] for name in pk_names: @@ -202,40 +205,44 @@ def arrow_drop_duplicates( previous_arr = col[:-1] this_arr = col[1:] diffs.append( - pc.not_equal(previous_arr, this_arr) # type: ignore + pc.not_equal(previous_arr, this_arr) # type: ignore ) keep_tail = diffs[0] for d in diffs[1:]: - keep_tail = pc.or_(keep_tail, d) # type: ignore + keep_tail = pc.or_(keep_tail, d) # type: ignore keep = pc.fill_null(keep_tail, True) if isinstance(keep, pa.ChunkedArray): keep = keep.combine_chunks() - keep = pa.concat_arrays([ - pa.array([True], type=pa.bool_()), - keep, - ]) + keep = pa.concat_arrays( + [ + pa.array([True], type=pa.bool_()), + keep, + ] + ) deduped = sorted_table.filter(keep) - + return deduped -def conservative_load_parquet(path: Path, wanted_cols: list[str], chunksize: int | None = None) -> pa.Table: +def conservative_load_parquet( + path: Path, wanted_cols: list[str], chunksize: int | None = None +) -> pa.Table: delimiter = infer_delim(path) encoding = infer_encoding(path)["encoding"] convert_opts = pv.ConvertOptions( - strings_can_be_null=True, + strings_can_be_null=True, include_columns=wanted_cols, ) def _invalid_row_handler(row): logger.warning("Skipping malformed CSV row: %r", row[:200]) return "skip" - + parse_opts = pv.ParseOptions( delimiter=delimiter, ignore_empty_lines=True, quote_char=False, - invalid_row_handler=_invalid_row_handler + invalid_row_handler=_invalid_row_handler, ) read_opts = pv.ReadOptions( block_size=chunksize or 64_000, @@ -253,6 +260,7 @@ def _invalid_row_handler(row): for batch in reader: yield batch + def check_line_ending(raw_header: str) -> str: if raw_header.endswith("\r\n"): return "\r\n" @@ -261,9 +269,12 @@ def check_line_ending(raw_header: str) -> str: elif raw_header.endswith("\r"): return "\r" else: - logger.warning("Unable to detect line ending from header: %r. Defaulting to '\\n'", raw_header) + logger.warning( + "Unable to detect line ending from header: %r. Defaulting to '\\n'", raw_header + ) return "\n" + def quick_load_pg( *, path: Path, @@ -281,7 +292,7 @@ def quick_load_pg( table_ref = qualified(connection, tablename, physical_schema=schema) - encoding = infer_encoding(path)['encoding'] or 'utf-8' + encoding = infer_encoding(path)["encoding"] or "utf-8" if not _SAFE_ENCODING.match(encoding): raise ValueError(f"Unsafe encoding value from chardet: {encoding!r}") delimiter = infer_delim(path) @@ -305,7 +316,7 @@ def quick_load_pg( """ else: raise ValueError(f"Unknown quote_mode: {quote_mode}") - + # Peek at the CSV header to build an explicit column list for COPY. # Without this, PostgreSQL expects ALL table columns including internal staging # columns like _rownum (GENERATED ALWAYS AS IDENTITY), which the CSV doesn't have. @@ -315,7 +326,10 @@ def quick_load_pg( # _hash is an internal convention for encrypted/hashed columns; strip it so # CSV headers map to the base column names that PostgreSQL COPY expects. # also strip quotes and lowercase - _csv_cols = [c.strip().strip('"').lower().replace('_hash', '') for c in _raw_hdr.rstrip(_nl).split(delimiter)] + _csv_cols = [ + c.strip().strip('"').lower().replace("_hash", "") + for c in _raw_hdr.rstrip(_nl).split(delimiter) + ] _cols_sql = ", ".join(f'"{c}"' for c in _csv_cols) logger.info(f"Bulk loading {table_ref} via COPY (encoding={encoding}, delimiter={delimiter})") @@ -325,19 +339,19 @@ def quick_load_pg( with open(path, "rb") as f: stream = NormalisedCSVStream(f, encoding=encoding, delimiter=delimiter) with cur.copy( - f''' + f""" COPY {table_ref} ({_cols_sql}) FROM STDIN WITH ( {copy_options} ) - ''' + """ ) as copy: while data := stream.read(COPY_BLOCK_SIZE): copy.write(data) session.flush() total = session.execute( - sa.text(f'SELECT COUNT(*) FROM {table_ref}'), **connection_arguments + sa.text(f"SELECT COUNT(*) FROM {table_ref}"), **connection_arguments ).scalar_one() return total except Exception as e: diff --git a/src/orm_loader/mappers/__init__.py b/src/orm_loader/mappers/__init__.py index 1fd7352..a360384 100644 --- a/src/orm_loader/mappers/__init__.py +++ b/src/orm_loader/mappers/__init__.py @@ -8,7 +8,6 @@ # will not affect package-level consumers. A later breaking release can rename # the modules and remove the legacy paths. - from .materialised_view_contracts import ( CreateMaterializedViewIndex, DropMaterializedView, diff --git a/src/orm_loader/mappers/materialised_view_mixin.py b/src/orm_loader/mappers/materialised_view_mixin.py index 39a2447..5f6ea72 100644 --- a/src/orm_loader/mappers/materialised_view_mixin.py +++ b/src/orm_loader/mappers/materialised_view_mixin.py @@ -12,6 +12,7 @@ from .materialised_view_contracts import MaterializedViewIndex + class CreateMaterializedView(DDLElement): """ `CreateMaterializedView` @@ -51,13 +52,11 @@ def __init__( self.with_data = with_data self.if_not_exists = if_not_exists + @compiler.compiles(CreateMaterializedView) def _create_view( - element: CreateMaterializedView, - compiler: sa.sql.compiler.SQLCompiler, - **kwargs: Any + element: CreateMaterializedView, compiler: sa.sql.compiler.SQLCompiler, **kwargs: Any ) -> str: - """ `_create_view` @@ -77,8 +76,8 @@ def _create_view( population = "" if element.with_data else " WITH NO DATA" return f"CREATE MATERIALIZED VIEW {existence}{element.name} as {compiled}{population}" -class MaterializedViewMixin: +class MaterializedViewMixin: """ `MaterializedViewMixin` @@ -162,13 +161,13 @@ class DailyObservationCountsMV(Base, MaterializedViewMixin): observation_date = sa.Column(sa.Date, nullable=False) concept_id = sa.Column(sa.Integer, nullable=False) n_observations = sa.Column(sa.Integer, nullable=False) - - + + ``` Query like a normal mapped class: ```python - + rows = ( session.query(DailyObservationCount) .filter(DailyObservationCount.observation_date >= date(2025, 1, 1)) @@ -184,6 +183,7 @@ class DailyObservationCountsMV(Base, MaterializedViewMixin): * Treat as immutable cache """ + __mv_name__: str __mv_select__: sa.sql.Select[Any] __mv_dependencies__: set[str] = set() @@ -193,7 +193,7 @@ class DailyObservationCountsMV(Base, MaterializedViewMixin): @classmethod def _resolve_schema_tag(cls, schema_tag: str | None) -> str | None: """Effective schema_tag for one lifecycle call. - A declratively mapped class's own mapped table's schema is authoritative, + A declratively mapped class's own mapped table's schema is authoritative, overriding any ``__mv_schema_tag__``. """ if schema_tag is not None: @@ -242,7 +242,7 @@ def create_mv( with engine.begin() as conn: RecentObservationMV.create_mv(conn) - + ``` This emits SQL equivalent to: @@ -359,8 +359,11 @@ def drop_mv( backend = resolve_backend(bind) tag = cls._resolve_schema_tag(schema_tag) backend.drop_materialized_view( - bind, cls.__mv_name__, schema=physical_schema_of(bind, schema_tag=tag), - if_exists=if_exists, cascade=cascade, + bind, + cls.__mv_name__, + schema=physical_schema_of(bind, schema_tag=tag), + if_exists=if_exists, + cascade=cascade, ) @@ -390,9 +393,7 @@ def resolve_mv_refresh_order(mv_classes: list[type[MaterializedViewMixin]]) -> l graph[dep].add(cls.__mv_name__) indegree[cls.__mv_name__] += 1 - queue = deque( - name for name, deg in indegree.items() if deg == 0 - ) + queue = deque(name for name, deg in indegree.items() if deg == 0) ordered: list[str] = [] @@ -406,18 +407,17 @@ def resolve_mv_refresh_order(mv_classes: list[type[MaterializedViewMixin]]) -> l queue.append(downstream) if len(ordered) != len(indegree): - raise RuntimeError( - "Cycle detected in materialized view dependencies" - ) + raise RuntimeError("Cycle detected in materialized view dependencies") return [name_to_mv[name] for name in ordered] -def refresh_all_mvs(bind: "sa.engine.Connection | sa.engine.Engine", mv_classes: list[type[MaterializedViewMixin]]) -> None: - +def refresh_all_mvs( + bind: "sa.engine.Connection | sa.engine.Engine", mv_classes: list[type[MaterializedViewMixin]] +) -> None: """ `refresh_all_mvs` - + Handle refreshing multiple materialized views in dependency order. Examples diff --git a/src/orm_loader/registry/__init__.py b/src/orm_loader/registry/__init__.py index 1cc6701..78c6ca4 100644 --- a/src/orm_loader/registry/__init__.py +++ b/src/orm_loader/registry/__init__.py @@ -64,4 +64,4 @@ def validate(self, *, model, spec=None, fields=None) -> list[ValidationIssue print(report.to_json()) sys.exit(report.exit_code()) -""" \ No newline at end of file +""" diff --git a/src/orm_loader/registry/registry.py b/src/orm_loader/registry/registry.py index b05f430..812c880 100644 --- a/src/orm_loader/registry/registry.py +++ b/src/orm_loader/registry/registry.py @@ -1,4 +1,3 @@ - from dataclasses import dataclass from typing import Optional, Type import sqlalchemy as sa @@ -27,9 +26,9 @@ No domain rules or business logic are enforced here. """ + @dataclass(frozen=True) class TableSpec: - """ Table-level specification descriptor. @@ -49,12 +48,14 @@ class TableSpec: user_guidance Optional additional guidance for implementers. """ + table_name: str schema: str is_required: bool description: str user_guidance: Optional[str] = None + @dataclass(frozen=True) class FieldSpec: """ @@ -82,6 +83,7 @@ class FieldSpec: fk_field Referenced field name, if the field is a foreign key. """ + table_name: str field_name: str data_type: str @@ -91,6 +93,7 @@ class FieldSpec: fk_table: str | None fk_field: str | None + @dataclass(frozen=True) class ModelDescriptor: """ @@ -114,13 +117,13 @@ class ModelDescriptor: foreign_keys Mapping of column name to referenced (table, field). """ + model_class: Type[ORMTableProtocol] table_name: str columns: dict[str, sa.Column] primary_keys: set[str] foreign_keys: dict[str, tuple[str, str]] # col -> (table, field) - @classmethod def from_model(cls, model: Type[ORMTableProtocol]) -> "ModelDescriptor": """ @@ -161,7 +164,7 @@ def from_model(cls, model: Type[ORMTableProtocol]) -> "ModelDescriptor": primary_keys={c.name for c in table.primary_key.columns}, foreign_keys=fks, ) - + @property def cls(self) -> Type[ORMTableProtocol]: """ @@ -204,8 +207,8 @@ def load_table_specs(csv_resource) -> dict[str, TableSpec]: ) return out -def load_field_specs(csv_resource) -> dict[str, dict[str, FieldSpec]]: +def load_field_specs(csv_resource) -> dict[str, dict[str, FieldSpec]]: """ Load field specifications from a CSV resource. @@ -253,10 +256,10 @@ class ModelRegistry: Load table and field specifications from CSV files (currently only OMOP format supported). TODO: support generalised specification formats via LinkML or similar. - Register ORM model classes and compare against specifications to confirm accurate and + Register ORM model classes and compare against specifications to confirm accurate and complete implementation. - Model-specific constraints can be created to extend context-specific validation such + Model-specific constraints can be created to extend context-specific validation such as OMOP domain constraints, value set adherence, etc. Validation logic itself is implemented elsewhere and consumes the @@ -361,10 +364,9 @@ def missing_required_tables(self) -> set[str]: Required table names that are not implemented. """ return { - t for t, spec in self._table_specs.items() - if spec.is_required and t not in self._models + t for t, spec in self._table_specs.items() if spec.is_required and t not in self._models } - + def discover_models(self, package: str) -> None: """ Discover and register ORM models from a Python package. @@ -379,9 +381,7 @@ def discover_models(self, package: str) -> None: """ module = importlib.import_module(package) - for _, modname, _ in pkgutil.walk_packages( - module.__path__, module.__name__ + "." - ): + for _, modname, _ in pkgutil.walk_packages(module.__path__, module.__name__ + "."): mod = importlib.import_module(modname) for obj in vars(mod).values(): diff --git a/src/orm_loader/registry/validation.py b/src/orm_loader/registry/validation.py index ece6633..3e934d2 100644 --- a/src/orm_loader/registry/validation.py +++ b/src/orm_loader/registry/validation.py @@ -3,6 +3,7 @@ from .validation_report import ValidationIssue, SeverityLevel from .registry import ModelDescriptor, TableSpec, FieldSpec + class Validator(Protocol): def validate( self, @@ -10,12 +11,10 @@ def validate( model: ModelDescriptor, spec: TableSpec | None = None, fields: dict[str, FieldSpec] | None = None, - ) -> list[ValidationIssue]: - ... + ) -> list[ValidationIssue]: ... class ColumnNullabilityValidator: - """ Ensures required columns are NOT NULL in the ORM model. """ @@ -55,9 +54,9 @@ def validate( ) return issues - -class ColumnPresenceValidator: + +class ColumnPresenceValidator: """ Ensures all specified columns are present on the ORM model. """ @@ -89,6 +88,7 @@ def validate( return issues + class PrimaryKeyValidator: """ Validates primary key presence and correctness. @@ -142,11 +142,7 @@ def validate( ) if fields: - spec_pks = { - name - for name, fs in fields.items() - if fs.is_primary_key - } + spec_pks = {name for name, fs in fields.items() if fs.is_primary_key} for missing in sorted(spec_pks - pk_cols): issues.append( @@ -183,9 +179,8 @@ def validate( return issues - + class ForeignKeyShapeValidator: - """ Validates structural correctness of foreign key definitions. """ @@ -226,4 +221,4 @@ def validate( ) ) - return issues \ No newline at end of file + return issues diff --git a/src/orm_loader/registry/validation_presets.py b/src/orm_loader/registry/validation_presets.py index 2abec0f..a763287 100644 --- a/src/orm_loader/registry/validation_presets.py +++ b/src/orm_loader/registry/validation_presets.py @@ -1,10 +1,11 @@ from .validation import ( - ColumnNullabilityValidator, - ColumnPresenceValidator, - PrimaryKeyValidator, + ColumnNullabilityValidator, + ColumnPresenceValidator, + PrimaryKeyValidator, ForeignKeyShapeValidator, ) + def always_on_validators(): return [ ColumnPresenceValidator(), diff --git a/src/orm_loader/registry/validation_report.py b/src/orm_loader/registry/validation_report.py index 422fbf6..b0a07b4 100644 --- a/src/orm_loader/registry/validation_report.py +++ b/src/orm_loader/registry/validation_report.py @@ -30,6 +30,7 @@ class SeverityLevel(Enum): WARN = "WARN" INFO = "INFO" + @dataclass class ValidationIssue: """ @@ -55,14 +56,16 @@ class ValidationIssue: hint Optional human-readable guidance. """ + table: str - level: SeverityLevel + level: SeverityLevel message: str field: Optional[str] = None expected: Optional[str] = None actual: Optional[str] = None hint: Optional[str] = None + @dataclass class ValidationReport: """ @@ -75,6 +78,7 @@ class ValidationReport: - JSON serialisation - CI/CD pipeline integration """ + def __init__(self, *, model_version: str, model_name: Optional[str] = None): """ Initialise a new validation report. @@ -89,7 +93,7 @@ def __init__(self, *, model_version: str, model_name: Optional[str] = None): self.model_version = model_version self.model_name = model_name self.issues: list[ValidationIssue] = [] - + def add(self, issue: ValidationIssue) -> None: """ Add a validation issue to the report. @@ -100,7 +104,7 @@ def add(self, issue: ValidationIssue) -> None: Validation issue to record. """ self.issues.append(issue) - + def is_valid(self) -> bool: """ Return whether the report contains no issues. @@ -113,7 +117,6 @@ def is_valid(self) -> bool: return not self.issues def summary(self) -> str: - """ Return a one-line human-readable summary. @@ -127,7 +130,7 @@ def summary(self) -> str: by[i.level] += 1 model = self.model_name.upper() if self.model_name else "MODEL" return f"{model} v{self.model_version}: {by[SeverityLevel.ERROR]} error(s), {by[SeverityLevel.WARN]} warning(s), {by[SeverityLevel.INFO]} info" - + def render_text_report(self) -> str: """ Render a human-readable multi-line text report. @@ -155,7 +158,7 @@ def render_text_report(self) -> str: lines.append(f" {icon} {i.message}{field}{hint}") return "\n".join(lines) - + def to_dict(self) -> dict: """ Return a structured dictionary representation of the report. @@ -187,7 +190,7 @@ def to_dict(self) -> dict: for i in self.issues ], } - + def to_json(self) -> str: """ Serialise the report to a JSON string. @@ -198,7 +201,7 @@ def to_json(self) -> str: JSON-formatted validation report. """ return json.dumps(self.to_dict(), indent=2) - + def exit_code(self) -> int: """ Return a process exit code suitable for CI/CD pipelines. @@ -208,4 +211,4 @@ def exit_code(self) -> int: int 1 if any ERROR-level issues are present, otherwise 0. """ - return 1 if any(i.level == SeverityLevel.ERROR for i in self.issues) else 0 \ No newline at end of file + return 1 if any(i.level == SeverityLevel.ERROR for i in self.issues) else 0 diff --git a/src/orm_loader/registry/validation_runner.py b/src/orm_loader/registry/validation_runner.py index a29486d..d567441 100644 --- a/src/orm_loader/registry/validation_runner.py +++ b/src/orm_loader/registry/validation_runner.py @@ -17,6 +17,7 @@ - supporting optional fail-fast behaviour """ + class ValidationRunner: """ Coordinates execution of validators against a model registry. @@ -24,6 +25,7 @@ class ValidationRunner: The runner itself is deliberately simple and does not embed any validation logic. All rules are provided via Validator instances. """ + def __init__(self, validators: list[Validator], fail_fast: bool = False): """ Initialise a validation runner. @@ -38,7 +40,6 @@ def __init__(self, validators: list[Validator], fail_fast: bool = False): self.validators = validators self.fail_fast = fail_fast - def run(self, registry: ModelRegistry) -> ValidationReport: """ Run validation across all registered models. @@ -53,9 +54,7 @@ def run(self, registry: ModelRegistry) -> ValidationReport: ValidationReport Aggregated validation report. """ - report = ValidationReport( - model_version=registry.model_version - ) + report = ValidationReport(model_version=registry.model_version) for table_name, desc in registry.models().items(): table_spec = registry._table_specs.get(table_name) @@ -71,10 +70,7 @@ def run(self, registry: ModelRegistry) -> ValidationReport: for issue in issues: report.add(issue) - if ( - self.fail_fast - and issue.level == SeverityLevel.ERROR - ): + if self.fail_fast and issue.level == SeverityLevel.ERROR: return report return report diff --git a/src/orm_loader/tables/__init__.py b/src/orm_loader/tables/__init__.py index de740ac..7232aac 100644 --- a/src/orm_loader/tables/__init__.py +++ b/src/orm_loader/tables/__init__.py @@ -11,4 +11,4 @@ "IdAllocator", "ORMTableProtocol", "CSVTableProtocol", -] \ No newline at end of file +] diff --git a/src/orm_loader/tables/loadable_table.py b/src/orm_loader/tables/loadable_table.py index 1f29b6d..117e7b3 100644 --- a/src/orm_loader/tables/loadable_table.py +++ b/src/orm_loader/tables/loadable_table.py @@ -103,7 +103,7 @@ def create_staging_table( @classmethod @contextmanager def manage_indices( - cls: Type['CSVTableProtocol'], + cls: Type["CSVTableProtocol"], session: so.Session, index_strategy: str = "auto", *, @@ -127,9 +127,11 @@ def manage_indices( connection = session.connection(bind_arguments={"mapper": cls}) inspector = sa.inspect(connection) schema = physical_schema_of(connection, schema_tag=cls.__table__.schema) - existing_in_db = {idx['name'] for idx in inspector.get_indexes(cls.__tablename__, schema=schema)} + existing_in_db = { + idx["name"] for idx in inspector.get_indexes(cls.__tablename__, schema=schema) + } to_drop = [i for i in indices if i.name in existing_in_db] - + if to_drop: logger.info(f"Table `{table_name}`: Dropping {len(to_drop)} active indices.") drop_started = perf_counter() @@ -168,7 +170,7 @@ def manage_indices( finally: logger.info(f"Table `{table_name}`: Restoring foreign key checks.") fk_restore_started = perf_counter() - + except Exception as e: session.rollback() logger.error(f"Table `{table_name}`: Merge operation failed - {e}") @@ -185,36 +187,43 @@ def manage_indices( connection = session.connection(bind_arguments={"mapper": cls}) inspector = sa.inspect(connection) schema = physical_schema_of(connection, schema_tag=cls.__table__.schema) - existing_idx_names = {idx['name'] for idx in inspector.get_indexes(table_name, schema=schema)} - + existing_idx_names = { + idx["name"] for idx in inspector.get_indexes(table_name, schema=schema) + } + for idx in indices: if idx.name not in existing_idx_names: try: - logger.info(f"Table `{table_name}`: Restoring missing index: {idx.name}") + logger.info( + f"Table `{table_name}`: Restoring missing index: {idx.name}" + ) create_started = perf_counter() session.execute(sa.schema.CreateIndex(idx)) logger.info( f"Table `{table_name}`: Restored missing index `{idx.name}` in " f"{_format_elapsed(perf_counter() - create_started)}." ) - logger.info(f"Table `{table_name}`: Committing restored index `{idx.name}`.") + logger.info( + f"Table `{table_name}`: Committing restored index `{idx.name}`." + ) commit_started = perf_counter() session.commit() logger.info( f"Table `{table_name}`: Commit after restoring index `{idx.name}` " f"completed in {_format_elapsed(perf_counter() - commit_started)}." - ) + ) except Exception as e: session.rollback() logger.error(f"Table `{table_name}`: Failed to restore {idx.name}: {e}") else: - logger.debug(f"Table `{table_name}`: Index {idx.name} already exists on disk. Skipping.") + logger.debug( + f"Table `{table_name}`: Index {idx.name} already exists on disk. Skipping." + ) logger.info( f"Table `{table_name}`: Index verification/rebuild completed in " f"{_format_elapsed(perf_counter() - rebuild_started)}." ) - @classmethod def get_staging_table( cls: Type[CSVTableProtocol], @@ -260,7 +269,9 @@ def get_staging_table( if staging_schema is None and backend.dialect == Dialect.SQLITE: staging_schema = cls.__table__.schema if not inspector.has_table(staging_name, schema=backend.staging_schema): - logger.debug(f"Staging table {staging_name} does not exist; recreating",) + logger.debug( + f"Staging table {staging_name} does not exist; recreating", + ) cls.create_staging_table(session, staging_schema_tag=staging_schema_tag) connection = session.connection(bind_arguments={"mapper": cls}) @@ -303,7 +314,9 @@ def load_staging( ) total = 0 - cls.create_staging_table(loader_context.session, staging_schema_tag=loader_context.staging_schema_tag) + cls.create_staging_table( + loader_context.session, staging_schema_tag=loader_context.staging_schema_tag + ) try: total = backend.load_staging_fast(loader_context=loader_context) @@ -312,19 +325,14 @@ def load_staging( except Exception as e: loader_context.session.rollback() logger.warning(f"Fast-path load failed for {cls.__tablename__}: {e}") - logger.info('Falling back to ORM-based load functionality') + logger.info("Falling back to ORM-based load functionality") - total = cls.orm_staging_load( - loader=loader, - loader_context=loader_context - ) + total = cls.orm_staging_load(loader=loader, loader_context=loader_context) return total @classmethod def orm_staging_load( - cls: Type[CSVTableProtocol], - loader: LoaderInterface, - loader_context: LoaderContext + cls: Type[CSVTableProtocol], loader: LoaderInterface, loader_context: LoaderContext ) -> int: """ Load data into the staging table using an ORM-based loader. @@ -357,7 +365,6 @@ def _select_loader(cls: Type[CSVTableProtocol], path: Path) -> LoaderInterface: else: return PandasLoader() - @classmethod def load_csv( cls: Type[CSVTableProtocol], @@ -374,7 +381,6 @@ def load_csv( merge_batch_size: int | None = None, staging_schema_tag: str | None = None, ) -> int: - """ Load a CSV (or CSV-like) file into the target table. @@ -465,7 +471,9 @@ def load_csv( # Merge staging to target (Wrapped in our index dropper!) logger.info(f"Table `{cls.__tablename__}`: Merging staging data into target table") - with cls.manage_indices(session, index_strategy=index_strategy, staging_schema_tag=staging_schema_tag): + with cls.manage_indices( + session, index_strategy=index_strategy, staging_schema_tag=staging_schema_tag + ): cls.merge_from_staging( session, merge_strategy=merge_strategy, @@ -475,9 +483,10 @@ def load_csv( cls.drop_staging_table(session, staging_schema_tag=staging_schema_tag) - logger.info(f"Table `{cls.__tablename__}`: Successfully finished ingestion. Total rows: {total}") + logger.info( + f"Table `{cls.__tablename__}`: Successfully finished ingestion. Total rows: {total}" + ) return total - @classmethod def _target_has_rows( @@ -493,7 +502,6 @@ def _target_has_rows( ).first() return row is not None - @classmethod def merge_from_staging( cls: Type[CSVTableProtocol], @@ -589,7 +597,7 @@ def merge_from_staging( ) else: raise ValueError(f"Unknown merge strategy '{merge_strategy}'") - + @classmethod def drop_staging_table( cls: Type[CSVTableProtocol], diff --git a/src/orm_loader/tables/orm_table.py b/src/orm_loader/tables/orm_table.py index 29fe02d..49512f6 100644 --- a/src/orm_loader/tables/orm_table.py +++ b/src/orm_loader/tables/orm_table.py @@ -14,9 +14,9 @@ def _resolves_to_autoincrement(c: sa.Column) -> bool: own default ``autoincrement="auto"`` sentinel the same way SQLAlchemy itself does at DDL-emission time. - SQLAlchemy restricts real autoincrement behaviour to a single-column - integer primary key (or an explicit ``autoincrement=True``). - Treating every `"auto"` column as autoincrementing would wrongly + SQLAlchemy restricts real autoincrement behaviour to a single-column + integer primary key (or an explicit ``autoincrement=True``). + Treating every `"auto"` column as autoincrementing would wrongly exclude genuinely-required string PKs from `required_columns()`. """ if c.autoincrement is False: @@ -28,6 +28,7 @@ def _resolves_to_autoincrement(c: sa.Column) -> bool: pk_columns = list(c.table.primary_key.columns) return len(pk_columns) == 1 and pk_columns[0] is c and isinstance(c.type, sa.Integer) + """ ORMTableBase ============ @@ -138,7 +139,7 @@ def pk_values(cls, obj: Any) -> dict[str, Any]: A dictionary mapping primary key column names to values. """ return {c.key: getattr(obj, c.key) for c in cls.pk_columns() if c.key is not None} - + @classmethod def pk_tuple(cls, obj: Any) -> tuple[Any, ...]: """ @@ -156,11 +157,7 @@ def pk_tuple(cls, obj: Any) -> tuple[Any, ...]: tuple A tuple of primary key values. """ - return tuple( - getattr(obj, c.key) - for c in cls.pk_columns() - if c.key is not None - ) + return tuple(getattr(obj, c.key) for c in cls.pk_columns() if c.key is not None) @classmethod def model_columns(cls) -> dict[str, sa.ColumnElement[Any]]: @@ -174,7 +171,7 @@ def model_columns(cls) -> dict[str, sa.ColumnElement[Any]]: """ mapper = cls.mapper_for() return {c.key: c for c in mapper.columns} - + @classmethod def required_columns(cls) -> set[str]: """ @@ -229,9 +226,7 @@ def max_id(cls, session: so.Session) -> int: """ pks = cls.pk_columns() if len(pks) != 1: - raise ValueError( - f"{cls.__name__} has composite PK; max_id() not supported" - ) + raise ValueError(f"{cls.__name__} has composite PK; max_id() not supported") pk = pks[0] return session.query(sa.func.max(pk)).scalar() or 0 @@ -259,7 +254,6 @@ def allocator(cls, session: so.Session) -> IdAllocator: """ return IdAllocator(cls.max_id(session)) - @classmethod def clean_kwargs( cls, @@ -291,13 +285,10 @@ def clean_kwargs( if strict: missing = cls.required_columns() - cleaned.keys() if missing: - raise ValueError( - f"Missing required fields for {cls.__name__}: {sorted(missing)}" - ) + raise ValueError(f"Missing required fields for {cls.__name__}: {sorted(missing)}") return cleaned - @classmethod def from_dict( cls, @@ -331,14 +322,10 @@ def from_dict( except StatementError as e: # SQLAlchemy type coercion / bind errors msg = ( - f"SQLAlchemy failed to bind values for {cls.__name__}.\n" - f"Cleaned kwargs: {cleaned}\n" + f"SQLAlchemy failed to bind values for {cls.__name__}.\nCleaned kwargs: {cleaned}\n" ) raise ValueError(msg) from e except Exception as e: - msg = ( - f"Unexpected error constructing {cls.__name__}.\n" - f"Cleaned kwargs: {cleaned}\n" - ) - raise RuntimeError(msg) from e \ No newline at end of file + msg = f"Unexpected error constructing {cls.__name__}.\nCleaned kwargs: {cleaned}\n" + raise RuntimeError(msg) from e diff --git a/src/orm_loader/tables/serialisable_table.py b/src/orm_loader/tables/serialisable_table.py index 2310855..e4988eb 100644 --- a/src/orm_loader/tables/serialisable_table.py +++ b/src/orm_loader/tables/serialisable_table.py @@ -34,7 +34,7 @@ def json_default(obj: Any) -> str: return obj.isoformat() raise TypeError(f"Object of type {type(obj)} is not JSON serialisable") - + class SerialisableTableInterface(ORMTableBase): """ Mixin for SQLAlchemy ORM tables providing explicit serialisation helpers. @@ -60,7 +60,6 @@ def to_dict( only: set[str] | None = None, exclude: set[str] | None = None, ) -> dict[str, Any]: - """ Convert the ORM instance into a dictionary. @@ -117,7 +116,7 @@ def to_json(self, **kwargs: Unpack[ToDictKwargs]) -> str: default=json_default, sort_keys=True, ) - + def fingerprint(self) -> str: """ Compute a stable fingerprint for the ORM instance. @@ -136,7 +135,7 @@ def fingerprint(self) -> str: """ payload = self.to_json(include_nulls=True) return hashlib.sha256(payload.encode("utf-8")).hexdigest() - + def __iter__(self) -> Iterator[tuple[str, Any]]: """ Iterate over the ORM instance as ``(key, value)`` pairs. @@ -163,4 +162,4 @@ def __json__(self) -> dict[str, Any]: dict[str, Any] A dictionary representation of the ORM row. """ - return self.to_dict() \ No newline at end of file + return self.to_dict() diff --git a/src/orm_loader/tables/typing.py b/src/orm_loader/tables/typing.py index b1a3b14..9c118b8 100644 --- a/src/orm_loader/tables/typing.py +++ b/src/orm_loader/tables/typing.py @@ -1,16 +1,30 @@ -from typing import Protocol, ClassVar, runtime_checkable, TYPE_CHECKING, Optional, Type, Dict, Any, Unpack, TypedDict +from typing import ( + Protocol, + ClassVar, + runtime_checkable, + TYPE_CHECKING, + Optional, + Type, + Dict, + Any, + Unpack, + TypedDict, +) import sqlalchemy.orm as so import sqlalchemy as sa from pathlib import Path from contextlib import AbstractContextManager + if TYPE_CHECKING: from ..loaders import LoaderContext, LoaderInterface + class ToDictKwargs(TypedDict, total=False): include_nulls: bool only: set[str] | None exclude: set[str] | None + @runtime_checkable class ORMTableProtocol(Protocol): """ @@ -47,6 +61,7 @@ def model_columns(cls) -> dict[str, sa.ColumnElement[Any]]: ... @classmethod def required_columns(cls) -> set[str]: ... + @runtime_checkable class CSVTableProtocol(ORMTableProtocol, Protocol): """ @@ -67,10 +82,14 @@ class CSVTableProtocol(ORMTableProtocol, Protocol): def _select_loader(cls, path: Path) -> "LoaderInterface": ... @classmethod - def create_staging_table(cls, session: so.Session, *, staging_schema_tag: str | None = None) -> None: ... + def create_staging_table( + cls, session: so.Session, *, staging_schema_tag: str | None = None + ) -> None: ... @classmethod - def load_staging(cls: Type["CSVTableProtocol"], loader: "LoaderInterface", loader_context: "LoaderContext") -> int: ... + def load_staging( + cls: Type["CSVTableProtocol"], loader: "LoaderInterface", loader_context: "LoaderContext" + ) -> int: ... @classmethod def load_csv( @@ -90,10 +109,14 @@ def load_csv( ) -> int: ... @classmethod - def orm_staging_load(cls, loader: "LoaderInterface", loader_context: "LoaderContext") -> int: ... + def orm_staging_load( + cls, loader: "LoaderInterface", loader_context: "LoaderContext" + ) -> int: ... @classmethod - def get_staging_table(cls, session: so.Session, *, staging_schema_tag: str | None = None) -> sa.Table: ... + def get_staging_table( + cls, session: so.Session, *, staging_schema_tag: str | None = None + ) -> sa.Table: ... @classmethod def merge_from_staging( @@ -106,17 +129,22 @@ def merge_from_staging( ) -> None: ... @classmethod - def drop_staging_table(cls, session: so.Session, *, staging_schema_tag: str | None = None) -> None: ... + def drop_staging_table( + cls, session: so.Session, *, staging_schema_tag: str | None = None + ) -> None: ... @classmethod def _target_has_rows(cls, session: so.Session) -> bool: ... @classmethod def manage_indices( - cls, session: so.Session, index_strategy: str = "auto", *, staging_schema_tag: str | None = None - ) -> AbstractContextManager[None]: - ... - + cls, + session: so.Session, + index_strategy: str = "auto", + *, + staging_schema_tag: str | None = None, + ) -> AbstractContextManager[None]: ... + @runtime_checkable class SerializedTableProtocol(Protocol): diff --git a/tests/backends/test_base_backend.py b/tests/backends/test_base_backend.py index 34154a2..0172aff 100644 --- a/tests/backends/test_base_backend.py +++ b/tests/backends/test_base_backend.py @@ -70,9 +70,7 @@ def create_staging_table( ) -> None: return None - def drop_staging_table( - self, _table_cls: Type[CSVTableProtocol], _session: so.Session - ) -> None: + def drop_staging_table(self, _table_cls: Type[CSVTableProtocol], _session: so.Session) -> None: return None def merge_replace( @@ -170,7 +168,7 @@ def test_backend_capabilities_defaults(): def test_database_backend_is_abstract(): with pytest.raises(TypeError): - DatabaseBackend() # type: ignore + DatabaseBackend() # type: ignore def test_fake_backend_can_implement_contract(): @@ -334,9 +332,7 @@ def capabilities(self) -> BackendCapabilities: NotImplementedError, match="has not implemented create_materialized_view", ): - backend.create_materialized_view( - cast(Engine, None), "mv_test", cast(Any, None) - ) + backend.create_materialized_view(cast(Engine, None), "mv_test", cast(Any, None)) with pytest.raises( NotImplementedError, diff --git a/tests/backends/test_postgres_backend.py b/tests/backends/test_postgres_backend.py index 774cc25..457c93b 100644 --- a/tests/backends/test_postgres_backend.py +++ b/tests/backends/test_postgres_backend.py @@ -18,7 +18,9 @@ _TARGET_TABLE = ComputedColumnTable.__tablename__ _STAGING_TABLE = f"_staging_{_TARGET_TABLE}" _PREPARER = postgresql.dialect().identifier_preparer -_STAGING_TABLE_WITH_SCHEMA: str = qualified(_PREPARER, _STAGING_TABLE, physical_schema=STAGING_SCHEMA) +_STAGING_TABLE_WITH_SCHEMA: str = qualified( + _PREPARER, _STAGING_TABLE, physical_schema=STAGING_SCHEMA +) _ComputedTableCls = cast("Type[CSVTableProtocol]", ComputedColumnTable) @@ -106,13 +108,13 @@ def test_postgres_backend_rejects_non_positive_merge_batch_size(merge_batch_size merge(*args, merge_batch_size=merge_batch_size) - - def test_postgres_backend_default_staging_schema_is_none(): backend = PostgresBackend() assert backend.staging_schema is None - assert backend.qualified_staging_name(_TARGET_TABLE) == qualified(_PREPARER, _STAGING_TABLE, physical_schema=None) + assert backend.qualified_staging_name(_TARGET_TABLE) == qualified( + _PREPARER, _STAGING_TABLE, physical_schema=None + ) def test_postgres_backend_create_staging_table_drops_computed_columns(pg_session): @@ -131,7 +133,7 @@ def test_postgres_backend_drop_staging_table(): backend.drop_staging_table(_ComputedTableCls, _sess(session)) - assert session.statements == [f'DROP TABLE IF EXISTS {_STAGING_TABLE_WITH_SCHEMA}'] + assert session.statements == [f"DROP TABLE IF EXISTS {_STAGING_TABLE_WITH_SCHEMA}"] def test_postgres_backend_fk_methods_emit_expected_sql(): @@ -176,18 +178,23 @@ def test_postgres_backend_materialized_view_respects_schema(pg_db) -> None: selectable = sa.select(sa.literal(1).label("n")) engine = pg_db.connection.engine - with isolated_test_schema(engine, prefix="mv_primary") as primary_schema, \ - isolated_test_schema(engine, prefix="mv_vocab") as vocab_schema: + with ( + isolated_test_schema(engine, prefix="mv_primary") as primary_schema, + isolated_test_schema(engine, prefix="mv_vocab") as vocab_schema, + ): with engine.begin() as conn: - backend.create_materialized_view(conn, "mv_schema_test", selectable, schema=vocab_schema) + backend.create_materialized_view( + conn, "mv_schema_test", selectable, schema=vocab_schema + ) backend.refresh_materialized_view(conn, "mv_schema_test", schema=vocab_schema) with engine.connect() as conn: assert sa.inspect(conn).has_table("mv_schema_test", schema=vocab_schema) assert not sa.inspect(conn).has_table("mv_schema_test", schema=primary_schema) - assert conn.execute( - sa.text(f'SELECT n FROM "{vocab_schema}".mv_schema_test') - ).scalar() == 1 + assert ( + conn.execute(sa.text(f'SELECT n FROM "{vocab_schema}".mv_schema_test')).scalar() + == 1 + ) def test_postgres_backend_materialized_view_methods_emit_expected_sql(): @@ -198,8 +205,11 @@ def test_postgres_backend_materialized_view_methods_emit_expected_sql(): backend.create_materialized_view(_sess(session), "mv_test", selectable) backend.refresh_materialized_view(_sess(session), "mv_test") - assert any('CREATE MATERIALIZED VIEW IF NOT EXISTS mv_test as SELECT' in sql for sql in session.statements) - assert any('REFRESH MATERIALIZED VIEW mv_test;' == sql for sql in session.statements) + assert any( + "CREATE MATERIALIZED VIEW IF NOT EXISTS mv_test as SELECT" in sql + for sql in session.statements + ) + assert any("REFRESH MATERIALIZED VIEW mv_test;" == sql for sql in session.statements) def test_postgres_backend_rejects_mismatched_dialect_bind(): @@ -241,12 +251,15 @@ def test_postgres_backend_quotes_unqualified_materialized_view_name(): session = _FakeSession() selectable = sa.select(sa.literal(1).label("n")) - backend.create_materialized_view(_sess(session), 'mv name', selectable) + backend.create_materialized_view(_sess(session), "mv name", selectable) backend.create_materialized_view_index( - _sess(session), 'mv name', MaterializedViewIndex(name="mv_name_idx", columns=("n",)) + _sess(session), "mv name", MaterializedViewIndex(name="mv_name_idx", columns=("n",)) ) - assert any('CREATE MATERIALIZED VIEW IF NOT EXISTS "mv name" as SELECT' in sql for sql in session.statements) + assert any( + 'CREATE MATERIALIZED VIEW IF NOT EXISTS "mv name" as SELECT' in sql + for sql in session.statements + ) assert any('ON "mv name" ("n")' in sql for sql in session.statements) @@ -296,7 +309,7 @@ def test_postgres_backend_drop_materialized_view_default_args(): backend.drop_materialized_view(_sess(session), "mv_test", schema="reporting") - assert session.statements == ['DROP MATERIALIZED VIEW IF EXISTS reporting.mv_test'] + assert session.statements == ["DROP MATERIALIZED VIEW IF EXISTS reporting.mv_test"] def test_postgres_backend_drop_materialized_view_cascade_and_if_exists_false(): @@ -307,11 +320,14 @@ def test_postgres_backend_drop_materialized_view_cascade_and_if_exists_false(): _sess(session), "mv_test", schema="reporting", if_exists=False, cascade=True ) - assert session.statements == ['DROP MATERIALIZED VIEW reporting.mv_test CASCADE'] + assert session.statements == ["DROP MATERIALIZED VIEW reporting.mv_test CASCADE"] def test_postgres_backend_drop_materialized_view_failure_preserves_cause(): - from orm_loader.mappers.materialised_view_errors import MaterializationError, MaterializationOperation + from orm_loader.mappers.materialised_view_errors import ( + MaterializationError, + MaterializationOperation, + ) original = RuntimeError("boom") backend = PostgresBackend() @@ -328,7 +344,10 @@ def test_postgres_backend_drop_materialized_view_failure_preserves_cause(): def test_postgres_backend_create_materialized_view_failure_preserves_cause(): - from orm_loader.mappers.materialised_view_errors import MaterializationError, MaterializationOperation + from orm_loader.mappers.materialised_view_errors import ( + MaterializationError, + MaterializationOperation, + ) original = RuntimeError("boom") backend = PostgresBackend() @@ -384,9 +403,7 @@ def test_postgres_backend_refresh_concurrently_declared_but_database_rejects_it_ ) assert exc_info.value.failure.cause.orig is orig - assert session.statements == [ - 'REFRESH MATERIALIZED VIEW CONCURRENTLY reporting.mv_test;' - ] + assert session.statements == ["REFRESH MATERIALIZED VIEW CONCURRENTLY reporting.mv_test;"] def test_postgres_backend_refresh_concurrently_unrelated_operational_error_propagates_unchanged(): @@ -428,9 +445,7 @@ def test_postgres_backend_refresh_concurrently_with_declared_index_emits_concurr declared_indexes=(index,), ) - assert session.statements[-1] == ( - 'REFRESH MATERIALIZED VIEW CONCURRENTLY reporting.mv_test;' - ) + assert session.statements[-1] == ("REFRESH MATERIALIZED VIEW CONCURRENTLY reporting.mv_test;") def test_postgres_backend_materialized_view_lifecycle_is_schema_isolated_with_adversarial_identifiers( @@ -466,20 +481,26 @@ def test_postgres_backend_materialized_view_lifecycle_is_schema_isolated_with_ad backend.drop_materialized_view(conn, name, schema=left_schema) with engine.connect() as conn: - assert conn.execute( - sa.text( - "SELECT EXISTS (SELECT 1 FROM pg_matviews " - "WHERE schemaname = :schema AND matviewname = :name)" - ), - {"schema": left_schema, "name": name}, - ).scalar() is False - assert conn.execute( - sa.text( - "SELECT EXISTS (SELECT 1 FROM pg_matviews " - "WHERE schemaname = :schema AND matviewname = :name)" - ), - {"schema": right_schema, "name": name}, - ).scalar() is True + assert ( + conn.execute( + sa.text( + "SELECT EXISTS (SELECT 1 FROM pg_matviews " + "WHERE schemaname = :schema AND matviewname = :name)" + ), + {"schema": left_schema, "name": name}, + ).scalar() + is False + ) + assert ( + conn.execute( + sa.text( + "SELECT EXISTS (SELECT 1 FROM pg_matviews " + "WHERE schemaname = :schema AND matviewname = :name)" + ), + {"schema": right_schema, "name": name}, + ).scalar() + is True + ) finally: with engine.begin() as cleanup_conn: for schema in (left_schema, right_schema): @@ -516,9 +537,7 @@ def test_postgres_backend_materialized_view_legacy_unqualified_path_still_round_ backend = PostgresBackend() conn = pg_db.connection - backend.create_materialized_view( - conn, "mv_legacy_test", sa.select(sa.literal(1).label("n")) - ) + backend.create_materialized_view(conn, "mv_legacy_test", sa.select(sa.literal(1).label("n"))) backend.refresh_materialized_view(conn, "mv_legacy_test") assert conn.execute(sa.text("SELECT n FROM mv_legacy_test")).scalar() == 1 @@ -574,7 +593,6 @@ def test_postgres_backend_enable_fk_raises_when_show_returns_non_string(): raise AssertionError("Expected RuntimeError when SHOW returns a non-string") - def test_postgres_backend_engine_with_replica_role_unregisters_listener(monkeypatch): backend = PostgresBackend() events: list[tuple[str, object, str]] = [] diff --git a/tests/backends/test_shared_backend.py b/tests/backends/test_shared_backend.py index 8c85250..1c3e37d 100644 --- a/tests/backends/test_shared_backend.py +++ b/tests/backends/test_shared_backend.py @@ -41,7 +41,9 @@ def merge_backend(request: pytest.FixtureRequest) -> tuple[DatabaseBackend, "so. def test_merge_replace_single_pk(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: backend, session = merge_backend backend.create_staging_table(_ComputedTableCls, session) - staging = _ComputedTableCls.get_staging_table(session, staging_schema_tag=backend.staging_schema_tag) + staging = _ComputedTableCls.get_staging_table( + session, staging_schema_tag=backend.staging_schema_tag + ) session.execute( sa.insert(ComputedColumnTable), @@ -58,7 +60,9 @@ def test_merge_replace_single_pk(merge_backend: tuple[DatabaseBackend, "so.Sessi def test_merge_replace_composite_pk(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: backend, session = merge_backend backend.create_staging_table(_CompositeTableCls, session) - staging = _CompositeTableCls.get_staging_table(session, staging_schema_tag=backend.staging_schema_tag) + staging = _CompositeTableCls.get_staging_table( + session, staging_schema_tag=backend.staging_schema_tag + ) session.execute( sa.insert(CompositeTable), @@ -72,10 +76,14 @@ def test_merge_replace_composite_pk(merge_backend: tuple[DatabaseBackend, "so.Se assert remaining == [(2, 2)] -def test_merge_insert_excludes_computed_columns(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_insert_excludes_computed_columns( + merge_backend: tuple[DatabaseBackend, "so.Session"], +) -> None: backend, session = merge_backend backend.create_staging_table(_ComputedTableCls, session) - staging = _ComputedTableCls.get_staging_table(session, staging_schema_tag=backend.staging_schema_tag) + staging = _ComputedTableCls.get_staging_table( + session, staging_schema_tag=backend.staging_schema_tag + ) session.execute(sa.insert(staging), [{"id": 1, "name": "alpha"}]) backend.merge_insert(_ComputedTableCls, session) @@ -84,15 +92,17 @@ def test_merge_insert_excludes_computed_columns(merge_backend: tuple[DatabaseBac assert (row.id, row.name, row.slug) == (1, "alpha", "alpha") -def test_merge_upsert_excludes_computed_columns(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_upsert_excludes_computed_columns( + merge_backend: tuple[DatabaseBackend, "so.Session"], +) -> None: backend, session = merge_backend backend.create_staging_table(_ComputedTableCls, session) - staging = _ComputedTableCls.get_staging_table(session, staging_schema_tag=backend.staging_schema_tag) + staging = _ComputedTableCls.get_staging_table( + session, staging_schema_tag=backend.staging_schema_tag + ) session.execute(sa.insert(ComputedColumnTable), [{"id": 1, "name": "existing"}]) - session.execute( - sa.insert(staging), [{"id": 1, "name": "ignored"}, {"id": 2, "name": "new"}] - ) + session.execute(sa.insert(staging), [{"id": 1, "name": "ignored"}, {"id": 2, "name": "new"}]) backend.merge_upsert(_ComputedTableCls, session, ["id"]) @@ -108,9 +118,7 @@ def test_postgres_merge_insert_paginates_rownums_with_initial_gap(pg_session) -> staging = _ComputedTableCls.get_staging_table( pg_session, staging_schema_tag=backend.staging_schema_tag ) - pg_session.execute( - sa.insert(staging), [{"id": i, "name": f"row{i}"} for i in range(10)] - ) + pg_session.execute(sa.insert(staging), [{"id": i, "name": f"row{i}"} for i in range(10)]) pg_session.execute( sa.text( 'ALTER TABLE "staging"."_staging_computed_column_table" ' @@ -127,16 +135,20 @@ def test_postgres_merge_insert_paginates_rownums_with_initial_gap(pg_session) -> backend.merge_insert(_ComputedTableCls, pg_session, merge_batch_size=2) - ids = pg_session.execute( - sa.select(ComputedColumnTable.id).order_by(ComputedColumnTable.id) - ).scalars().all() + ids = ( + pg_session.execute(sa.select(ComputedColumnTable.id).order_by(ComputedColumnTable.id)) + .scalars() + .all() + ) assert ids == list(range(10)) def test_merge_replace_paginated_path(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: backend, session = merge_backend backend.create_staging_table(_ComputedTableCls, session) - staging = _ComputedTableCls.get_staging_table(session, staging_schema_tag=backend.staging_schema_tag) + staging = _ComputedTableCls.get_staging_table( + session, staging_schema_tag=backend.staging_schema_tag + ) session.execute( sa.insert(ComputedColumnTable), [{"id": i, "name": f"orig{i}"} for i in range(10)] @@ -154,7 +166,9 @@ def test_merge_replace_paginated_path(merge_backend: tuple[DatabaseBackend, "so. def test_merge_insert_paginated_path(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: backend, session = merge_backend backend.create_staging_table(_ComputedTableCls, session) - staging = _ComputedTableCls.get_staging_table(session, staging_schema_tag=backend.staging_schema_tag) + staging = _ComputedTableCls.get_staging_table( + session, staging_schema_tag=backend.staging_schema_tag + ) session.execute(sa.insert(staging), [{"id": i, "name": f"row{i}"} for i in range(10)]) backend.merge_insert(_ComputedTableCls, session, merge_batch_size=3) @@ -166,11 +180,11 @@ def test_merge_insert_paginated_path(merge_backend: tuple[DatabaseBackend, "so.S def test_merge_upsert_paginated_path(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: backend, session = merge_backend backend.create_staging_table(_ComputedTableCls, session) - staging = _ComputedTableCls.get_staging_table(session, staging_schema_tag=backend.staging_schema_tag) - - session.execute( - sa.insert(ComputedColumnTable), [{"id": i, "name": "kept"} for i in range(5)] + staging = _ComputedTableCls.get_staging_table( + session, staging_schema_tag=backend.staging_schema_tag ) + + session.execute(sa.insert(ComputedColumnTable), [{"id": i, "name": "kept"} for i in range(5)]) session.execute( sa.insert(staging), [{"id": i, "name": "should-not-overwrite"} for i in range(10)] ) @@ -178,4 +192,7 @@ def test_merge_upsert_paginated_path(merge_backend: tuple[DatabaseBackend, "so.S backend.merge_upsert(_ComputedTableCls, session, ["id"], merge_batch_size=3) rows = {r.id: r.name for r in session.execute(sa.select(ComputedColumnTable)).scalars().all()} - assert rows == {**{i: "kept" for i in range(5)}, **{i: "should-not-overwrite" for i in range(5, 10)}} + assert rows == { + **{i: "kept" for i in range(5)}, + **{i: "should-not-overwrite" for i in range(5, 10)}, + } diff --git a/tests/backends/test_sqlite_backend.py b/tests/backends/test_sqlite_backend.py index 94f5f7b..8187f1d 100644 --- a/tests/backends/test_sqlite_backend.py +++ b/tests/backends/test_sqlite_backend.py @@ -92,7 +92,6 @@ def test_sqlite_backend_drop_staging_table(): assert session.statements == [f'DROP TABLE IF EXISTS "{_STAGING_TABLE}"'] - def test_sqlite_backend_disable_fk_reads_then_sets(): backend = SQLiteBackend() session = _FakeSession(scalar_result=1) @@ -101,8 +100,8 @@ def test_sqlite_backend_disable_fk_reads_then_sets(): assert previous == 1 assert session.statements == [ - "PRAGMA foreign_keys", # read current state - "PRAGMA foreign_keys = OFF", # set to OFF + "PRAGMA foreign_keys", # read current state + "PRAGMA foreign_keys = OFF", # set to OFF ] @@ -114,8 +113,8 @@ def test_sqlite_backend_enable_fk_reads_then_sets(): assert previous == 0 assert session.statements == [ - "PRAGMA foreign_keys", # read current state - "PRAGMA foreign_keys = ON", # set to ON + "PRAGMA foreign_keys", # read current state + "PRAGMA foreign_keys = ON", # set to ON ] diff --git a/tests/conftest.py b/tests/conftest.py index af3c066..d44fa75 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,7 +19,9 @@ @pytest.fixture def engine(): with isolated_test_database( - OrmLoaderConfig, "test_orm_db_sqlite", dialect="sqlite", + OrmLoaderConfig, + "test_orm_db_sqlite", + dialect="sqlite", ) as db: engine = db.connection.engine Base.metadata.create_all(engine) @@ -36,6 +38,7 @@ def session(engine): # Postgres fixtures # --------------------------------------------------------------------------- + @pytest.fixture def pg_db(request): """Isolated PostgreSQL test database. Everything done through @@ -43,7 +46,10 @@ def pg_db(request): that's rolled back on exit, so concurrent test runs can't collide and nothing needs manual cleanup.""" with isolated_test_database( - OrmLoaderConfig, "test_orm_db_pg", request=request, schema_claims=[staging_schema_claim()], + OrmLoaderConfig, + "test_orm_db_pg", + request=request, + schema_claims=[staging_schema_claim()], ) as db: yield db @@ -65,7 +71,10 @@ def schema_scoped_session( """A Session on committed scoped test schemas, with the staging schema claimed and *table* already created.""" with scoped_test_schema( - pg_db.resolved, prefix=prefix, split_roles=split_roles, schema_claims=[staging_schema_claim()] + pg_db.resolved, + prefix=prefix, + split_roles=split_roles, + schema_claims=[staging_schema_claim()], ) as scoped: with scoped.engine.begin() as conn: table.create(conn, checkfirst=True) diff --git a/tests/helpers/test_bootstrap.py b/tests/helpers/test_bootstrap.py index 41224bb..6f46834 100644 --- a/tests/helpers/test_bootstrap.py +++ b/tests/helpers/test_bootstrap.py @@ -86,7 +86,9 @@ def test_create_tables_crossing_subset_resolves_local_fk_to_existing_table(tmp_p parent = sa.Table( "local_parent", metadata, sa.Column("id", sa.Integer, primary_key=True), schema="primary" ) - sa.Table("remote_parent", metadata, sa.Column("id", sa.Integer, primary_key=True), schema="vocab") + sa.Table( + "remote_parent", metadata, sa.Column("id", sa.Integer, primary_key=True), schema="vocab" + ) child = sa.Table( "crossing_child", metadata, @@ -102,9 +104,9 @@ def test_create_tables_crossing_subset_resolves_local_fk_to_existing_table(tmp_p create_tables(connection, [child], resolved=resolved) assert sa.inspect(engine).has_table("crossing_child") - assert [fk["referred_table"] for fk in sa.inspect(engine).get_foreign_keys("crossing_child")] == [ - "local_parent" - ] + assert [ + fk["referred_table"] for fk in sa.inspect(engine).get_foreign_keys("crossing_child") + ] == ["local_parent"] engine.dispose() diff --git a/tests/loaders/test_casting.py b/tests/loaders/test_casting.py index 4a0ac70..67f4ccb 100644 --- a/tests/loaders/test_casting.py +++ b/tests/loaders/test_casting.py @@ -10,82 +10,105 @@ import pytest from datetime import date, datetime + def test_perform_cast_integer(): assert perform_cast("123", sa.Integer(), on_error=None) == 123 + def test_perform_cast_invalid_integer_returns_none(): errors = [] - def on_error(v): errors.append(v) + + def on_error(v): + errors.append(v) result = perform_cast("abc", sa.Integer(), on_error=on_error) assert result is None assert errors == ["abc"] + def test_perform_cast_boolean(): assert perform_cast("yes", sa.Boolean(), on_error=None) is True assert perform_cast("no", sa.Boolean(), on_error=None) is False + def test_integer_from_numeric_string(): assert perform_cast("42", sa.Integer(), on_error=None) == 42 + def test_integer_from_float_string(): assert perform_cast("42.0", sa.Integer(), on_error=None) == 42 + def test_integer_from_actual_float(): assert perform_cast(42.0, sa.Integer(), on_error=None) == 42 + def test_integer_whitespace(): assert perform_cast(" 7 ", sa.Integer(), on_error=None) == 7 + def test_integer_invalid_string_returns_none_and_records_error(): errors = [] result = perform_cast("Episode", sa.Integer(), on_error=errors.append) assert result is None assert errors == ["Episode"] + def test_integer_empty_string_is_null_not_error(): errors = [] assert perform_cast("", sa.Integer(), on_error=errors.append) is None assert errors == [] + def test_date_from_date(): d = date(2020, 5, 17) assert perform_cast(d, sa.Date(), on_error=None) == d + def test_date_from_datetime(): dt = datetime(2020, 5, 17, 14, 30) assert perform_cast(dt, sa.Date(), on_error=None) == date(2020, 5, 17) + def test_date_yyyymmdd(): assert perform_cast("20170824", sa.Date(), on_error=None) == date(2017, 8, 24) + def test_date_dd_mmm_yyyy(): assert perform_cast("24-AUG-2017", sa.Date(), on_error=None) == date(2017, 8, 24) + def test_date_iso(): assert perform_cast("2017-08-24", sa.Date(), on_error=None) == date(2017, 8, 24) + def test_date_dd_mm_yyyy(): assert perform_cast("24/08/2017", sa.Date(), on_error=None) == date(2017, 8, 24) + def test_date_rejects_iso_datetime(): assert perform_cast("2017-08-24T12:00:00", sa.Date(), on_error=None) is None + def test_date_rejects_invalid_date(): assert perform_cast("2017-99-99", sa.Date(), on_error=None) is None + def test_date_rejects_fuzzy(): assert perform_cast("Aug 24 2017", sa.Date(), on_error=None) is None + def test_datetime_from_datetime(): dt = datetime(2020, 5, 17, 14, 30, 5) assert perform_cast(dt, sa.DateTime(), on_error=None) == dt + def test_datetime_from_date(): d = date(2020, 5, 17) assert perform_cast(d, sa.DateTime(), on_error=None) == datetime(2020, 5, 17, 0, 0) + def test_datetime_iso_basic(): assert perform_cast( "2017-08-24T12:34:56", @@ -93,6 +116,7 @@ def test_datetime_iso_basic(): on_error=None, ) == datetime(2017, 8, 24, 12, 34, 56) + def test_datetime_iso_with_seconds_only(): assert perform_cast( "2017-08-24T00:00:00", @@ -108,6 +132,7 @@ def test_datetime_from_yyyymmdd(): on_error=None, ) == datetime(2017, 8, 24, 0, 0) + def test_datetime_from_dd_mmm_yyyy(): assert perform_cast( "24-AUG-2017", @@ -115,6 +140,7 @@ def test_datetime_from_dd_mmm_yyyy(): on_error=None, ) == datetime(2017, 8, 24, 0, 0) + def test_datetime_from_iso_date(): assert perform_cast( "2017-08-24", @@ -122,12 +148,17 @@ def test_datetime_from_iso_date(): on_error=None, ) == datetime(2017, 8, 24, 0, 0) + def test_datetime_dateutil_strict_accept(): - assert perform_cast( - "2017-08-24 something", - sa.DateTime(), - on_error=None, - ) is None + assert ( + perform_cast( + "2017-08-24 something", + sa.DateTime(), + on_error=None, + ) + is None + ) + def test_datetime_dateutil_exact_match(): assert perform_cast( @@ -140,6 +171,7 @@ def test_datetime_dateutil_exact_match(): def test_datetime_rejects_invalid(): assert perform_cast("2017-99-99", sa.DateTime(), on_error=None) is None + def test_datetime_rejects_garbage(): assert perform_cast("not a date", sa.DateTime(), on_error=None) is None @@ -215,7 +247,10 @@ def test_column_cast_rule_enum_type_resolves_known_value(): # Matches by .value (the raw display text) but returns .name -- sa.Enum's # own default column-storage convention. register_column_cast_rule("authors", "role", enum_type=Role) - assert cast_scalar("first author", sa.String(), table_name="authors", column_name="role") == "FIRST_AUTHOR" + assert ( + cast_scalar("first author", sa.String(), table_name="authors", column_name="role") + == "FIRST_AUTHOR" + ) def test_column_cast_rule_enum_type_records_unknown_value(): @@ -247,7 +282,9 @@ def test_column_with_no_registered_rule_is_unaffected(): def test_perform_cast_threads_table_and_column_name(): register_column_cast_rule("authors", "role", enum_type=Role) assert ( - perform_cast("last author", sa.String(), on_error=None, table_name="authors", column_name="role") + perform_cast( + "last author", sa.String(), on_error=None, table_name="authors", column_name="role" + ) == "LAST_AUTHOR" ) @@ -262,6 +299,7 @@ def test_custom_scalar_cast_rule(): register_column_cast_rule("t", "c", scalar=lambda v: v.upper()) assert cast_scalar("shout", sa.String(), table_name="t", column_name="c") == "SHOUT" + class Grade(IntEnum): LOW = 1 HIGH = 2 @@ -296,9 +334,10 @@ def test_enum_type_unknown_value_still_fails(): # The widened matching must not start accepting non-members. register_column_cast_rule("g", "grade", enum_type=Grade) errors = [] - assert cast_scalar( - 99, sa.Enum(Grade), table_name="g", column_name="grade", on_error=errors.append - ) is None + assert ( + cast_scalar(99, sa.Enum(Grade), table_name="g", column_name="grade", on_error=errors.append) + is None + ) assert errors == [99] @@ -306,7 +345,10 @@ def test_enum_type_unknown_value_still_fails(): def test_enum_type_blank_values_are_null_not_failures(raw): register_column_cast_rule("authors", "role", enum_type=Role) errors = [] - assert cast_scalar( - raw, sa.Enum(Role), table_name="authors", column_name="role", on_error=errors.append - ) is None + assert ( + cast_scalar( + raw, sa.Enum(Role), table_name="authors", column_name="role", on_error=errors.append + ) + is None + ) assert errors == [] diff --git a/tests/loaders/test_dedupe.py b/tests/loaders/test_dedupe.py index 34a6c31..b61e33e 100644 --- a/tests/loaders/test_dedupe.py +++ b/tests/loaders/test_dedupe.py @@ -27,17 +27,18 @@ class DedupTable(Base, CSVLoadableTableInterface): def test_arrow_drop_duplicates_simple(): - table = pa.table({ - "id": [1, 1, 2], - "x": ["a", "a", "b"], - }) + table = pa.table( + { + "id": [1, 1, 2], + "x": ["a", "a", "b"], + } + ) deduped = arrow_drop_duplicates(table, ["id"]) assert deduped.num_rows == 2 assert deduped["id"].to_pylist() == [1, 2] - def test_internal_deduplication(session, engine, tmp_path): Base.metadata.create_all(engine) @@ -58,4 +59,4 @@ def test_internal_deduplication(session, engine, tmp_path): ) session.commit() - assert inserted == 2 \ No newline at end of file + assert inserted == 2 diff --git a/tests/loaders/test_helpers.py b/tests/loaders/test_helpers.py index 75775ba..eb635d2 100644 --- a/tests/loaders/test_helpers.py +++ b/tests/loaders/test_helpers.py @@ -22,6 +22,7 @@ def test_normalised_csv_stream_strips_quoted_header_with_leading_whitespace(): stream = NormalisedCSVStream(io.BytesIO(raw), encoding="utf-8", delimiter=",") assert stream.read() == b"id,name\n1,alpha\n" + def test_column_casting_stats_records_examples(): stats = ColumnCastingStats() stats.record("bad1") @@ -32,6 +33,7 @@ def test_column_casting_stats_records_examples(): assert stats.count == 4 assert len(stats.examples) == 3 + def test_table_casting_stats_aggregation(): stats = TableCastingStats(table_name="test") stats.record(column="a", value="x") @@ -41,16 +43,19 @@ def test_table_casting_stats_aggregation(): assert stats.total_failures == 3 assert stats.has_failures() is True + def test_infer_delim_csv(tmp_path): p = tmp_path / "x.csv" p.write_text("a,b,c\n1,2,3\n") assert infer_delim(p) == "," + def test_infer_delim_tsv(tmp_path): p = tmp_path / "x.tsv" p.write_text("a\tb\tc\n1\t2\t3\n") assert infer_delim(p) == "\t" + def test_infer_encoding_utf8(tmp_path): p = tmp_path / "x.csv" p.write_text("hello") @@ -94,6 +99,7 @@ def test_infer_quote_mode_unbalanced_quote_returns_literal(tmp_path): # resolve_quote_mode # --------------------------------------------------------------------------- + def test_resolve_quote_mode_explicit_csv_passthrough(): # Explicit modes are returned unchanged and never read the file assert resolve_quote_mode("csv", Path("does-not-exist.csv"), ",") == "csv" diff --git a/tests/loaders/test_loader_e2e.py b/tests/loaders/test_loader_e2e.py index 6275c1a..0321b46 100644 --- a/tests/loaders/test_loader_e2e.py +++ b/tests/loaders/test_loader_e2e.py @@ -86,20 +86,18 @@ def test_initial_csv_load_for_a_non_primary_schema_table(session, tmp_path): schema-tag coverage.""" csv_path = tmp_path / "test_vocab_role_table.csv" - pd.DataFrame( - [{"id": 1, "name": "alpha"}, {"id": 2, "name": "beta"}] - ).to_csv(csv_path, index=False, sep="\t") - - inserted = _VocabSchemaTable.load_csv( - session, csv_path, dedupe=False, loader=PandasLoader() + pd.DataFrame([{"id": 1, "name": "alpha"}, {"id": 2, "name": "beta"}]).to_csv( + csv_path, index=False, sep="\t" ) + + inserted = _VocabSchemaTable.load_csv(session, csv_path, dedupe=False, loader=PandasLoader()) session.commit() assert inserted == 2 - rows = session.execute( - sa.select(VocabSchemaTable).order_by(VocabSchemaTable.id) - ).scalars().all() + rows = ( + session.execute(sa.select(VocabSchemaTable).order_by(VocabSchemaTable.id)).scalars().all() + ) assert [(r.id, r.name) for r in rows] == [(1, "alpha"), (2, "beta")] @@ -396,7 +394,9 @@ def test_insert_if_empty_raises_on_non_empty_target(session, engine, tmp_path): pd.DataFrame([{"id": 2, "name": "beta"}]).to_csv(csv_path, index=False, sep="\t") - with pytest.raises(ValueError, match="is not empty; cannot use merge strategy 'insert_if_empty'"): + with pytest.raises( + ValueError, match="is not empty; cannot use merge strategy 'insert_if_empty'" + ): _SimpleTable.load_csv( session, csv_path, @@ -442,7 +442,10 @@ def test_empty_target_routes_merge_to_insert_if_empty(session, tmp_path, caplog, messages = [record.getMessage() for record in caplog.records] - assert any("Checking whether target table is empty for merge optimisation." in message for message in messages) + assert any( + "Checking whether target table is empty for merge optimisation." in message + for message in messages + ) assert any( f"Target table is empty; routing merge strategy `{merge_strategy}` to insert-if-empty fast path." in message diff --git a/tests/loaders/test_pandas_loader.py b/tests/loaders/test_pandas_loader.py index 3cc0e0a..e0f7022 100644 --- a/tests/loaders/test_pandas_loader.py +++ b/tests/loaders/test_pandas_loader.py @@ -5,6 +5,7 @@ from orm_loader.loaders.loader_interface import PandasLoader from tests.models import PandasLoaderTable + def test_loader_context_fields(session): ctx = LoaderContext( @@ -21,6 +22,7 @@ def test_loader_context_fields(session): assert ctx.normalise is False assert ctx.dedupe is True + def test_pandas_loader_case_insensitive_headers(tmp_path, session): """ PandasLoader should accept mixed-case CSV headers and map them @@ -28,11 +30,7 @@ def test_pandas_loader_case_insensitive_headers(tmp_path, session): """ csv = tmp_path / "test_case_headers.csv" - csv.write_text( - "ID,Value\n" - "1,alpha\n" - "2,beta\n" - ) + csv.write_text("ID,Value\n1,alpha\n2,beta\n") ctx = LoaderContext( tableclass=PandasLoaderTable, @@ -46,9 +44,7 @@ def test_pandas_loader_case_insensitive_headers(tmp_path, session): n = PandasLoader.orm_file_load(ctx) - rows = session.execute( - sa.text('SELECT id, value FROM test_pandas_loader ORDER BY id') - ).all() + rows = session.execute(sa.text("SELECT id, value FROM test_pandas_loader ORDER BY id")).all() assert n == 2 assert rows == [(1, "alpha"), (2, "beta")] @@ -62,11 +58,7 @@ def test_pandas_loader_literal_mode_preserves_quotes(tmp_path, session): """ csv = tmp_path / "quoted.csv" - csv.write_text( - 'id,value\n' - '1,"hello"\n' - '2,"world"\n' - ) + csv.write_text('id,value\n1,"hello"\n2,"world"\n') ctx = LoaderContext( tableclass=PandasLoaderTable, @@ -81,9 +73,7 @@ def test_pandas_loader_literal_mode_preserves_quotes(tmp_path, session): n = PandasLoader.orm_file_load(ctx) - rows = session.execute( - sa.text('SELECT id, value FROM test_pandas_loader ORDER BY id') - ).all() + rows = session.execute(sa.text("SELECT id, value FROM test_pandas_loader ORDER BY id")).all() assert n == 2 assert rows == [(1, '"hello"'), (2, '"world"')] @@ -97,11 +87,7 @@ def test_pandas_loader_csv_mode_strips_quotes(tmp_path, session): """ csv = tmp_path / "quoted.csv" - csv.write_text( - 'id,value\n' - '1,"hello"\n' - '2,"world"\n' - ) + csv.write_text('id,value\n1,"hello"\n2,"world"\n') ctx = LoaderContext( tableclass=PandasLoaderTable, @@ -116,9 +102,7 @@ def test_pandas_loader_csv_mode_strips_quotes(tmp_path, session): n = PandasLoader.orm_file_load(ctx) - rows = session.execute( - sa.text('SELECT id, value FROM test_pandas_loader ORDER BY id') - ).all() + rows = session.execute(sa.text("SELECT id, value FROM test_pandas_loader ORDER BY id")).all() assert n == 2 assert rows == [(1, "hello"), (2, "world")] @@ -131,11 +115,7 @@ def test_pandas_loader_by_delimiter_tab_preserves_quotes(tmp_path, session): """ tsv = tmp_path / "tab.tsv" - tsv.write_text( - 'id\tvalue\n' - '1\t"hi"\n' - '2\tbye\n' - ) + tsv.write_text('id\tvalue\n1\t"hi"\n2\tbye\n') ctx = LoaderContext( tableclass=PandasLoaderTable, @@ -150,19 +130,19 @@ def test_pandas_loader_by_delimiter_tab_preserves_quotes(tmp_path, session): n = PandasLoader.orm_file_load(ctx) - rows = session.execute( - sa.text('SELECT id, value FROM test_pandas_loader ORDER BY id') - ).all() + rows = session.execute(sa.text("SELECT id, value FROM test_pandas_loader ORDER BY id")).all() assert n == 2 assert rows == [(1, '"hi"'), (2, "bye")] def test_pandas_dedupe_internal(session, tmp_path): - df = pd.DataFrame({ - "id": [1, 1, 2], - "value": [10, 10, 20], - }) + df = pd.DataFrame( + { + "id": [1, 1, 2], + "value": [10, 10, 20], + } + ) ctx = LoaderContext( tableclass=PandasLoaderTable, diff --git a/tests/loaders/test_pg_loader.py b/tests/loaders/test_pg_loader.py index 20f37cf..ccf7d66 100644 --- a/tests/loaders/test_pg_loader.py +++ b/tests/loaders/test_pg_loader.py @@ -3,7 +3,12 @@ import pytest from orm_loader.backends import STAGING_SCHEMA, resolve_backend from orm_loader.loaders.data.converters import _COLUMN_CAST_RULES, register_column_cast_rule -from orm_loader.loaders.loading_helpers import infer_encoding, infer_delim, check_line_ending, quick_load_pg +from orm_loader.loaders.loading_helpers import ( + infer_encoding, + infer_delim, + check_line_ending, + quick_load_pg, +) from tests.models import EnumTable, Role, SimpleTable @@ -16,18 +21,31 @@ def test_copy_into_staging_with_extra_identity_column(pg_session, tmp_path): SimpleTable.create_staging_table(pg_session, staging_schema_tag=STAGING_SCHEMA) staging_name = resolve_backend(pg_session).staging_name_for_table(SimpleTable.__tablename__) - cols = pg_session.execute(sa.text( - "SELECT column_name FROM information_schema.columns" - " WHERE table_schema = :s AND table_name = :t" - ), {"s": STAGING_SCHEMA, "t": staging_name}).scalars().all() + cols = ( + pg_session.execute( + sa.text( + "SELECT column_name FROM information_schema.columns" + " WHERE table_schema = :s AND table_name = :t" + ), + {"s": STAGING_SCHEMA, "t": staging_name}, + ) + .scalars() + .all() + ) assert "_rownum" in cols, "staging table should have _rownum before COPY" - total = quick_load_pg(path=csv, session=pg_session, tablename=staging_name, schema=STAGING_SCHEMA) + total = quick_load_pg( + path=csv, session=pg_session, tablename=staging_name, schema=STAGING_SCHEMA + ) assert total == 2 - rownums = pg_session.execute( - sa.text(f'SELECT _rownum FROM "{STAGING_SCHEMA}"."{staging_name}" ORDER BY _rownum') - ).scalars().all() + rownums = ( + pg_session.execute( + sa.text(f'SELECT _rownum FROM "{STAGING_SCHEMA}"."{staging_name}" ORDER BY _rownum') + ) + .scalars() + .all() + ) assert rownums == [1, 2], "_rownum must be auto-populated by IDENTITY sequence" @@ -51,7 +69,6 @@ def test_copy_and_orm_path_equivalence(pg_session, tmp_path): ] - def test_postgres_copy_fast_path(pg_session, tmp_path): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -61,6 +78,7 @@ def test_postgres_copy_fast_path(pg_session, tmp_path): assert inserted == 1 + def test_postgres_copy_fast_path_is_used(pg_session, tmp_path, monkeypatch): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -72,6 +90,7 @@ def fake_quick_load_pg(*args, **kwargs): return 1 import orm_loader.loaders.loading_helpers as loading_helpers + monkeypatch.setattr(loading_helpers, "quick_load_pg", fake_quick_load_pg) inserted = SimpleTable.load_csv(pg_session, csv, staging_schema_tag=STAGING_SCHEMA) @@ -80,6 +99,7 @@ def fake_quick_load_pg(*args, **kwargs): assert called["copy"] is True assert inserted == 1 + def test_copy_failure_falls_back_to_orm(pg_session, tmp_path, monkeypatch): csv = tmp_path / "test_table.csv" pd.DataFrame([{"id": 1, "name": "alpha"}]).to_csv(csv, index=False) @@ -111,7 +131,9 @@ def test_postgres_upsert_does_not_update(pg_session, tmp_path): pd.DataFrame([{"id": 1, "name": "alpha_updated"}]).to_csv(csv, index=False) - SimpleTable.load_csv(pg_session, csv, merge_strategy="upsert", staging_schema_tag=STAGING_SCHEMA) + SimpleTable.load_csv( + pg_session, csv, merge_strategy="upsert", staging_schema_tag=STAGING_SCHEMA + ) pg_session.commit() rows = pg_session.execute(sa.select(SimpleTable)).scalars().all() @@ -154,7 +176,9 @@ def test_postgres_insert_if_empty_raises_on_non_empty_target(pg_session, tmp_pat pd.DataFrame([{"id": 2, "name": "beta"}]).to_csv(csv, index=False) - with pytest.raises(ValueError, match="is not empty; cannot use merge strategy 'insert_if_empty'"): + with pytest.raises( + ValueError, match="is not empty; cannot use merge strategy 'insert_if_empty'" + ): SimpleTable.load_csv( pg_session, csv, @@ -166,15 +190,13 @@ def test_postgres_insert_if_empty_raises_on_non_empty_target(pg_session, tmp_pat def test_postgres_copy_large_batch(pg_session, tmp_path): csv = tmp_path / "test_table.csv" - df = pd.DataFrame( - [{"id": i, "name": f"name_{i}"} for i in range(1, 10_000)] - ) + df = pd.DataFrame([{"id": i, "name": f"name_{i}"} for i in range(1, 10_000)]) df.to_csv(csv, index=False) inserted = SimpleTable.load_csv(pg_session, csv, staging_schema_tag=STAGING_SCHEMA) pg_session.commit() - count = pg_session.execute(sa.text('SELECT COUNT(*) FROM test_table')).scalar() + count = pg_session.execute(sa.text("SELECT COUNT(*) FROM test_table")).scalar() assert count == 9999 assert inserted == 9999 @@ -186,12 +208,15 @@ def test_staging_schema_matches_target(pg_session, tmp_path): SimpleTable.create_staging_table(pg_session, staging_schema_tag=STAGING_SCHEMA) staging_name = resolve_backend(pg_session).staging_name_for_table(SimpleTable.__tablename__) - cols = pg_session.execute(sa.text(""" + cols = pg_session.execute( + sa.text(""" SELECT column_name, data_type FROM information_schema.columns WHERE table_schema = :s AND table_name = :table ORDER BY ordinal_position - """), {"s": STAGING_SCHEMA, "table": staging_name}).all() + """), + {"s": STAGING_SCHEMA, "table": staging_name}, + ).all() data_cols = [(name, dtype) for name, dtype in cols if name != "_rownum"] assert data_cols == [ @@ -208,6 +233,7 @@ def test_infer_encoding_ascii_promoted_to_utf8(tmp_path): enc = infer_encoding(p) assert enc["encoding"] == "utf-8" + @pytest.mark.xfail(reason="chardet todo - finalise encoding inference") def test_infer_encoding_utf8(tmp_path): p = tmp_path / "utf8.csv" @@ -218,18 +244,21 @@ def test_infer_encoding_utf8(tmp_path): assert enc_str is not None assert enc_str.lower().startswith("utf") + def test_infer_delim_tab(tmp_path): p = tmp_path / "tab.csv" p.write_text("id\tname\n1\talpha\n") assert infer_delim(p) == "\t" + def test_infer_delim_comma(tmp_path): p = tmp_path / "comma.csv" p.write_text("id,name\n1,alpha\n") assert infer_delim(p) == "," + @pytest.mark.parametrize( "raw,expected", [ @@ -259,7 +288,7 @@ def test_quick_load_pg_basic(pg_session, tmp_path): tablename="test_table", ) - rows = pg_session.execute(sa.text('SELECT id, name FROM test_table ORDER BY id')).all() + rows = pg_session.execute(sa.text("SELECT id, name FROM test_table ORDER BY id")).all() assert total == 2 assert rows == [(1, "alpha"), (2, "beta")] @@ -269,7 +298,7 @@ def test_quick_load_pg_lowercases_header(pg_session, tmp_path): csv.write_text("ID,NAME\n1,alpha\n") total = quick_load_pg(path=csv, session=pg_session, tablename="test_table") - row = pg_session.execute(sa.text('SELECT id, name FROM test_table')).one() + row = pg_session.execute(sa.text("SELECT id, name FROM test_table")).one() assert total == 1 assert row == (1, "alpha") @@ -324,10 +353,13 @@ def test_quick_load_pg_equivalence_with_orm(pg_session, tmp_path): pg_session.commit() df = pd.read_csv(csv) - pg_session.execute(sa.text(""" + pg_session.execute( + sa.text(""" INSERT INTO test_table (id, name) VALUES (:id, :name) - """), df.to_dict(orient="records")) + """), + df.to_dict(orient="records"), + ) pg_session.commit() rows_orm = pg_session.execute(sa.text("SELECT id, name FROM test_table ORDER BY id")).all() @@ -338,30 +370,23 @@ def test_quick_load_pg_equivalence_with_orm(pg_session, tmp_path): def test_quick_load_pg_trailing_blank_lines(pg_session, tmp_path): csv = tmp_path / "test_table.csv" - csv.write_text( - "id,name\n" - "1,alpha\r\n" - "2,beta\r\n" - ) + csv.write_text("id,name\n1,alpha\r\n2,beta\r\n") total = quick_load_pg(path=csv, session=pg_session, tablename="test_table") - rows = pg_session.execute( - sa.text("SELECT id, name FROM test_table ORDER BY id") - ).all() + rows = pg_session.execute(sa.text("SELECT id, name FROM test_table ORDER BY id")).all() assert total == 2 assert rows == [(1, "alpha"), (2, "beta")] -def test_copy_fails_with_raw_carriage_returns_but_succeeds_after_normalisation(pg_session, tmp_path): + +def test_copy_fails_with_raw_carriage_returns_but_succeeds_after_normalisation( + pg_session, tmp_path +): csv = tmp_path / "test_table.csv" # Intentionally include CRLF to simulate Windows-originated files - raw = ( - b"id,name\r\n" - b"1,alpha\r\n" - b"2,beta\r\n" - ) + raw = b"id,name\r\n1,alpha\r\n2,beta\r\n" csv.write_bytes(raw) @@ -385,9 +410,7 @@ def test_copy_fails_with_raw_carriage_returns_but_succeeds_after_normalisation(p total = quick_load_pg(path=csv, session=pg_session, tablename="test_table") pg_session.commit() - rows = pg_session.execute( - sa.text('SELECT id, name FROM test_table ORDER BY id') - ).all() + rows = pg_session.execute(sa.text("SELECT id, name FROM test_table ORDER BY id")).all() assert total == 2 assert rows == [(1, "alpha"), (2, "beta")] diff --git a/tests/loaders/test_schema_translate_map.py b/tests/loaders/test_schema_translate_map.py index 9d6a15b..464cd2c 100644 --- a/tests/loaders/test_schema_translate_map.py +++ b/tests/loaders/test_schema_translate_map.py @@ -30,7 +30,10 @@ def test_load_csv_respects_non_default_schema_end_to_end(pg_db, tmp_path): - with schema_scoped_session(pg_db, SimpleTable.__table__, prefix="loader_primary") as (scoped, session): + with schema_scoped_session(pg_db, SimpleTable.__table__, prefix="loader_primary") as ( + scoped, + session, + ): schema = scoped.schemas[SchemaRole.PRIMARY] csv_path = tmp_path / "test_table.csv" pd.DataFrame( @@ -38,7 +41,11 @@ def test_load_csv_respects_non_default_schema_end_to_end(pg_db, tmp_path): ).to_csv(csv_path, index=False, sep="\t") inserted = SimpleTable.load_csv( - session, csv_path, dedupe=False, loader=PandasLoader(), staging_schema_tag=STAGING_SCHEMA + session, + csv_path, + dedupe=False, + loader=PandasLoader(), + staging_schema_tag=STAGING_SCHEMA, ) session.commit() @@ -61,7 +68,10 @@ def test_replace_merge_respects_non_default_schema_end_to_end(pg_db, tmp_path): """A second load_csv() call with merge_strategy="replace" against the same non-default schema. Proves the merge path itself, not just the initial insert-if-empty fast path, qualifies correctly.""" - with schema_scoped_session(pg_db, SimpleTable.__table__, prefix="loader_replace") as (scoped, session): + with schema_scoped_session(pg_db, SimpleTable.__table__, prefix="loader_replace") as ( + scoped, + session, + ): schema = scoped.schemas[SchemaRole.PRIMARY] def _write_and_load(rows: list[dict], path_name: str) -> int: @@ -102,7 +112,11 @@ def test_load_csv_respects_non_primary_schema_tag_end_to_end(pg_db, tmp_path): ) inserted = VocabSchemaTable.load_csv( - session, csv_path, dedupe=False, loader=PandasLoader(), staging_schema_tag=STAGING_SCHEMA + session, + csv_path, + dedupe=False, + loader=PandasLoader(), + staging_schema_tag=STAGING_SCHEMA, ) session.commit() @@ -110,7 +124,9 @@ def test_load_csv_respects_non_primary_schema_tag_end_to_end(pg_db, tmp_path): with scoped.engine.connect() as conn: rows = conn.execute( - sa.text(f'SELECT id, name FROM "{vocab_schema}"."test_vocab_role_table" ORDER BY id') + sa.text( + f'SELECT id, name FROM "{vocab_schema}"."test_vocab_role_table" ORDER BY id' + ) ).fetchall() leaked = conn.execute( sa.text(f"SELECT to_regclass('{primary_schema}.test_vocab_role_table')") diff --git a/tests/loaders/test_split_connection.py b/tests/loaders/test_split_connection.py index 36c1fec..d6ee2b5 100644 --- a/tests/loaders/test_split_connection.py +++ b/tests/loaders/test_split_connection.py @@ -1,5 +1,4 @@ -"""Tests split CDM/vocab connection instances. -""" +"""Tests split CDM/vocab connection instances.""" from __future__ import annotations @@ -33,9 +32,7 @@ def test_vocab_csv_load_routes_every_session_bind_to_vocab_file(tmp_path): ) ) resolved = resolver.resolve_database("test_cdm") - primary_engine, vocab_engine = resolved.create_engines( - schema_claims=[staging_schema_claim()] - ) + primary_engine, vocab_engine = resolved.create_engines(schema_claims=[staging_schema_claim()]) VocabSchemaTable.__table__.create(vocab_engine) class MapperRoutedSession(so.Session): @@ -47,7 +44,9 @@ def get_bind(self, mapper=None, clause=None, **kwargs): if mapper is None and clause is None: raise UnboundExecutionError("a mapper or clause is required") if mapper is not None: - engine = vocab_engine if mapper.__table__.schema == SchemaRole.VOCAB else primary_engine + engine = ( + vocab_engine if mapper.__table__.schema == SchemaRole.VOCAB else primary_engine + ) self._last_mapper_engine = engine return engine table = getattr(clause, "table", None) @@ -72,9 +71,7 @@ def get_bind(self, mapper=None, clause=None, **kwargs): session.close() with vocab_engine.connect() as connection: - rows = connection.execute( - sa.select(VocabSchemaTable.id, VocabSchemaTable.name) - ).all() + rows = connection.execute(sa.select(VocabSchemaTable.id, VocabSchemaTable.name)).all() assert rows == [(7, "vocab-row")] assert not sa.inspect(primary_engine).has_table("test_vocab_role_table") primary_engine.dispose() @@ -86,10 +83,15 @@ def test_load_csv_across_two_genuinely_separate_connections(pg_db, session, tmp_ SQLite, vocab connection, via the module-level ``engine``/``session`` fixtures) are two entirely different engines against two entirely different database systems.""" - with schema_scoped_session(pg_db, SimpleTable.__table__, prefix="loader_split") as (scoped, primary_session): + with schema_scoped_session(pg_db, SimpleTable.__table__, prefix="loader_split") as ( + scoped, + primary_session, + ): primary_schema = scoped.schemas[SchemaRole.PRIMARY] primary_csv = tmp_path / "test_table.csv" - pd.DataFrame([{"id": 1, "name": "primary-alpha"}]).to_csv(primary_csv, index=False, sep="\t") + pd.DataFrame([{"id": 1, "name": "primary-alpha"}]).to_csv( + primary_csv, index=False, sep="\t" + ) vocab_csv = tmp_path / "test_vocab_role_table.csv" pd.DataFrame([{"id": 1, "name": "vocab-alpha"}]).to_csv(vocab_csv, index=False, sep="\t") @@ -114,7 +116,9 @@ def test_load_csv_across_two_genuinely_separate_connections(pg_db, session, tmp_ assert primary_rows == [(1, "primary-alpha")] assert leaked_vocab_table_in_pg is None - vocab_rows = session.execute( - sa.select(VocabSchemaTable).order_by(VocabSchemaTable.id) - ).scalars().all() + vocab_rows = ( + session.execute(sa.select(VocabSchemaTable).order_by(VocabSchemaTable.id)) + .scalars() + .all() + ) assert [(r.id, r.name) for r in vocab_rows] == [(1, "vocab-alpha")] diff --git a/tests/mappers/test_materialised_view_contracts.py b/tests/mappers/test_materialised_view_contracts.py index b5c8a45..ae4a072 100644 --- a/tests/mappers/test_materialised_view_contracts.py +++ b/tests/mappers/test_materialised_view_contracts.py @@ -55,7 +55,9 @@ def test_materialized_view_index_defaults_to_non_unique(): def test_drop_materialized_view_default_compiles_with_if_exists(): ddl = DropMaterializedView("reporting.mv_test") - assert str(ddl.compile(dialect=_DIALECT)) == "DROP MATERIALIZED VIEW IF EXISTS reporting.mv_test" + assert ( + str(ddl.compile(dialect=_DIALECT)) == "DROP MATERIALIZED VIEW IF EXISTS reporting.mv_test" + ) def test_drop_materialized_view_if_exists_false_and_cascade(): @@ -69,8 +71,7 @@ def test_create_materialized_view_index_compiles_unique_index_ddl(): ddl = CreateMaterializedViewIndex("reporting.mv_test", index) assert str(ddl.compile(dialect=_DIALECT)) == ( - 'CREATE UNIQUE INDEX IF NOT EXISTS "mv_test_id_uq" ' - 'ON reporting.mv_test ("id", "person_id")' + 'CREATE UNIQUE INDEX IF NOT EXISTS "mv_test_id_uq" ON reporting.mv_test ("id", "person_id")' ) diff --git a/tests/mappers/test_materialised_view_mixin.py b/tests/mappers/test_materialised_view_mixin.py index e4cf523..407e402 100644 --- a/tests/mappers/test_materialised_view_mixin.py +++ b/tests/mappers/test_materialised_view_mixin.py @@ -48,7 +48,11 @@ def test_refresh_all_mvs_resolves_each_views_own_schema_tag(pg_db) -> None: class _IndexedPgMv(MaterializedViewMixin): __mv_name__ = "mv_indexed_pg_test" __mv_select__ = sa.select(sa.literal(1).label("row_id")) - __mv_indexes__ = (MaterializedViewIndex(name="mv_indexed_pg_test_row_id_uq", columns=("row_id",), unique=True),) + __mv_indexes__ = ( + MaterializedViewIndex( + name="mv_indexed_pg_test_row_id_uq", columns=("row_id",), unique=True + ), + ) def test_create_mv_with_declared_index_against_postgres(pg_db) -> None: @@ -63,7 +67,9 @@ def test_create_mv_with_declared_index_against_postgres(pg_db) -> None: with scoped.engine.connect() as conn: inspector = sa.inspect(conn) assert inspector.has_table("mv_indexed_pg_test", schema=schema) - index_names = {idx["name"] for idx in inspector.get_indexes("mv_indexed_pg_test", schema=schema)} + index_names = { + idx["name"] for idx in inspector.get_indexes("mv_indexed_pg_test", schema=schema) + } assert "mv_indexed_pg_test_row_id_uq" in index_names @@ -87,8 +93,8 @@ def test_drop_mv_cascade_against_postgres(pg_db) -> None: def test_refresh_mv_concurrently_against_postgres(pg_db) -> None: """refresh_mv(concurrently=True) through the mixin succeeds against a real database when a unique index is declared. - The eligibility check and error translation are already proven at the - backend level directly in test_postgres_backend.py. + The eligibility check and error translation are already proven at the + backend level directly in test_postgres_backend.py. This proves the mixin wires into that correctly end to end.""" with scoped_test_schema(pg_db.resolved, prefix="mv_refresh") as scoped: with scoped.engine.begin() as conn: @@ -264,12 +270,12 @@ def test_create_mv_forwards_if_not_exists_to_declared_indexes(fake_backend: _Fak assert fake_backend.calls[1][2] == {"schema": None, "if_not_exists": False} -def test_create_mv_engine_uses_one_transaction_for_view_and_indexes(monkeypatch: pytest.MonkeyPatch): +def test_create_mv_engine_uses_one_transaction_for_view_and_indexes( + monkeypatch: pytest.MonkeyPatch, +): engine = sa.create_engine("sqlite:///:memory:", future=True) with engine.begin() as connection: - connection.execute( - sa.text("CREATE TABLE materialization_steps (step TEXT NOT NULL)") - ) + connection.execute(sa.text("CREATE TABLE materialization_steps (step TEXT NOT NULL)")) connections: list[sa.engine.Connection] = [] fail_once = True @@ -277,16 +283,12 @@ def test_create_mv_engine_uses_one_transaction_for_view_and_indexes(monkeypatch: class TransactionalBackend(_FakeBackend): def create_materialized_view(self, bind, name, selectable, **kwargs): connections.append(bind) - bind.execute( - sa.text("INSERT INTO materialization_steps (step) VALUES ('view')") - ) + bind.execute(sa.text("INSERT INTO materialization_steps (step) VALUES ('view')")) def create_materialized_view_index(self, bind, name, index, **kwargs): nonlocal fail_once connections.append(bind) - bind.execute( - sa.text("INSERT INTO materialization_steps (step) VALUES ('index')") - ) + bind.execute(sa.text("INSERT INTO materialization_steps (step) VALUES ('index')")) if fail_once: fail_once = False raise RuntimeError("index failed") @@ -305,9 +307,10 @@ class TransactionalMv(MaterializedViewMixin): assert connections[0] is connections[1] with engine.connect() as connection: - assert connection.execute( - sa.text("SELECT COUNT(*) FROM materialization_steps") - ).scalar_one() == 0 + assert ( + connection.execute(sa.text("SELECT COUNT(*) FROM materialization_steps")).scalar_one() + == 0 + ) TransactionalMv.create_mv(engine) with engine.connect() as connection: diff --git a/tests/models.py b/tests/models.py index efc3b17..b0fc084 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,4 +1,3 @@ - from enum import Enum import sqlalchemy as sa @@ -19,6 +18,7 @@ class Flag(str, Enum): STANDARD = "S" CLASSIFICATION = "C" + class PandasLoaderTable(CSVLoadableTableInterface, Base): __tablename__ = "test_pandas_loader" __table_args__ = {"schema": SchemaRole.PRIMARY.value} diff --git a/tests/registry/test_model_descriptor.py b/tests/registry/test_model_descriptor.py index ac1e5ea..e33eec6 100644 --- a/tests/registry/test_model_descriptor.py +++ b/tests/registry/test_model_descriptor.py @@ -35,9 +35,7 @@ class Child(ORMTableBase, Base): desc = ModelDescriptor.from_model(Child) - assert desc.foreign_keys == { - "parent_id": ("parent", "id") - } + assert desc.foreign_keys == {"parent_id": ("parent", "id")} def test_model_descriptor_rejects_unmapped_class(): @@ -45,4 +43,4 @@ class NotMapped: pass with pytest.raises(NoInspectionAvailable): - ModelDescriptor.from_model(NotMapped) # type: ignore + ModelDescriptor.from_model(NotMapped) # type: ignore diff --git a/tests/registry/test_registry.py b/tests/registry/test_registry.py index 81d4736..071c3f0 100644 --- a/tests/registry/test_registry.py +++ b/tests/registry/test_registry.py @@ -13,8 +13,10 @@ Base = so.declarative_base() + class FakeResource: """Minimal stand-in for importlib.resources API.""" + def __init__(self, text: str): self._text = text @@ -54,7 +56,6 @@ def test_load_field_specs(): assert person_fields["gender_concept_id"].fk_table == "Concept" - def test_register_single_model(): class B(ORMTableBase, Base): __tablename__ = "b" @@ -88,7 +89,7 @@ def test_missing_required_tables(): reg = ModelRegistry(model_version="1.0") # Fake specs injected directly (unit-level test) - reg._table_specs = { # type: ignore + reg._table_specs = { # type: ignore "person": type("S", (), {"is_required": True})(), "visit": type("S", (), {"is_required": False})(), } @@ -102,11 +103,11 @@ class R(ORMTableBase, Base): id = sa.Column(sa.Integer, primary_key=True) reg = ModelRegistry(model_version="1.0") - reg._table_specs = { # type: ignore + reg._table_specs = { # type: ignore "r": type("S", (), {"is_required": True})(), "x": type("S", (), {"is_required": True})(), } reg.register_model(R) assert reg.known_tables() == {"r", "x"} - assert reg.registered_tables() == {"r"} \ No newline at end of file + assert reg.registered_tables() == {"r"} diff --git a/tests/registry/test_validation_e2e.py b/tests/registry/test_validation_e2e.py index af5c003..343bc94 100644 --- a/tests/registry/test_validation_e2e.py +++ b/tests/registry/test_validation_e2e.py @@ -9,6 +9,7 @@ Base = so.declarative_base() + class FakeCSVResource: def __init__(self, text: str): self._text = text @@ -16,6 +17,7 @@ def __init__(self, text: str): def open(self, *_, **__): return io.StringIO(self._text) + def test_validation_runner_happy_path(): class Person(ORMTableBase, Base): __tablename__ = "person" @@ -105,10 +107,7 @@ class Concept(ORMTableBase, Base): runner = ValidationRunner(validators=always_on_validators()) report = runner.run(registry) - assert any( - i.message == "PRIMARY_KEY_COLUMN_NULLABLE" - for i in report.issues - ) + assert any(i.message == "PRIMARY_KEY_COLUMN_NULLABLE" for i in report.issues) def test_foreign_key_not_in_spec_warns(): @@ -128,8 +127,7 @@ class Child(ORMTableBase, Base): ) field_csv = FakeCSVResource( - - """cdmTableName,cdmFieldName,isRequired,cdmDatatype,isPrimaryKey,isForeignKey + """cdmTableName,cdmFieldName,isRequired,cdmDatatype,isPrimaryKey,isForeignKey Child,id,Yes,integer,Yes,No Child,parent_id,No,integer,No,No """ @@ -143,10 +141,8 @@ class Child(ORMTableBase, Base): runner = ValidationRunner(validators=always_on_validators()) report = runner.run(registry) - assert any( - i.message == "FOREIGN_KEY_NOT_IN_SPEC" - for i in report.issues - ) + assert any(i.message == "FOREIGN_KEY_NOT_IN_SPEC" for i in report.issues) + def test_fail_fast_stops_on_first_error(): class Drug(ORMTableBase, Base): diff --git a/tests/tables/test_id_allocator.py b/tests/tables/test_id_allocator.py index e6adaf8..669db68 100644 --- a/tests/tables/test_id_allocator.py +++ b/tests/tables/test_id_allocator.py @@ -1,10 +1,12 @@ from orm_loader.tables.allocators import IdAllocator + def test_allocator_sequence(): alloc = IdAllocator(start=5) assert alloc.next() == 6 assert alloc.next() == 7 + def test_allocator_reserve(): alloc = IdAllocator(start=0) r = alloc.reserve(3) diff --git a/tests/tables/test_serialisable_table.py b/tests/tables/test_serialisable_table.py index ce989e8..9e123b7 100644 --- a/tests/tables/test_serialisable_table.py +++ b/tests/tables/test_serialisable_table.py @@ -5,19 +5,20 @@ Base = declarative_base() + class ExampleTable(SerialisableTableInterface, Base): __tablename__ = "example" - id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) + id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) a: so.Mapped[int] = so.mapped_column(sa.Integer, nullable=False) b: so.Mapped[int | None] = so.mapped_column(sa.Integer, nullable=True) def test_to_dict_excludes_nulls(): obj = ExampleTable() - obj.id=1 - obj.a=10 - obj.b=None + obj.id = 1 + obj.a = 10 + obj.b = None out = obj.to_dict() @@ -30,32 +31,32 @@ def test_to_dict_excludes_nulls(): def test_to_dict_includes_nulls_when_requested(): obj = ExampleTable() - obj.id=1 - obj.a=10 - obj.b=None + obj.id = 1 + obj.a = 10 + obj.b = None out = obj.to_dict(include_nulls=True) assert out["b"] is None -def test_fingerprint_is_stable(): + +def test_fingerprint_is_stable(): obj = ExampleTable() - obj.id=1 - obj.a=10 - obj.b=20 + obj.id = 1 + obj.a = 10 + obj.b = 20 fp1 = obj.fingerprint() fp2 = obj.fingerprint() assert fp1 == fp2 + def test_to_dict_only_and_exclude(): obj = ExampleTable() - obj.id=1 - obj.a=10 - obj.b=20 + obj.id = 1 + obj.a = 10 + obj.b = 20 assert obj.to_dict(only={"a"}) == {"a": 10} assert obj.to_dict(exclude={"b"}) == {"id": 1, "a": 10} - - diff --git a/tests/tables/test_typing_protocols.py b/tests/tables/test_typing_protocols.py index 0aa29c3..1f36e7a 100644 --- a/tests/tables/test_typing_protocols.py +++ b/tests/tables/test_typing_protocols.py @@ -11,6 +11,7 @@ Base = so.declarative_base() + def test_orm_table_protocol_conformance(): class Q(ORMTableBase, Base): __tablename__ = "f" @@ -22,6 +23,7 @@ class Q(ORMTableBase, Base): assert callable(Q.pk_names) assert callable(Q.model_columns) + def test_serialized_table_protocol_conformance(): class G(SerialisableTableInterface, ORMTableBase, Base): __tablename__ = "g" @@ -38,13 +40,14 @@ class G(SerialisableTableInterface, ORMTableBase, Base): assert callable(obj.to_json) assert callable(obj.fingerprint) + def test_csv_table_protocol_conformance(): class P(CSVLoadableTableInterface, ORMTableBase, Base): __tablename__ = "p" id = sa.Column(sa.Integer, primary_key=True) - assert isinstance(P, CSVTableProtocol) + assert isinstance(P, CSVTableProtocol) # Required class-level API assert callable(P.load_csv) assert callable(P.create_staging_table) - assert callable(P.merge_from_staging) \ No newline at end of file + assert callable(P.merge_from_staging) From e13c1800d398bcf7e19f42104103582bf148c221 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 22 Oct 2026 02:42:49 +0000 Subject: [PATCH 3/8] Ignore the format commit in blame --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..97a40a8 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# ruff format, line length 100 +b84a31eee8373113ba20d9e89f4ff5b3d98cc58b From 52083402a2c1347b5821dc7851c112adadab6ea9 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 22 Oct 2026 03:07:07 +0000 Subject: [PATCH 4/8] Apply safe ruff fixes --- src/orm_loader/backends/__init__.py | 22 ++++++------ src/orm_loader/backends/base.py | 35 +++++++++---------- src/orm_loader/backends/postgres.py | 20 +++++------ src/orm_loader/backends/sqlite.py | 12 +++---- src/orm_loader/helpers/__init__.py | 20 +++++------ src/orm_loader/helpers/bulk.py | 3 +- src/orm_loader/helpers/discovery.py | 1 + src/orm_loader/helpers/null_handlers.py | 3 +- src/orm_loader/loaders/__init__.py | 6 ++-- src/orm_loader/loaders/data/__init__.py | 4 +-- src/orm_loader/loaders/data/converters.py | 16 +++++---- src/orm_loader/loaders/data_classes.py | 15 ++++---- src/orm_loader/loaders/loader_interface.py | 24 +++++++------ src/orm_loader/mappers/__init__.py | 12 +++---- .../mappers/materialised_view_mixin.py | 4 +-- src/orm_loader/registry/__init__.py | 14 ++++---- src/orm_loader/registry/registry.py | 19 +++++----- src/orm_loader/registry/validation.py | 6 ++-- src/orm_loader/registry/validation_presets.py | 2 +- src/orm_loader/registry/validation_report.py | 14 ++++---- src/orm_loader/registry/validation_runner.py | 3 +- src/orm_loader/tables/__init__.py | 8 ++--- src/orm_loader/tables/loadable_table.py | 23 ++++++------ src/orm_loader/tables/orm_table.py | 8 +++-- src/orm_loader/tables/serialisable_table.py | 8 ++--- src/orm_loader/tables/typing.py | 23 ++++++------ tests/backends/test_base_backend.py | 24 ++++++------- tests/backends/test_postgres_backend.py | 19 +++++----- tests/backends/test_reserved_schema.py | 2 +- tests/backends/test_shared_backend.py | 26 +++++++------- tests/backends/test_sqlite_backend.py | 4 +-- tests/conftest.py | 2 +- tests/loaders/test_casting.py | 5 +-- tests/loaders/test_dedupe.py | 14 ++++---- tests/loaders/test_helpers.py | 3 +- tests/loaders/test_loader_e2e.py | 22 ++++++------ tests/loaders/test_pandas_loader.py | 4 ++- tests/loaders/test_parquet_loader.py | 11 +++--- tests/loaders/test_pg_loader.py | 12 +++---- tests/loaders/test_schema_translate_map.py | 1 - .../test_materialised_view_contracts.py | 3 +- tests/mappers/test_materialised_view_mixin.py | 2 +- tests/models.py | 3 +- tests/registry/test_model_descriptor.py | 3 +- tests/registry/test_registry.py | 11 +++--- tests/registry/test_validation_e2e.py | 3 +- tests/tables/test_serialisable_table.py | 3 +- tests/tables/test_typing_protocols.py | 5 +-- 48 files changed, 264 insertions(+), 243 deletions(-) diff --git a/src/orm_loader/backends/__init__.py b/src/orm_loader/backends/__init__.py index 2c9c799..32a5841 100644 --- a/src/orm_loader/backends/__init__.py +++ b/src/orm_loader/backends/__init__.py @@ -1,21 +1,22 @@ -from .postgres import PostgresBackend -from .resolve import resolve_backend -from .sqlite import SQLiteBackend -from .base import ( - BackendCapabilities, - DatabaseBackend, - Dialect, - STAGING_SCHEMA, - staging_schema_claim, -) from ..mappers.materialised_view_errors import ( ConcurrentRefreshNotEligibleError, MaterializationError, MaterializationFailure, MaterializationOperation, ) +from .base import ( + STAGING_SCHEMA, + BackendCapabilities, + DatabaseBackend, + Dialect, + staging_schema_claim, +) +from .postgres import PostgresBackend +from .resolve import resolve_backend +from .sqlite import SQLiteBackend __all__ = [ + "STAGING_SCHEMA", "BackendCapabilities", "ConcurrentRefreshNotEligibleError", "DatabaseBackend", @@ -24,7 +25,6 @@ "MaterializationFailure", "MaterializationOperation", "PostgresBackend", - "STAGING_SCHEMA", "SQLiteBackend", "resolve_backend", "staging_schema_claim", diff --git a/src/orm_loader/backends/base.py b/src/orm_loader/backends/base.py index 7224256..42d425c 100644 --- a/src/orm_loader/backends/base.py +++ b/src/orm_loader/backends/base.py @@ -1,19 +1,18 @@ from __future__ import annotations from abc import ABC, abstractmethod +from collections.abc import Callable, Generator from contextlib import AbstractContextManager, contextmanager from dataclasses import dataclass -from collections.abc import Generator from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, ParamSpec, Type, TypeVar, cast +from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Dialect, SchemaClaim, open_connection, qualified from sqlalchemy.engine import Connection, Engine from sqlalchemy.sql.compiler import IdentifierPreparer -from oa_configurator import Dialect, SchemaClaim, open_connection, qualified - if TYPE_CHECKING: from ..loaders.data_classes import LoaderContext from ..mappers.materialised_view_contracts import MaterializedViewIndex @@ -263,7 +262,7 @@ def _as_connection( def _insertable_column_names( self, - table_cls: Type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], ) -> list[str]: """ Return column names safe to include in generic insert statements. @@ -276,7 +275,7 @@ def _insertable_column_names( @abstractmethod def create_staging_table( self, - table_cls: Type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, ) -> None: """Create a staging table for the supplied ORM table class.""" @@ -284,14 +283,14 @@ def create_staging_table( @abstractmethod def drop_staging_table( self, - table_cls: Type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, ) -> None: """Drop a staging table if it exists.""" def load_staging_fast( self, - loader_context: "LoaderContext", + loader_context: LoaderContext, ) -> int | None: """ Attempt a backend-native fast-path load. @@ -331,7 +330,7 @@ def restore_fk_check( @abstractmethod def merge_replace( self, - table_cls: Type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, pk_cols: list[str], *, @@ -342,7 +341,7 @@ def merge_replace( @abstractmethod def merge_upsert( self, - table_cls: Type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, pk_cols: list[str], *, @@ -353,7 +352,7 @@ def merge_upsert( @abstractmethod def merge_insert( self, - table_cls: Type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, *, merge_batch_size: int | None = None, @@ -362,7 +361,7 @@ def merge_insert( def merge_context( self, - table_cls: Type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, ) -> AbstractContextManager[None]: """Disable backend FK checks for a table's merge and restore them afterward.""" @@ -413,7 +412,7 @@ def bulk_load_context( @requires_capability("supports_materialized_views", "materialized views") def create_materialized_view( self, - bind: "Engine | Connection", + bind: Engine | Connection, name: str, selectable: sa.sql.Select[Any], *, @@ -434,12 +433,12 @@ def create_materialized_view( @requires_capability("supports_materialized_views", "materialized views") def refresh_materialized_view( self, - bind: "Engine | Connection", + bind: Engine | Connection, name: str, *, schema: str | None = None, concurrently: bool = False, - declared_indexes: tuple["MaterializedViewIndex", ...] = (), + declared_indexes: tuple[MaterializedViewIndex, ...] = (), ) -> None: """Refresh a materialized view. @@ -457,7 +456,7 @@ def refresh_materialized_view( @requires_capability("supports_materialized_views", "materialized views") def drop_materialized_view( self, - bind: "Engine | Connection", + bind: Engine | Connection, name: str, *, schema: str | None = None, @@ -480,9 +479,9 @@ def drop_materialized_view( @requires_capability("supports_materialized_views", "materialized views") def create_materialized_view_index( self, - bind: "Engine | Connection", + bind: Engine | Connection, name: str, - index: "MaterializedViewIndex", + index: MaterializedViewIndex, *, schema: str | None = None, if_not_exists: bool = True, diff --git a/src/orm_loader/backends/postgres.py b/src/orm_loader/backends/postgres.py index 7c49533..54a30aa 100644 --- a/src/orm_loader/backends/postgres.py +++ b/src/orm_loader/backends/postgres.py @@ -67,7 +67,7 @@ def capabilities(self) -> BackendCapabilities: def create_staging_table( self, - table_cls: type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, ) -> None: table = table_cls.__table__ @@ -111,7 +111,7 @@ def create_staging_table( def drop_staging_table( self, - table_cls: type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, ) -> None: session.execute( @@ -121,7 +121,7 @@ def drop_staging_table( def load_staging_fast( self, - loader_context: "LoaderContext", + loader_context: LoaderContext, ) -> int | None: from ..loaders.loading_helpers import quick_load_pg @@ -186,7 +186,7 @@ def restore_fk_check( ) def _staging_rownum_index( - self, table_cls: type["CSVTableProtocol"], staging: sa.Table, session: so.Session + self, table_cls: type[CSVTableProtocol], staging: sa.Table, session: so.Session ) -> None: staging_name = self.staging_name_for_table(table_cls.__tablename__) idx = sa.Index(f"{staging_name}_rownum_idx", staging.c._rownum) @@ -218,7 +218,7 @@ def _staging_rownum_bound(staging: sa.Table, session: so.Session) -> int: def merge_replace( self, - table_cls: type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, pk_cols: list[str], *, @@ -263,7 +263,7 @@ def merge_replace( def merge_upsert( self, - table_cls: type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, pk_cols: list[str], *, @@ -308,7 +308,7 @@ def _upsert(select_: Any) -> sa.Insert: def merge_insert( self, - table_cls: type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, *, merge_batch_size: int | None = None, @@ -390,7 +390,7 @@ def refresh_materialized_view( *, schema: str | None = None, concurrently: bool = False, - declared_indexes: tuple["MaterializedViewIndex", ...] = (), + declared_indexes: tuple[MaterializedViewIndex, ...] = (), ) -> None: with self._as_connection(bind) as conn: if concurrently: @@ -465,7 +465,7 @@ def create_materialized_view_index( self, bind: Engine | Connection, name: str, - index: "MaterializedViewIndex", + index: MaterializedViewIndex, *, schema: str | None = None, if_not_exists: bool = True, @@ -494,7 +494,7 @@ def create_materialized_view_index( ) from error @contextmanager - def engine_with_replica_role(self, engine: "Engine"): + def engine_with_replica_role(self, engine: Engine): def _set_replica_role( dbapi_conn: sa.engine.interfaces.DBAPIConnection, _, diff --git a/src/orm_loader/backends/sqlite.py b/src/orm_loader/backends/sqlite.py index a421100..ed0aa8c 100644 --- a/src/orm_loader/backends/sqlite.py +++ b/src/orm_loader/backends/sqlite.py @@ -102,7 +102,7 @@ def default_index_strategy(self) -> str: def create_staging_table( self, - table_cls: type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, ) -> None: staging_name = self.staging_name_for_table(table_cls.__tablename__) @@ -127,7 +127,7 @@ def create_staging_table( def drop_staging_table( self, - table_cls: type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, ) -> None: staging_ref = self.identifier_preparer.quote_identifier( @@ -173,7 +173,7 @@ def _staging_rowid() -> sa.ColumnElement[int]: def merge_replace( self, - table_cls: type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, pk_cols: list[str], *, @@ -222,7 +222,7 @@ def _delete(extra: sa.ColumnElement[bool] | None = None) -> sa.Delete: def merge_upsert( self, - table_cls: type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, pk_cols: list[str], *, @@ -263,7 +263,7 @@ def _upsert(select_: Any) -> sa.Insert: def merge_insert( self, - table_cls: type["CSVTableProtocol"], + table_cls: type[CSVTableProtocol], session: so.Session, *, merge_batch_size: int | None = None, @@ -306,7 +306,7 @@ def configure_dbapi_connection( cursor.execute("PRAGMA defer_foreign_keys = ON;") cursor.close() - def install_engine_hooks(self, engine: "Engine") -> None: + def install_engine_hooks(self, engine: Engine) -> None: @event.listens_for(engine, "connect") def _enable_sqlite_foreign_keys( dbapi_connection: sa.engine.interfaces.DBAPIConnection, _connection_record: Any diff --git a/src/orm_loader/helpers/__init__.py b/src/orm_loader/helpers/__init__.py index 18707e2..a87b360 100644 --- a/src/orm_loader/helpers/__init__.py +++ b/src/orm_loader/helpers/__init__.py @@ -1,27 +1,27 @@ -from .errors import IngestError, ValidationError from .bootstrap import bootstrap, create_db, create_tables +from .bulk import bulk_load_context, engine_with_replica_role +from .discovery import get_model_by_tablename +from .errors import IngestError, ValidationError +from .metadata import Base +from .null_handlers import normalise_null from .sqlite import ( attach_sqlite_bulk_load_pragmas, explain_sqlite_fk_error, restore_sqlite_journal_mode, ) -from .bulk import bulk_load_context, engine_with_replica_role -from .metadata import Base -from .discovery import get_model_by_tablename -from .null_handlers import normalise_null __all__ = [ + "Base", "IngestError", "ValidationError", + "attach_sqlite_bulk_load_pragmas", "bootstrap", + "bulk_load_context", "create_db", "create_tables", - "attach_sqlite_bulk_load_pragmas", - "explain_sqlite_fk_error", - "restore_sqlite_journal_mode", - "bulk_load_context", "engine_with_replica_role", - "Base", + "explain_sqlite_fk_error", "get_model_by_tablename", "normalise_null", + "restore_sqlite_journal_mode", ] diff --git a/src/orm_loader/helpers/bulk.py b/src/orm_loader/helpers/bulk.py index 533f7e5..88857ed 100644 --- a/src/orm_loader/helpers/bulk.py +++ b/src/orm_loader/helpers/bulk.py @@ -1,8 +1,9 @@ import logging +from collections.abc import Generator from contextlib import contextmanager + from sqlalchemy import Engine from sqlalchemy.orm import Session -from typing import Generator logger = logging.getLogger(__name__) diff --git a/src/orm_loader/helpers/discovery.py b/src/orm_loader/helpers/discovery.py index 5f8ae5b..a8ba5c9 100644 --- a/src/orm_loader/helpers/discovery.py +++ b/src/orm_loader/helpers/discovery.py @@ -1,4 +1,5 @@ from typing import TypeVar + from .metadata import Base ModelT = TypeVar("ModelT", bound=Base) diff --git a/src/orm_loader/helpers/null_handlers.py b/src/orm_loader/helpers/null_handlers.py index fb3face..8198e11 100644 --- a/src/orm_loader/helpers/null_handlers.py +++ b/src/orm_loader/helpers/null_handlers.py @@ -1,6 +1,7 @@ -import pandas as pd from typing import Any +import pandas as pd + _NULL_STRINGS = {"", "nan", "null", "none", "na", "n/a"} diff --git a/src/orm_loader/loaders/__init__.py b/src/orm_loader/loaders/__init__.py index b1a84eb..8678f43 100644 --- a/src/orm_loader/loaders/__init__.py +++ b/src/orm_loader/loaders/__init__.py @@ -1,14 +1,14 @@ -from .loader_interface import LoaderInterface, PandasLoader, ParquetLoader from .data_classes import LoaderContext, TableCastingStats +from .loader_interface import LoaderInterface, PandasLoader, ParquetLoader from .loading_helpers import infer_delim, infer_encoding, quick_load_pg __all__ = [ - "LoaderInterface", "LoaderContext", + "LoaderInterface", "PandasLoader", + "ParquetLoader", "TableCastingStats", "infer_delim", "infer_encoding", "quick_load_pg", - "ParquetLoader", ] diff --git a/src/orm_loader/loaders/data/__init__.py b/src/orm_loader/loaders/data/__init__.py index 8fbe877..6df6e71 100644 --- a/src/orm_loader/loaders/data/__init__.py +++ b/src/orm_loader/loaders/data/__init__.py @@ -1,6 +1,6 @@ -from .converters import perform_cast, cast_arrow_column +from .converters import cast_arrow_column, perform_cast __all__ = [ - "perform_cast", "cast_arrow_column", + "perform_cast", ] diff --git a/src/orm_loader/loaders/data/converters.py b/src/orm_loader/loaders/data/converters.py index 6b5643e..faf165b 100644 --- a/src/orm_loader/loaders/data/converters.py +++ b/src/orm_loader/loaders/data/converters.py @@ -1,15 +1,17 @@ -from sqlalchemy import ColumnElement -from sqlalchemy.types import Integer, Float, Boolean, Date, DateTime, String, Text, TypeEngine -from typing import Any, Callable -from enum import Enum import math +import re +from collections.abc import Callable from dataclasses import dataclass +from datetime import date, datetime +from enum import Enum +from typing import Any + +import pandas as pd import pyarrow as pa import pyarrow.compute as pc -import pandas as pd -import re -from datetime import datetime, date from dateutil import parser +from sqlalchemy import ColumnElement +from sqlalchemy.types import Boolean, Date, DateTime, Float, Integer, String, Text, TypeEngine from ..data_classes import TableCastingStats diff --git a/src/orm_loader/loaders/data_classes.py b/src/orm_loader/loaders/data_classes.py index 9596ff1..ac931d7 100644 --- a/src/orm_loader/loaders/data_classes.py +++ b/src/orm_loader/loaders/data_classes.py @@ -1,11 +1,12 @@ -from typing import Any, Type, List, Dict, TYPE_CHECKING from dataclasses import dataclass, field -import sqlalchemy as sa -import sqlalchemy.orm as so +from logging import getLogger from pathlib import Path +from typing import TYPE_CHECKING, Any + import pandas as pd import pyarrow as pa -from logging import getLogger +import sqlalchemy as sa +import sqlalchemy.orm as so logger = getLogger(__name__) @@ -72,7 +73,7 @@ class LoaderContext: resolves the same schema. """ - tableclass: Type["CSVTableProtocol"] + tableclass: type["CSVTableProtocol"] session: so.Session path: Path staging_table: sa.Table @@ -185,7 +186,7 @@ class ColumnCastingStats: """ count: int = 0 - examples: List[Any] = field(default_factory=list) + examples: list[Any] = field(default_factory=list) def record(self, value: Any, example_limit: int = 3): self.count += 1 @@ -200,7 +201,7 @@ class TableCastingStats: """ table_name: str - columns: Dict[str, ColumnCastingStats] = field(default_factory=dict) + columns: dict[str, ColumnCastingStats] = field(default_factory=dict) def record( self, diff --git a/src/orm_loader/loaders/loader_interface.py b/src/orm_loader/loaders/loader_interface.py index 2ba5213..a8ab0ff 100644 --- a/src/orm_loader/loaders/loader_interface.py +++ b/src/orm_loader/loaders/loader_interface.py @@ -1,22 +1,26 @@ from __future__ import annotations -from typing import Any, Iterable, Type, TYPE_CHECKING + import csv as _csv -import pandas as pd import logging +from collections.abc import Iterable +from functools import reduce +from typing import TYPE_CHECKING, Any + +import pandas as pd import pyarrow as pa -import pyarrow.dataset as ds import pyarrow.compute as pc -from functools import reduce -from .data_classes import LoaderContext, TableCastingStats, LoaderInterface +import pyarrow.dataset as ds + +from ..helpers import IngestError +from .data import cast_arrow_column, perform_cast +from .data_classes import LoaderContext, LoaderInterface, TableCastingStats from .loading_helpers import ( + arrow_drop_duplicates, + conservative_load_parquet, infer_delim, infer_encoding, - conservative_load_parquet, - arrow_drop_duplicates, resolve_quote_mode, ) -from .data import perform_cast, cast_arrow_column -from ..helpers import IngestError if TYPE_CHECKING: from ..tables.typing import CSVTableProtocol @@ -47,7 +51,7 @@ def _require_columns_present( - tableclass: Type["CSVTableProtocol"], present_columns: Iterable[str] + tableclass: type[CSVTableProtocol], present_columns: Iterable[str] ) -> None: """ Raise before staging if a genuinely required column is entirely diff --git a/src/orm_loader/mappers/__init__.py b/src/orm_loader/mappers/__init__.py index a360384..cdcbe0d 100644 --- a/src/orm_loader/mappers/__init__.py +++ b/src/orm_loader/mappers/__init__.py @@ -13,18 +13,18 @@ DropMaterializedView, MaterializedViewIndex, ) -from .materialised_view_mixin import ( - CreateMaterializedView, - MaterializedViewMixin, - refresh_all_mvs, - resolve_mv_refresh_order, -) from .materialised_view_errors import ( ConcurrentRefreshNotEligibleError, MaterializationError, MaterializationFailure, MaterializationOperation, ) +from .materialised_view_mixin import ( + CreateMaterializedView, + MaterializedViewMixin, + refresh_all_mvs, + resolve_mv_refresh_order, +) __all__ = [ "ConcurrentRefreshNotEligibleError", diff --git a/src/orm_loader/mappers/materialised_view_mixin.py b/src/orm_loader/mappers/materialised_view_mixin.py index 5f6ea72..bce7e0d 100644 --- a/src/orm_loader/mappers/materialised_view_mixin.py +++ b/src/orm_loader/mappers/materialised_view_mixin.py @@ -2,13 +2,13 @@ from typing import Any import sqlalchemy as sa -from sqlalchemy.ext import compiler -from sqlalchemy.schema import DDLElement from oa_configurator import ( Role, open_connection, physical_schema_of, ) +from sqlalchemy.ext import compiler +from sqlalchemy.schema import DDLElement from .materialised_view_contracts import MaterializedViewIndex diff --git a/src/orm_loader/registry/__init__.py b/src/orm_loader/registry/__init__.py index 78c6ca4..f7f64ce 100644 --- a/src/orm_loader/registry/__init__.py +++ b/src/orm_loader/registry/__init__.py @@ -1,18 +1,18 @@ -from .registry import ModelRegistry, ModelDescriptor, TableSpec, FieldSpec -from .validation_report import ValidationIssue, SeverityLevel +from .registry import FieldSpec, ModelDescriptor, ModelRegistry, TableSpec from .validation import Validator -from .validation_runner import ValidationRunner from .validation_presets import always_on_validators +from .validation_report import SeverityLevel, ValidationIssue +from .validation_runner import ValidationRunner __all__ = [ - "ModelRegistry", + "FieldSpec", "ModelDescriptor", + "ModelRegistry", + "SeverityLevel", "TableSpec", - "FieldSpec", - "Validator", "ValidationIssue", - "SeverityLevel", "ValidationRunner", + "Validator", "always_on_validators", ] diff --git a/src/orm_loader/registry/registry.py b/src/orm_loader/registry/registry.py index 812c880..f6e714f 100644 --- a/src/orm_loader/registry/registry.py +++ b/src/orm_loader/registry/registry.py @@ -1,10 +1,11 @@ -from dataclasses import dataclass -from typing import Optional, Type -import sqlalchemy as sa import csv import importlib -import pkgutil import logging +import pkgutil +from dataclasses import dataclass + +import sqlalchemy as sa + from ..tables.typing import ORMTableProtocol logger = logging.getLogger(__name__) @@ -53,7 +54,7 @@ class TableSpec: schema: str is_required: bool description: str - user_guidance: Optional[str] = None + user_guidance: str | None = None @dataclass(frozen=True) @@ -118,14 +119,14 @@ class ModelDescriptor: Mapping of column name to referenced (table, field). """ - model_class: Type[ORMTableProtocol] + model_class: type[ORMTableProtocol] table_name: str columns: dict[str, sa.Column] primary_keys: set[str] foreign_keys: dict[str, tuple[str, str]] # col -> (table, field) @classmethod - def from_model(cls, model: Type[ORMTableProtocol]) -> "ModelDescriptor": + def from_model(cls, model: type[ORMTableProtocol]) -> "ModelDescriptor": """ Construct a ModelDescriptor from an ORM model class. @@ -166,7 +167,7 @@ def from_model(cls, model: Type[ORMTableProtocol]) -> "ModelDescriptor": ) @property - def cls(self) -> Type[ORMTableProtocol]: + def cls(self) -> type[ORMTableProtocol]: """ Alias for the underlying ORM model class. @@ -266,7 +267,7 @@ class ModelRegistry: registry as contextual input. """ - def __init__(self, *, model_version: str, model_name: Optional[str] = None): + def __init__(self, *, model_version: str, model_name: str | None = None): """ Initialise a new model registry. diff --git a/src/orm_loader/registry/validation.py b/src/orm_loader/registry/validation.py index 3e934d2..1e9143b 100644 --- a/src/orm_loader/registry/validation.py +++ b/src/orm_loader/registry/validation.py @@ -1,7 +1,9 @@ from __future__ import annotations + from typing import Protocol -from .validation_report import ValidationIssue, SeverityLevel -from .registry import ModelDescriptor, TableSpec, FieldSpec + +from .registry import FieldSpec, ModelDescriptor, TableSpec +from .validation_report import SeverityLevel, ValidationIssue class Validator(Protocol): diff --git a/src/orm_loader/registry/validation_presets.py b/src/orm_loader/registry/validation_presets.py index a763287..9719e8d 100644 --- a/src/orm_loader/registry/validation_presets.py +++ b/src/orm_loader/registry/validation_presets.py @@ -1,8 +1,8 @@ from .validation import ( ColumnNullabilityValidator, ColumnPresenceValidator, - PrimaryKeyValidator, ForeignKeyShapeValidator, + PrimaryKeyValidator, ) diff --git a/src/orm_loader/registry/validation_report.py b/src/orm_loader/registry/validation_report.py index b0a07b4..4e8ad04 100644 --- a/src/orm_loader/registry/validation_report.py +++ b/src/orm_loader/registry/validation_report.py @@ -1,9 +1,7 @@ +import json from collections import defaultdict from dataclasses import dataclass from enum import Enum -from typing import Optional -import json - """ Validation Reporting @@ -60,10 +58,10 @@ class ValidationIssue: table: str level: SeverityLevel message: str - field: Optional[str] = None - expected: Optional[str] = None - actual: Optional[str] = None - hint: Optional[str] = None + field: str | None = None + expected: str | None = None + actual: str | None = None + hint: str | None = None @dataclass @@ -79,7 +77,7 @@ class ValidationReport: - CI/CD pipeline integration """ - def __init__(self, *, model_version: str, model_name: Optional[str] = None): + def __init__(self, *, model_version: str, model_name: str | None = None): """ Initialise a new validation report. diff --git a/src/orm_loader/registry/validation_runner.py b/src/orm_loader/registry/validation_runner.py index d567441..cf10b16 100644 --- a/src/orm_loader/registry/validation_runner.py +++ b/src/orm_loader/registry/validation_runner.py @@ -1,7 +1,6 @@ -from .validation_report import ValidationReport, SeverityLevel from .registry import ModelRegistry from .validation import Validator - +from .validation_report import SeverityLevel, ValidationReport """ Validation Runner diff --git a/src/orm_loader/tables/__init__.py b/src/orm_loader/tables/__init__.py index 7232aac..0d7e54d 100644 --- a/src/orm_loader/tables/__init__.py +++ b/src/orm_loader/tables/__init__.py @@ -2,13 +2,13 @@ from .loadable_table import CSVLoadableTableInterface from .orm_table import ORMTableBase from .serialisable_table import SerialisableTableInterface -from .typing import ORMTableProtocol, CSVTableProtocol +from .typing import CSVTableProtocol, ORMTableProtocol __all__ = [ - "ORMTableBase", "CSVLoadableTableInterface", - "SerialisableTableInterface", + "CSVTableProtocol", "IdAllocator", + "ORMTableBase", "ORMTableProtocol", - "CSVTableProtocol", + "SerialisableTableInterface", ] diff --git a/src/orm_loader/tables/loadable_table.py b/src/orm_loader/tables/loadable_table.py index 117e7b3..a9ca268 100644 --- a/src/orm_loader/tables/loadable_table.py +++ b/src/orm_loader/tables/loadable_table.py @@ -1,9 +1,10 @@ # pyright: reportPrivateUsage=false import logging +from collections.abc import Generator from contextlib import contextmanager from pathlib import Path from time import perf_counter -from typing import Any, Generator, Type +from typing import Any import sqlalchemy as sa import sqlalchemy.orm as so @@ -69,7 +70,7 @@ class CSVLoadableTableInterface(ORMTableBase): @classmethod def create_staging_table( - cls: Type[CSVTableProtocol], + cls: type[CSVTableProtocol], session: so.Session, *, staging_schema_tag: str | None = None, @@ -103,7 +104,7 @@ def create_staging_table( @classmethod @contextmanager def manage_indices( - cls: Type["CSVTableProtocol"], + cls: type["CSVTableProtocol"], session: so.Session, index_strategy: str = "auto", *, @@ -226,7 +227,7 @@ def manage_indices( @classmethod def get_staging_table( - cls: Type[CSVTableProtocol], + cls: type[CSVTableProtocol], session: so.Session, *, staging_schema_tag: str | None = None, @@ -285,7 +286,7 @@ def get_staging_table( @classmethod def load_staging( - cls: Type[CSVTableProtocol], + cls: type[CSVTableProtocol], loader: LoaderInterface, loader_context: LoaderContext, ) -> int: @@ -332,7 +333,7 @@ def load_staging( @classmethod def orm_staging_load( - cls: Type[CSVTableProtocol], loader: LoaderInterface, loader_context: LoaderContext + cls: type[CSVTableProtocol], loader: LoaderInterface, loader_context: LoaderContext ) -> int: """ Load data into the staging table using an ORM-based loader. @@ -345,7 +346,7 @@ def orm_staging_load( return loader.orm_file_load(ctx=loader_context) @classmethod - def _select_loader(cls: Type[CSVTableProtocol], path: Path) -> LoaderInterface: + def _select_loader(cls: type[CSVTableProtocol], path: Path) -> LoaderInterface: """ Select an appropriate loader based on file type. @@ -367,7 +368,7 @@ def _select_loader(cls: Type[CSVTableProtocol], path: Path) -> LoaderInterface: @classmethod def load_csv( - cls: Type[CSVTableProtocol], + cls: type[CSVTableProtocol], session: so.Session, path: Path, *, @@ -490,7 +491,7 @@ def load_csv( @classmethod def _target_has_rows( - cls: Type[CSVTableProtocol], + cls: type[CSVTableProtocol], session: so.Session, ) -> bool: """ @@ -504,7 +505,7 @@ def _target_has_rows( @classmethod def merge_from_staging( - cls: Type[CSVTableProtocol], + cls: type[CSVTableProtocol], session: so.Session, merge_strategy: str = "replace", *, @@ -600,7 +601,7 @@ def merge_from_staging( @classmethod def drop_staging_table( - cls: Type[CSVTableProtocol], + cls: type[CSVTableProtocol], session: so.Session, *, staging_schema_tag: str | None = None, diff --git a/src/orm_loader/tables/orm_table.py b/src/orm_loader/tables/orm_table.py index 49512f6..89ad46e 100644 --- a/src/orm_loader/tables/orm_table.py +++ b/src/orm_loader/tables/orm_table.py @@ -1,10 +1,12 @@ +import logging +from typing import Any + import sqlalchemy as sa import sqlalchemy.orm as so from sqlalchemy.exc import NoInspectionAvailable, StatementError -from typing import Any -import logging -from .allocators import IdAllocator + from ..helpers import normalise_null +from .allocators import IdAllocator logger = logging.getLogger(__name__) diff --git a/src/orm_loader/tables/serialisable_table.py b/src/orm_loader/tables/serialisable_table.py index e4988eb..8e18e60 100644 --- a/src/orm_loader/tables/serialisable_table.py +++ b/src/orm_loader/tables/serialisable_table.py @@ -1,8 +1,8 @@ -from typing import Any, Unpack -from collections.abc import Iterator -import json -import hashlib import datetime +import hashlib +import json +from collections.abc import Iterator +from typing import Any, Unpack from .orm_table import ORMTableBase from .typing import ToDictKwargs diff --git a/src/orm_loader/tables/typing.py b/src/orm_loader/tables/typing.py index 9c118b8..9635b49 100644 --- a/src/orm_loader/tables/typing.py +++ b/src/orm_loader/tables/typing.py @@ -1,19 +1,18 @@ +from contextlib import AbstractContextManager +from pathlib import Path from typing import ( - Protocol, - ClassVar, - runtime_checkable, TYPE_CHECKING, - Optional, - Type, - Dict, Any, - Unpack, + ClassVar, + Optional, + Protocol, TypedDict, + Unpack, + runtime_checkable, ) -import sqlalchemy.orm as so + import sqlalchemy as sa -from pathlib import Path -from contextlib import AbstractContextManager +import sqlalchemy.orm as so if TYPE_CHECKING: from ..loaders import LoaderContext, LoaderInterface @@ -88,7 +87,7 @@ def create_staging_table( @classmethod def load_staging( - cls: Type["CSVTableProtocol"], loader: "LoaderInterface", loader_context: "LoaderContext" + cls: type["CSVTableProtocol"], loader: "LoaderInterface", loader_context: "LoaderContext" ) -> int: ... @classmethod @@ -165,7 +164,7 @@ def to_dict( include_nulls: bool = False, only: set[str] | None = None, exclude: set[str] | None = None, - ) -> Dict[str, Any]: ... + ) -> dict[str, Any]: ... def to_json(self, **kwargs: Unpack[ToDictKwargs]) -> str: ... diff --git a/tests/backends/test_base_backend.py b/tests/backends/test_base_backend.py index 0172aff..8d3db75 100644 --- a/tests/backends/test_base_backend.py +++ b/tests/backends/test_base_backend.py @@ -3,9 +3,10 @@ import importlib import importlib.abc import sys +from collections.abc import Sequence from importlib.machinery import ModuleSpec from types import ModuleType -from typing import TYPE_CHECKING, Sequence, Type, cast, Any +from typing import TYPE_CHECKING, Any, cast import pytest import sqlalchemy as sa @@ -66,16 +67,16 @@ def staging_name_for_table(tablename: str) -> str: return f"_staging_{tablename}" def create_staging_table( - self, _table_cls: Type[CSVTableProtocol], _session: so.Session + self, _table_cls: type[CSVTableProtocol], _session: so.Session ) -> None: return None - def drop_staging_table(self, _table_cls: Type[CSVTableProtocol], _session: so.Session) -> None: + def drop_staging_table(self, _table_cls: type[CSVTableProtocol], _session: so.Session) -> None: return None def merge_replace( self, - _table_cls: Type[CSVTableProtocol], + _table_cls: type[CSVTableProtocol], _session: so.Session, _target_name: str, _pk_cols: list[str], @@ -86,7 +87,7 @@ def merge_replace( def merge_upsert( self, - _table_cls: Type[CSVTableProtocol], + _table_cls: type[CSVTableProtocol], _session: so.Session, _target_name: str, _pk_cols: list[str], @@ -97,7 +98,7 @@ def merge_upsert( def merge_insert( self, - _table_cls: Type[CSVTableProtocol], + _table_cls: type[CSVTableProtocol], _session: so.Session, _target_name: str, *, @@ -154,7 +155,7 @@ class _ComputedTable: ) -_ComputedTableCls = cast("Type[CSVTableProtocol]", _ComputedTable) +_ComputedTableCls = cast("type[CSVTableProtocol]", _ComputedTable) def test_backend_capabilities_defaults(): @@ -184,7 +185,7 @@ def test_fake_backend_can_implement_contract(): assert backend.resolve_index_strategy("keep") == "keep" assert backend.load_staging_fast(cast("LoaderContext", None)) is None - with backend.merge_context(cast("Type[CSVTableProtocol]", None), cast(so.Session, None)): + with backend.merge_context(cast("type[CSVTableProtocol]", None), cast(so.Session, None)): pass @@ -258,9 +259,8 @@ def capabilities(self) -> BackendCapabilities: def test_bulk_load_context_rolls_back_and_restores(session): backend = FakeBackend() - with pytest.raises(RuntimeError, match="boom"): - with backend.bulk_load_context(session): - raise RuntimeError("boom") + with pytest.raises(RuntimeError, match="boom"), backend.bulk_load_context(session): + raise RuntimeError("boom") assert backend.calls == [ ("disable_fk_check", session), @@ -269,7 +269,7 @@ def test_bulk_load_context_rolls_back_and_restores(session): def test_backends_package_exports(): - import orm_loader.backends as backends + from orm_loader import backends assert backends.DatabaseBackend is DatabaseBackend assert backends.BackendCapabilities is BackendCapabilities diff --git a/tests/backends/test_postgres_backend.py b/tests/backends/test_postgres_backend.py index 457c93b..018cd48 100644 --- a/tests/backends/test_postgres_backend.py +++ b/tests/backends/test_postgres_backend.py @@ -1,16 +1,16 @@ from __future__ import annotations -import sqlalchemy.event as sae -from typing import TYPE_CHECKING, Type, cast +from typing import TYPE_CHECKING, cast import pytest import sqlalchemy as sa +import sqlalchemy.event as sae import sqlalchemy.orm as so +from oa_configurator import SCHEMA_TRANSLATE_MAP_KEY, Role, qualified +from oa_configurator.testing import isolated_test_schema from sqlalchemy.dialects import postgresql from sqlalchemy.engine import Engine -from oa_configurator import SCHEMA_TRANSLATE_MAP_KEY, Role, qualified -from oa_configurator.testing import isolated_test_schema from orm_loader.backends import STAGING_SCHEMA, Dialect, PostgresBackend from orm_loader.backends.resolve import resolve_backend from tests.models import ComputedColumnTable @@ -22,7 +22,7 @@ _PREPARER, _STAGING_TABLE, physical_schema=STAGING_SCHEMA ) -_ComputedTableCls = cast("Type[CSVTableProtocol]", ComputedColumnTable) +_ComputedTableCls = cast("type[CSVTableProtocol]", ComputedColumnTable) if TYPE_CHECKING: @@ -229,9 +229,10 @@ def test_postgres_backend_rejects_mismatched_dialect_bind(): def test_postgres_backend_rejects_dialect_that_drifted_after_resolve_backend(): """A bind resolved to PostgresBackend, then a differently-dialected bind handed to one of its methods, must not run Postgres-only DDL against it.""" - from orm_loader.backends.resolve import resolve_backend from sqlalchemy.dialects import sqlite + from orm_loader.backends.resolve import resolve_backend + postgres_session = _FakeSession() backend = resolve_backend(_sess(postgres_session)) assert isinstance(backend, PostgresBackend) @@ -291,8 +292,8 @@ def test_postgres_backend_create_materialized_view_index_emits_expected_sql(): def test_postgres_backend_create_materialized_view_index_failure_mentions_index_name(): - from orm_loader.mappers.materialised_view_errors import MaterializationError from orm_loader.mappers.materialised_view_contracts import MaterializedViewIndex + from orm_loader.mappers.materialised_view_errors import MaterializationError original = RuntimeError("boom") backend = PostgresBackend() @@ -378,8 +379,8 @@ def test_postgres_backend_refresh_concurrently_without_declared_unique_index_rai def test_postgres_backend_refresh_concurrently_declared_but_database_rejects_it_translates_error(): psycopg = pytest.importorskip("psycopg") - from orm_loader.mappers.materialised_view_errors import ConcurrentRefreshNotEligibleError from orm_loader.mappers.materialised_view_contracts import MaterializedViewIndex + from orm_loader.mappers.materialised_view_errors import ConcurrentRefreshNotEligibleError orig = psycopg.errors.ObjectNotInPrerequisiteState( 'cannot refresh materialized view "reporting.mv_test" concurrently\n' @@ -510,8 +511,8 @@ def test_postgres_backend_materialized_view_lifecycle_is_schema_isolated_with_ad def test_postgres_backend_refresh_concurrently_raises_when_declared_index_was_never_created(pg_db): - from orm_loader.mappers.materialised_view_errors import ConcurrentRefreshNotEligibleError from orm_loader.mappers.materialised_view_contracts import MaterializedViewIndex + from orm_loader.mappers.materialised_view_errors import ConcurrentRefreshNotEligibleError backend = PostgresBackend() index = MaterializedViewIndex(name="mv_missing_index_test_uq", columns=("row_id",), unique=True) diff --git a/tests/backends/test_reserved_schema.py b/tests/backends/test_reserved_schema.py index 92eaed7..9e85312 100644 --- a/tests/backends/test_reserved_schema.py +++ b/tests/backends/test_reserved_schema.py @@ -17,6 +17,7 @@ from __future__ import annotations +import pytest from oa_configurator import ( CDMDatabaseConfig, GenericDatabaseConfig, @@ -24,7 +25,6 @@ SchemaOwnershipError, ) from oa_configurator.testing import reset_schema_registry_rows -import pytest from orm_loader.backends import STAGING_SCHEMA, staging_schema_claim diff --git a/tests/backends/test_shared_backend.py b/tests/backends/test_shared_backend.py index 1c3e37d..838a16c 100644 --- a/tests/backends/test_shared_backend.py +++ b/tests/backends/test_shared_backend.py @@ -1,26 +1,26 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Type, cast +from typing import TYPE_CHECKING, cast import pytest import sqlalchemy as sa - from oa_configurator.testing import DIALECT_PARAMS + from orm_loader.backends import STAGING_SCHEMA, DatabaseBackend from orm_loader.backends.resolve import resolve_backend -from tests.models import ComputedColumnTable, CompositeTable +from tests.models import CompositeTable, ComputedColumnTable if TYPE_CHECKING: import sqlalchemy.orm as so from orm_loader.tables.typing import CSVTableProtocol -_ComputedTableCls = cast("Type[CSVTableProtocol]", ComputedColumnTable) -_CompositeTableCls = cast("Type[CSVTableProtocol]", CompositeTable) +_ComputedTableCls = cast("type[CSVTableProtocol]", ComputedColumnTable) +_CompositeTableCls = cast("type[CSVTableProtocol]", CompositeTable) @pytest.fixture(params=DIALECT_PARAMS) -def merge_backend(request: pytest.FixtureRequest) -> tuple[DatabaseBackend, "so.Session"]: +def merge_backend(request: pytest.FixtureRequest) -> tuple[DatabaseBackend, so.Session]: """Same merge-method contract exercised against both real backends. Only the postgresql param ever requests pg_session, so the sqlite param never needs a database. @@ -38,7 +38,7 @@ def merge_backend(request: pytest.FixtureRequest) -> tuple[DatabaseBackend, "so. return resolve_backend(session), session -def test_merge_replace_single_pk(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_replace_single_pk(merge_backend: tuple[DatabaseBackend, so.Session]) -> None: backend, session = merge_backend backend.create_staging_table(_ComputedTableCls, session) staging = _ComputedTableCls.get_staging_table( @@ -57,7 +57,7 @@ def test_merge_replace_single_pk(merge_backend: tuple[DatabaseBackend, "so.Sessi assert remaining == [2] -def test_merge_replace_composite_pk(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_replace_composite_pk(merge_backend: tuple[DatabaseBackend, so.Session]) -> None: backend, session = merge_backend backend.create_staging_table(_CompositeTableCls, session) staging = _CompositeTableCls.get_staging_table( @@ -77,7 +77,7 @@ def test_merge_replace_composite_pk(merge_backend: tuple[DatabaseBackend, "so.Se def test_merge_insert_excludes_computed_columns( - merge_backend: tuple[DatabaseBackend, "so.Session"], + merge_backend: tuple[DatabaseBackend, so.Session], ) -> None: backend, session = merge_backend backend.create_staging_table(_ComputedTableCls, session) @@ -93,7 +93,7 @@ def test_merge_insert_excludes_computed_columns( def test_merge_upsert_excludes_computed_columns( - merge_backend: tuple[DatabaseBackend, "so.Session"], + merge_backend: tuple[DatabaseBackend, so.Session], ) -> None: backend, session = merge_backend backend.create_staging_table(_ComputedTableCls, session) @@ -143,7 +143,7 @@ def test_postgres_merge_insert_paginates_rownums_with_initial_gap(pg_session) -> assert ids == list(range(10)) -def test_merge_replace_paginated_path(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_replace_paginated_path(merge_backend: tuple[DatabaseBackend, so.Session]) -> None: backend, session = merge_backend backend.create_staging_table(_ComputedTableCls, session) staging = _ComputedTableCls.get_staging_table( @@ -163,7 +163,7 @@ def test_merge_replace_paginated_path(merge_backend: tuple[DatabaseBackend, "so. assert remaining == 0 -def test_merge_insert_paginated_path(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_insert_paginated_path(merge_backend: tuple[DatabaseBackend, so.Session]) -> None: backend, session = merge_backend backend.create_staging_table(_ComputedTableCls, session) staging = _ComputedTableCls.get_staging_table( @@ -177,7 +177,7 @@ def test_merge_insert_paginated_path(merge_backend: tuple[DatabaseBackend, "so.S assert ids == list(range(10)) -def test_merge_upsert_paginated_path(merge_backend: tuple[DatabaseBackend, "so.Session"]) -> None: +def test_merge_upsert_paginated_path(merge_backend: tuple[DatabaseBackend, so.Session]) -> None: backend, session = merge_backend backend.create_staging_table(_ComputedTableCls, session) staging = _ComputedTableCls.get_staging_table( diff --git a/tests/backends/test_sqlite_backend.py b/tests/backends/test_sqlite_backend.py index 8187f1d..864477a 100644 --- a/tests/backends/test_sqlite_backend.py +++ b/tests/backends/test_sqlite_backend.py @@ -2,7 +2,7 @@ import sqlite3 from pathlib import Path -from typing import TYPE_CHECKING, Type, cast +from typing import TYPE_CHECKING, cast import pytest import sqlalchemy as sa @@ -37,7 +37,7 @@ def scalar(self): return _Result(self.scalar_result) -_ComputedTableCls = cast("Type[CSVTableProtocol]", ComputedColumnTable) +_ComputedTableCls = cast("type[CSVTableProtocol]", ComputedColumnTable) def _sess(s: _FakeSession) -> so.Session: diff --git a/tests/conftest.py b/tests/conftest.py index d44fa75..9255295 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,9 +6,9 @@ import sqlalchemy as sa import sqlalchemy.orm as so from dotenv import load_dotenv - from oa_configurator import Role from oa_configurator.testing import ScopedTestSchema, isolated_test_database, scoped_test_schema + from orm_loader.backends import staging_schema_claim from orm_loader.config import OrmLoaderConfig from tests.models import Base diff --git a/tests/loaders/test_casting.py b/tests/loaders/test_casting.py index 67f4ccb..f6e2ddc 100644 --- a/tests/loaders/test_casting.py +++ b/tests/loaders/test_casting.py @@ -1,14 +1,15 @@ +from datetime import date, datetime from enum import Enum, IntEnum +import pytest import sqlalchemy as sa + from orm_loader.loaders.data.converters import ( _COLUMN_CAST_RULES, cast_scalar, perform_cast, register_column_cast_rule, ) -import pytest -from datetime import date, datetime def test_perform_cast_integer(): diff --git a/tests/loaders/test_dedupe.py b/tests/loaders/test_dedupe.py index b61e33e..4b189f0 100644 --- a/tests/loaders/test_dedupe.py +++ b/tests/loaders/test_dedupe.py @@ -1,14 +1,16 @@ -import pyarrow as pa -from typing import cast, Type -from oa_configurator import Role -from orm_loader.loaders.loading_helpers import arrow_drop_duplicates +from typing import cast + import pandas as pd +import pyarrow as pa import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from sqlalchemy.orm import DeclarativeBase + +from orm_loader.loaders.loader_interface import PandasLoader +from orm_loader.loaders.loading_helpers import arrow_drop_duplicates from orm_loader.tables.loadable_table import CSVLoadableTableInterface from orm_loader.tables.typing import CSVTableProtocol -from orm_loader.loaders.loader_interface import PandasLoader class Base(DeclarativeBase): @@ -23,7 +25,7 @@ class DedupTable(Base, CSVLoadableTableInterface): value: so.Mapped[str] = so.mapped_column(sa.String, nullable=False) -_DedupTable = cast(Type[CSVTableProtocol], DedupTable) +_DedupTable = cast(type[CSVTableProtocol], DedupTable) def test_arrow_drop_duplicates_simple(): diff --git a/tests/loaders/test_helpers.py b/tests/loaders/test_helpers.py index eb635d2..c046468 100644 --- a/tests/loaders/test_helpers.py +++ b/tests/loaders/test_helpers.py @@ -1,9 +1,8 @@ +import io from pathlib import Path import pytest -import io - from orm_loader.loaders.data_classes import ColumnCastingStats, TableCastingStats from orm_loader.loaders.loading_helpers import ( NormalisedCSVStream, diff --git a/tests/loaders/test_loader_e2e.py b/tests/loaders/test_loader_e2e.py index 0321b46..265dd0c 100644 --- a/tests/loaders/test_loader_e2e.py +++ b/tests/loaders/test_loader_e2e.py @@ -1,6 +1,6 @@ -from typing import Type, cast - import logging +from typing import cast + import numpy as np import pandas as pd import pytest @@ -30,12 +30,12 @@ # Typed aliases: Pylance cannot verify SQLAlchemy metaclass-generated attrs # satisfy CSVTableProtocol structurally, so we cast once per class here. -_SimpleTable = cast(Type[CSVTableProtocol], SimpleTable) -_RequiredTable = cast(Type[CSVTableProtocol], RequiredTable) -_CompositeTable = cast(Type[CSVTableProtocol], CompositeTable) -_EnumTable = cast(Type[CSVTableProtocol], EnumTable) -_ImpliedEnumTable = cast(Type[CSVTableProtocol], ImpliedEnumTable) -_VocabSchemaTable = cast(Type[CSVTableProtocol], VocabSchemaTable) +_SimpleTable = cast(type[CSVTableProtocol], SimpleTable) +_RequiredTable = cast(type[CSVTableProtocol], RequiredTable) +_CompositeTable = cast(type[CSVTableProtocol], CompositeTable) +_EnumTable = cast(type[CSVTableProtocol], EnumTable) +_ImpliedEnumTable = cast(type[CSVTableProtocol], ImpliedEnumTable) +_VocabSchemaTable = cast(type[CSVTableProtocol], VocabSchemaTable) @pytest.fixture(autouse=True) @@ -549,7 +549,7 @@ class NullableTable(Base, CSVLoadableTableInterface): flag: so.Mapped[str | None] = so.mapped_column(sa.String, nullable=True) Base.metadata.create_all(engine) - _NullableTable = cast(Type[CSVTableProtocol], NullableTable) + _NullableTable = cast(type[CSVTableProtocol], NullableTable) csv = tmp_path / "nullable_table.csv" pd.DataFrame( @@ -586,7 +586,7 @@ class TextTable(Base, CSVLoadableTableInterface): name: so.Mapped[str] = so.mapped_column(sa.String) Base.metadata.create_all(engine) - _TextTable = cast(Type[CSVTableProtocol], TextTable) + _TextTable = cast(type[CSVTableProtocol], TextTable) csv = tmp_path / "text_table.csv" @@ -614,7 +614,7 @@ class TextTable2(Base, CSVLoadableTableInterface): name: so.Mapped[str] = so.mapped_column(sa.String) Base.metadata.create_all(engine) - _TextTable2 = cast(Type[CSVTableProtocol], TextTable2) + _TextTable2 = cast(type[CSVTableProtocol], TextTable2) csv = tmp_path / "tab_table.csv" csv.write_text('id\tname\n1\t"foo\tbar"\n') diff --git a/tests/loaders/test_pandas_loader.py b/tests/loaders/test_pandas_loader.py index e0f7022..48821de 100644 --- a/tests/loaders/test_pandas_loader.py +++ b/tests/loaders/test_pandas_loader.py @@ -1,7 +1,9 @@ from pathlib import Path + +import pandas as pd import sqlalchemy as sa + from orm_loader.loaders.data_classes import LoaderContext -import pandas as pd from orm_loader.loaders.loader_interface import PandasLoader from tests.models import PandasLoaderTable diff --git a/tests/loaders/test_parquet_loader.py b/tests/loaders/test_parquet_loader.py index 96b00fe..f15b734 100644 --- a/tests/loaders/test_parquet_loader.py +++ b/tests/loaders/test_parquet_loader.py @@ -1,15 +1,16 @@ +from typing import cast + +import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -import pandas as pd import sqlalchemy as sa import sqlalchemy.orm as so from oa_configurator import Role from sqlalchemy.orm import DeclarativeBase -from typing import cast, Type +from orm_loader.loaders.loader_interface import ParquetLoader from orm_loader.tables.loadable_table import CSVLoadableTableInterface from orm_loader.tables.typing import CSVTableProtocol -from orm_loader.loaders.loader_interface import ParquetLoader class Base(DeclarativeBase): @@ -24,7 +25,7 @@ class ParquetTable(Base, CSVLoadableTableInterface): value: so.Mapped[int] = so.mapped_column(sa.Integer, nullable=False) -_ParquetTable = cast(Type[CSVTableProtocol], ParquetTable) +_ParquetTable = cast(type[CSVTableProtocol], ParquetTable) def test_parquet_loader(session, engine, tmp_path): @@ -63,7 +64,7 @@ class NullSentinelTable(Base, CSVLoadableTableInterface): role: so.Mapped[str | None] = so.mapped_column(sa.String, nullable=True) -_NullSentinelTable = cast(Type[CSVTableProtocol], NullSentinelTable) +_NullSentinelTable = cast(type[CSVTableProtocol], NullSentinelTable) _SENTINELS = ["NULL", "null", " NULL ", "n/a", "N/A", "NaN", "none", "None", " ", ""] diff --git a/tests/loaders/test_pg_loader.py b/tests/loaders/test_pg_loader.py index ccf7d66..ca30e65 100644 --- a/tests/loaders/test_pg_loader.py +++ b/tests/loaders/test_pg_loader.py @@ -1,15 +1,15 @@ -import sqlalchemy as sa import pandas as pd import pytest +import sqlalchemy as sa + from orm_loader.backends import STAGING_SCHEMA, resolve_backend from orm_loader.loaders.data.converters import _COLUMN_CAST_RULES, register_column_cast_rule from orm_loader.loaders.loading_helpers import ( - infer_encoding, - infer_delim, check_line_ending, + infer_delim, + infer_encoding, quick_load_pg, ) - from tests.models import EnumTable, Role, SimpleTable @@ -89,7 +89,7 @@ def fake_quick_load_pg(*args, **kwargs): called["copy"] = True return 1 - import orm_loader.loaders.loading_helpers as loading_helpers + from orm_loader.loaders import loading_helpers monkeypatch.setattr(loading_helpers, "quick_load_pg", fake_quick_load_pg) @@ -107,7 +107,7 @@ def test_copy_failure_falls_back_to_orm(pg_session, tmp_path, monkeypatch): # See test_postgres_copy_fast_path_is_used above: `quick_load_pg` is # imported lazily from `loaders.loading_helpers`, so that's what needs # patching, not `backends.postgres`. - import orm_loader.loaders.loading_helpers as loading_helpers + from orm_loader.loaders import loading_helpers def broken_copy(*args, **kwargs): raise RuntimeError("boom") diff --git a/tests/loaders/test_schema_translate_map.py b/tests/loaders/test_schema_translate_map.py index 464cd2c..470927b 100644 --- a/tests/loaders/test_schema_translate_map.py +++ b/tests/loaders/test_schema_translate_map.py @@ -24,7 +24,6 @@ from orm_loader.backends import STAGING_SCHEMA from orm_loader.loaders.loader_interface import PandasLoader - from tests.conftest import schema_scoped_session from tests.models import SimpleTable, VocabSchemaTable diff --git a/tests/mappers/test_materialised_view_contracts.py b/tests/mappers/test_materialised_view_contracts.py index ae4a072..8f30077 100644 --- a/tests/mappers/test_materialised_view_contracts.py +++ b/tests/mappers/test_materialised_view_contracts.py @@ -15,8 +15,7 @@ def test_mappers_package_exports_materialized_view_api(): - import orm_loader.mappers as mappers - + from orm_loader import mappers from orm_loader.mappers.materialised_view_errors import MaterializationError assert mappers.MaterializedViewIndex is MaterializedViewIndex diff --git a/tests/mappers/test_materialised_view_mixin.py b/tests/mappers/test_materialised_view_mixin.py index 407e402..53e9bdd 100644 --- a/tests/mappers/test_materialised_view_mixin.py +++ b/tests/mappers/test_materialised_view_mixin.py @@ -6,9 +6,9 @@ import pytest import sqlalchemy as sa import sqlalchemy.orm as so - from oa_configurator import Role from oa_configurator.testing import scoped_test_schema + from orm_loader.mappers.materialised_view_contracts import MaterializedViewIndex from orm_loader.mappers.materialised_view_mixin import ( MaterializedViewMixin, diff --git a/tests/models.py b/tests/models.py index b0fc084..25f39d2 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,9 +1,10 @@ from enum import Enum import sqlalchemy as sa +import sqlalchemy.orm as so from oa_configurator import Role as SchemaRole from sqlalchemy.orm import declarative_base -import sqlalchemy.orm as so + from orm_loader.tables import CSVLoadableTableInterface Base = declarative_base() diff --git a/tests/registry/test_model_descriptor.py b/tests/registry/test_model_descriptor.py index e33eec6..b22b270 100644 --- a/tests/registry/test_model_descriptor.py +++ b/tests/registry/test_model_descriptor.py @@ -1,7 +1,8 @@ +import pytest import sqlalchemy as sa import sqlalchemy.orm as so -import pytest from sqlalchemy.exc import NoInspectionAvailable + from orm_loader.registry.registry import ModelDescriptor from orm_loader.tables.orm_table import ORMTableBase diff --git a/tests/registry/test_registry.py b/tests/registry/test_registry.py index 071c3f0..74dceb9 100644 --- a/tests/registry/test_registry.py +++ b/tests/registry/test_registry.py @@ -1,15 +1,16 @@ import io -from orm_loader.registry.registry import ModelRegistry + import sqlalchemy as sa import sqlalchemy.orm as so -from orm_loader.tables.orm_table import ORMTableBase from orm_loader.registry.registry import ( - load_table_specs, - load_field_specs, - TableSpec, FieldSpec, + ModelRegistry, + TableSpec, + load_field_specs, + load_table_specs, ) +from orm_loader.tables.orm_table import ORMTableBase Base = so.declarative_base() diff --git a/tests/registry/test_validation_e2e.py b/tests/registry/test_validation_e2e.py index 343bc94..86a9f46 100644 --- a/tests/registry/test_validation_e2e.py +++ b/tests/registry/test_validation_e2e.py @@ -1,10 +1,11 @@ import io + import sqlalchemy as sa import sqlalchemy.orm as so from orm_loader.registry.registry import ModelRegistry -from orm_loader.registry.validation_runner import ValidationRunner from orm_loader.registry.validation_presets import always_on_validators +from orm_loader.registry.validation_runner import ValidationRunner from orm_loader.tables.orm_table import ORMTableBase Base = so.declarative_base() diff --git a/tests/tables/test_serialisable_table.py b/tests/tables/test_serialisable_table.py index 9e123b7..c9a1733 100644 --- a/tests/tables/test_serialisable_table.py +++ b/tests/tables/test_serialisable_table.py @@ -1,8 +1,9 @@ -from orm_loader.tables.serialisable_table import SerialisableTableInterface import sqlalchemy as sa import sqlalchemy.orm as so from sqlalchemy.orm import declarative_base +from orm_loader.tables.serialisable_table import SerialisableTableInterface + Base = declarative_base() diff --git a/tests/tables/test_typing_protocols.py b/tests/tables/test_typing_protocols.py index 1f36e7a..02fa17a 100644 --- a/tests/tables/test_typing_protocols.py +++ b/tests/tables/test_typing_protocols.py @@ -1,11 +1,12 @@ import sqlalchemy as sa import sqlalchemy.orm as so + +from orm_loader.tables.loadable_table import CSVLoadableTableInterface from orm_loader.tables.orm_table import ORMTableBase from orm_loader.tables.serialisable_table import SerialisableTableInterface -from orm_loader.tables.loadable_table import CSVLoadableTableInterface from orm_loader.tables.typing import ( - ORMTableProtocol, CSVTableProtocol, + ORMTableProtocol, SerializedTableProtocol, ) From eaadbcdf159308f6fa0fd51ce4af3b6a3a5d6f30 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 22 Oct 2026 03:07:28 +0000 Subject: [PATCH 5/8] Fix remaining ruff findings --- src/orm_loader/backends/postgres.py | 23 +++++++++--------- src/orm_loader/backends/sqlite.py | 4 ++-- src/orm_loader/helpers/discovery.py | 17 ++++++++----- src/orm_loader/helpers/null_handlers.py | 9 +++---- src/orm_loader/loaders/data/converters.py | 14 ++++++----- src/orm_loader/loaders/loader_interface.py | 19 +++++++-------- src/orm_loader/loaders/loading_helpers.py | 3 +-- .../mappers/materialised_view_mixin.py | 5 ++-- src/orm_loader/registry/validation.py | 2 +- src/orm_loader/tables/loadable_table.py | 6 ++--- src/orm_loader/tables/serialisable_table.py | 2 +- tests/backends/test_base_backend.py | 8 ++++--- tests/loaders/test_dedupe.py | 4 ++-- tests/loaders/test_loader_e2e.py | 10 ++++---- tests/loaders/test_parquet_loader.py | 8 +++---- tests/loaders/test_pg_loader.py | 5 ++-- tests/mappers/test_materialised_view_mixin.py | 24 ++++++++++--------- tests/models.py | 13 +++++----- 18 files changed, 92 insertions(+), 84 deletions(-) diff --git a/src/orm_loader/backends/postgres.py b/src/orm_loader/backends/postgres.py index 54a30aa..5987848 100644 --- a/src/orm_loader/backends/postgres.py +++ b/src/orm_loader/backends/postgres.py @@ -138,7 +138,7 @@ def load_staging_fast( @staticmethod def _normalize_fk_check_state(previous_state: str | int) -> str: if isinstance(previous_state, int): - raise ValueError( + raise ValueError( # noqa: TRY004 - int is an accepted type, never a valid role f"Invalid PostgreSQL session_replication_role {previous_state!r}: " "Postgres uses string roles ('origin', 'local', 'replica'), not integers. " "The value passed here should always come from this backend's own " @@ -159,7 +159,7 @@ def disable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> ).scalar() session.execute(sa.text("SET session_replication_role = 'replica'"), **execute_options) if not isinstance(previous_state, str): - raise RuntimeError("Expected PostgreSQL FK state to be a string") + raise RuntimeError("Expected PostgreSQL FK state to be a string") # noqa: TRY004 - bad database result return previous_state def enable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> str | int: @@ -169,7 +169,7 @@ def enable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> ).scalar() session.execute(sa.text("SET session_replication_role = 'origin'"), **execute_options) if not isinstance(previous_state, str): - raise RuntimeError("Expected PostgreSQL FK state to be a string") + raise RuntimeError("Expected PostgreSQL FK state to be a string") # noqa: TRY004 - bad database result return previous_state def restore_fk_check( @@ -393,16 +393,15 @@ def refresh_materialized_view( declared_indexes: tuple[MaterializedViewIndex, ...] = (), ) -> None: with self._as_connection(bind) as conn: - if concurrently: - if not any(index.unique for index in declared_indexes): - raise ConcurrentRefreshNotEligibleError( - MaterializationFailure( - operation=MaterializationOperation.REFRESH, - schema=schema, - name=name, - reason="no simple unique index is declared", - ) + if concurrently and not any(index.unique for index in declared_indexes): + raise ConcurrentRefreshNotEligibleError( + MaterializationFailure( + operation=MaterializationOperation.REFRESH, + schema=schema, + name=name, + reason="no simple unique index is declared", ) + ) safe_name = qualified(conn, name, physical_schema=schema) concurrency = "CONCURRENTLY " if concurrently else "" diff --git a/src/orm_loader/backends/sqlite.py b/src/orm_loader/backends/sqlite.py index ed0aa8c..5c5c295 100644 --- a/src/orm_loader/backends/sqlite.py +++ b/src/orm_loader/backends/sqlite.py @@ -143,7 +143,7 @@ def disable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> previous_state = session.execute(text("PRAGMA foreign_keys"), **execute_options).scalar() session.execute(text("PRAGMA foreign_keys = OFF"), **execute_options) if not isinstance(previous_state, int): - raise RuntimeError("Expected SQLite FK state to be an int") + raise RuntimeError("Expected SQLite FK state to be an int") # noqa: TRY004 - bad database result return previous_state def enable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> str | int: @@ -151,7 +151,7 @@ def enable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> previous_state = session.execute(text("PRAGMA foreign_keys"), **execute_options).scalar() session.execute(text("PRAGMA foreign_keys = ON"), **execute_options) if not isinstance(previous_state, int): - raise RuntimeError("Expected SQLite FK state to be an int") + raise RuntimeError("Expected SQLite FK state to be an int") # noqa: TRY004 - bad database result return previous_state def restore_fk_check( diff --git a/src/orm_loader/helpers/discovery.py b/src/orm_loader/helpers/discovery.py index a8ba5c9..377b92d 100644 --- a/src/orm_loader/helpers/discovery.py +++ b/src/orm_loader/helpers/discovery.py @@ -1,14 +1,19 @@ -from typing import TypeVar +from typing import overload from .metadata import Base -ModelT = TypeVar("ModelT", bound=Base) +@overload +def get_model_by_tablename(tablename: str) -> type[Base] | None: ... -def get_model_by_tablename( - tablename: str, - base: type[ModelT] = Base, # ty: ignore[invalid-parameter-default] -) -> type[ModelT] | None: + +@overload +def get_model_by_tablename[ModelT: Base]( + tablename: str, base: type[ModelT] +) -> type[ModelT] | None: ... + + +def get_model_by_tablename(tablename: str, base: type[Base] = Base) -> type[Base] | None: tablename = tablename.lower().strip() for mapper in base.registry.mappers: cls = mapper.class_ diff --git a/src/orm_loader/helpers/null_handlers.py b/src/orm_loader/helpers/null_handlers.py index 8198e11..bc2076f 100644 --- a/src/orm_loader/helpers/null_handlers.py +++ b/src/orm_loader/helpers/null_handlers.py @@ -9,12 +9,9 @@ def normalise_null(value: Any) -> Any | None: if value is None: return None - # pandas / numpy NaN - try: - if pd.isna(value): - return None - except Exception: - pass + # pandas / numpy NaN. Only scalars are nullable; array-likes are returned unchanged. + if pd.api.types.is_scalar(value) and pd.isna(value): + return None # string garbage if isinstance(value, str): diff --git a/src/orm_loader/loaders/data/converters.py b/src/orm_loader/loaders/data/converters.py index faf165b..f571995 100644 --- a/src/orm_loader/loaders/data/converters.py +++ b/src/orm_loader/loaders/data/converters.py @@ -17,6 +17,9 @@ _NUMERIC_RE = re.compile(r"^[+-]?\d+(\.\d+)?$") +# What an unparseable value raises. Any other exception is a bug and propagates. +_CAST_ERRORS = (ValueError, TypeError, ArithmeticError) + _AVAILABLE_DATE_FORMATS = ( "%Y%m%d", # 20170824 (athena standard) "%d-%b-%Y", # 24-AUG-2017 (oncology-branch vocab) @@ -162,9 +165,8 @@ def _cast_string(value: Any, sa_type: TypeEngine[Any]) -> str | None: s = _to_numeric_string(s) or "" - if isinstance(sa_type, (String, Text)) and sa_type.length: - if len(s) > sa_type.length: - return s[: sa_type.length] + if isinstance(sa_type, (String, Text)) and sa_type.length and len(s) > sa_type.length: + return s[: sa_type.length] return s @@ -392,7 +394,7 @@ def cast_scalar( if column_rule is not None: try: return column_rule(value) - except Exception: + except _CAST_ERRORS: if on_error: on_error(value) return None @@ -401,7 +403,7 @@ def cast_scalar( if isinstance(sa_type, rule.sa_type): try: return rule.scalar(value, sa_type) - except Exception: + except _CAST_ERRORS: if on_error: on_error(value) return None @@ -434,7 +436,7 @@ def cast_arrow_column( raw = v.as_py() try: values.append(column_rule(raw) if raw is not None else None) - except Exception: + except _CAST_ERRORS: if stats: stats.record(column=sa_col.name, value=raw) values.append(None) diff --git a/src/orm_loader/loaders/loader_interface.py b/src/orm_loader/loaders/loader_interface.py index a8ab0ff..9980738 100644 --- a/src/orm_loader/loaders/loader_interface.py +++ b/src/orm_loader/loaders/loader_interface.py @@ -3,7 +3,7 @@ import csv as _csv import logging from collections.abc import Iterable -from functools import reduce +from functools import partial, reduce from typing import TYPE_CHECKING, Any import pandas as pd @@ -121,15 +121,14 @@ def cast_to_model(cls, data: pd.DataFrame | pa.Table, ctx: LoaderContext) -> Any def _on_cast_error(value, *, _col=col_name): stats.record(column=_col, value=value) - df[col_name] = df[col_name].map( - lambda v: perform_cast( - v, - sa_col.type, - on_error=_on_cast_error, - table_name=table_name, - column_name=col_name, - ) + cast_value = partial( + perform_cast, + sa_type=sa_col.type, + on_error=_on_cast_error, + table_name=table_name, + column_name=col_name, ) + df[col_name] = df[col_name].map(cast_value) _require_columns_present(ctx.tableclass, df.columns) required_cols = list(ctx.tableclass.required_columns()) @@ -310,7 +309,7 @@ def orm_file_load(cls, ctx: LoaderContext) -> int: if ctx.dedupe: data = cls.dedupe(data, ctx) - if isinstance(data, pa.RecordBatch) or isinstance(data, pa.Table): + if isinstance(data, (pa.RecordBatch, pa.Table)): df = data.to_pandas() else: df = data diff --git a/src/orm_loader/loaders/loading_helpers.py b/src/orm_loader/loaders/loading_helpers.py index 20169b0..fc86c31 100644 --- a/src/orm_loader/loaders/loading_helpers.py +++ b/src/orm_loader/loaders/loading_helpers.py @@ -257,8 +257,7 @@ def _invalid_row_handler(row): parse_options=parse_opts, convert_options=convert_opts, ) as reader: - for batch in reader: - yield batch + yield from reader def check_line_ending(raw_header: str) -> str: diff --git a/src/orm_loader/mappers/materialised_view_mixin.py b/src/orm_loader/mappers/materialised_view_mixin.py index bce7e0d..643ce15 100644 --- a/src/orm_loader/mappers/materialised_view_mixin.py +++ b/src/orm_loader/mappers/materialised_view_mixin.py @@ -1,5 +1,6 @@ from collections import defaultdict, deque -from typing import Any +from collections.abc import Set as AbstractSet +from typing import Any, ClassVar import sqlalchemy as sa from oa_configurator import ( @@ -186,7 +187,7 @@ class DailyObservationCountsMV(Base, MaterializedViewMixin): __mv_name__: str __mv_select__: sa.sql.Select[Any] - __mv_dependencies__: set[str] = set() + __mv_dependencies__: ClassVar[AbstractSet[str]] = frozenset() __mv_schema_tag__: str = Role.PRIMARY.value __mv_indexes__: tuple[MaterializedViewIndex, ...] = () diff --git a/src/orm_loader/registry/validation.py b/src/orm_loader/registry/validation.py index 1e9143b..87bb1bc 100644 --- a/src/orm_loader/registry/validation.py +++ b/src/orm_loader/registry/validation.py @@ -76,7 +76,7 @@ def validate( issues: list[ValidationIssue] = [] model_cols = set(model.columns.keys()) - for field_name, field_spec in fields.items(): + for field_name in fields: if field_name not in model_cols: issues.append( ValidationIssue( diff --git a/src/orm_loader/tables/loadable_table.py b/src/orm_loader/tables/loadable_table.py index a9ca268..fce4467 100644 --- a/src/orm_loader/tables/loadable_table.py +++ b/src/orm_loader/tables/loadable_table.py @@ -213,9 +213,9 @@ def manage_indices( f"Table `{table_name}`: Commit after restoring index `{idx.name}` " f"completed in {_format_elapsed(perf_counter() - commit_started)}." ) - except Exception as e: + except Exception: session.rollback() - logger.error(f"Table `{table_name}`: Failed to restore {idx.name}: {e}") + logger.exception(f"Table `{table_name}`: Failed to restore {idx.name}") else: logger.debug( f"Table `{table_name}`: Index {idx.name} already exists on disk. Skipping." @@ -325,7 +325,7 @@ def load_staging( return total except Exception as e: loader_context.session.rollback() - logger.warning(f"Fast-path load failed for {cls.__tablename__}: {e}") + logger.warning(f"Fast-path load failed for {cls.__tablename__}: {e}", exc_info=True) logger.info("Falling back to ORM-based load functionality") total = cls.orm_staging_load(loader=loader, loader_context=loader_context) diff --git a/src/orm_loader/tables/serialisable_table.py b/src/orm_loader/tables/serialisable_table.py index 8e18e60..3289155 100644 --- a/src/orm_loader/tables/serialisable_table.py +++ b/src/orm_loader/tables/serialisable_table.py @@ -83,7 +83,7 @@ def to_dict( A dictionary representation of the ORM row. """ data: dict[str, Any] = {} - for key, _ in self.model_columns().items(): + for key in self.model_columns(): if only and key not in only: continue if exclude and key in exclude: diff --git a/tests/backends/test_base_backend.py b/tests/backends/test_base_backend.py index 8d3db75..87be4da 100644 --- a/tests/backends/test_base_backend.py +++ b/tests/backends/test_base_backend.py @@ -251,9 +251,11 @@ def capabilities(self) -> BackendCapabilities: backend = NoFKBackend() - with pytest.raises(NotImplementedError, match="does not support foreign key toggling"): - with backend.bulk_load_context(session): - pass + with ( + pytest.raises(NotImplementedError, match="does not support foreign key toggling"), + backend.bulk_load_context(session), + ): + pass def test_bulk_load_context_rolls_back_and_restores(session): diff --git a/tests/loaders/test_dedupe.py b/tests/loaders/test_dedupe.py index 4b189f0..8499318 100644 --- a/tests/loaders/test_dedupe.py +++ b/tests/loaders/test_dedupe.py @@ -1,4 +1,4 @@ -from typing import cast +from typing import ClassVar, cast import pandas as pd import pyarrow as pa @@ -19,7 +19,7 @@ class Base(DeclarativeBase): class DedupTable(Base, CSVLoadableTableInterface): __tablename__ = "dedup_table" - __table_args__ = {"schema": Role.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": Role.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) value: so.Mapped[str] = so.mapped_column(sa.String, nullable=False) diff --git a/tests/loaders/test_loader_e2e.py b/tests/loaders/test_loader_e2e.py index 265dd0c..c9926ab 100644 --- a/tests/loaders/test_loader_e2e.py +++ b/tests/loaders/test_loader_e2e.py @@ -1,5 +1,5 @@ import logging -from typing import cast +from typing import ClassVar, cast import numpy as np import pandas as pd @@ -543,7 +543,7 @@ def test_clean_nulls_passthrough(): def test_nullable_column_with_nan_does_not_crash(session, engine, tmp_path): class NullableTable(Base, CSVLoadableTableInterface): __tablename__ = "nullable_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) flag: so.Mapped[str | None] = so.mapped_column(sa.String, nullable=True) @@ -580,7 +580,7 @@ class NullableTable(Base, CSVLoadableTableInterface): def test_embedded_newline_in_field_is_preserved(session, engine, tmp_path): class TextTable(Base, CSVLoadableTableInterface): __tablename__ = "text_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) name: so.Mapped[str] = so.mapped_column(sa.String) @@ -608,7 +608,7 @@ class TextTable(Base, CSVLoadableTableInterface): def test_embedded_tab_in_field(session, engine, tmp_path): class TextTable2(Base, CSVLoadableTableInterface): __tablename__ = "tab_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) name: so.Mapped[str] = so.mapped_column(sa.String) @@ -641,7 +641,7 @@ def _make_ddl_tracker(engine): def _capture(*args): statement: str = args[2] upper = statement.strip().upper() - if upper.startswith("DROP INDEX") or upper.startswith("CREATE INDEX"): + if upper.startswith(("DROP INDEX", "CREATE INDEX")): ddl_log.append(statement.strip()) sae.listen(engine, "before_cursor_execute", _capture) diff --git a/tests/loaders/test_parquet_loader.py b/tests/loaders/test_parquet_loader.py index f15b734..8fffa8e 100644 --- a/tests/loaders/test_parquet_loader.py +++ b/tests/loaders/test_parquet_loader.py @@ -1,4 +1,4 @@ -from typing import cast +from typing import ClassVar, cast import pandas as pd import pyarrow as pa @@ -19,7 +19,7 @@ class Base(DeclarativeBase): class ParquetTable(Base, CSVLoadableTableInterface): __tablename__ = "parquet_table" - __table_args__ = {"schema": Role.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": Role.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) value: so.Mapped[int] = so.mapped_column(sa.Integer, nullable=False) @@ -57,7 +57,7 @@ class NullSentinelTable(Base, CSVLoadableTableInterface): treats text null sentinels the same way the scalar path does.""" __tablename__ = "null_sentinel_table" - __table_args__ = {"schema": Role.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": Role.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) name: so.Mapped[str | None] = so.mapped_column(sa.String, nullable=True) @@ -134,7 +134,7 @@ def test_arrow_path_nulls_float_nan(session, engine, tmp_path): class FloatTable(Base, CSVLoadableTableInterface): __tablename__ = "float_sentinel_table" - __table_args__ = {"schema": Role.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": Role.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) score: so.Mapped[float | None] = so.mapped_column(sa.Float, nullable=True) diff --git a/tests/loaders/test_pg_loader.py b/tests/loaders/test_pg_loader.py index ca30e65..76bdd7a 100644 --- a/tests/loaders/test_pg_loader.py +++ b/tests/loaders/test_pg_loader.py @@ -1,4 +1,5 @@ import pandas as pd +import psycopg import pytest import sqlalchemy as sa @@ -334,7 +335,7 @@ def test_quick_load_pg_rollback_on_error(pg_session, tmp_path): csv = tmp_path / "test_table.csv" csv.write_text("id,name\n1,alpha\n2,\n") # violates NOT NULL - with pytest.raises(Exception): + with pytest.raises(psycopg.errors.IntegrityError): quick_load_pg(path=csv, session=pg_session, tablename="test_table") rows = pg_session.execute(sa.text("SELECT COUNT(*) FROM test_table")).scalar_one() @@ -395,7 +396,7 @@ def test_copy_fails_with_raw_carriage_returns_but_succeeds_after_normalisation( try: quick_load_pg(path=csv, session=pg_session, tablename="test_table") pg_session.commit() - except Exception: + except psycopg.Error: failed = True # Clean up table for second attempt diff --git a/tests/mappers/test_materialised_view_mixin.py b/tests/mappers/test_materialised_view_mixin.py index 53e9bdd..14fa3c1 100644 --- a/tests/mappers/test_materialised_view_mixin.py +++ b/tests/mappers/test_materialised_view_mixin.py @@ -1,7 +1,7 @@ from __future__ import annotations import importlib -from typing import Any +from typing import Any, ClassVar import pytest import sqlalchemy as sa @@ -96,10 +96,12 @@ def test_refresh_mv_concurrently_against_postgres(pg_db) -> None: The eligibility check and error translation are already proven at the backend level directly in test_postgres_backend.py. This proves the mixin wires into that correctly end to end.""" - with scoped_test_schema(pg_db.resolved, prefix="mv_refresh") as scoped: - with scoped.engine.begin() as conn: - _IndexedPgMv.create_mv(conn) - _IndexedPgMv.refresh_mv(conn, concurrently=True) + with ( + scoped_test_schema(pg_db.resolved, prefix="mv_refresh") as scoped, + scoped.engine.begin() as conn, + ): + _IndexedPgMv.create_mv(conn) + _IndexedPgMv.refresh_mv(conn, concurrently=True) class _FakeBackend: @@ -213,7 +215,7 @@ class _MappedVocabMv(_MappedMvBase, MaterializedViewMixin): __mv_name__ = "mv_mapped_vocab" __mv_select__ = _SELECT __tablename__ = "mv_mapped_vocab" - __table_args__ = {"schema": Role.VOCAB.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": Role.VOCAB.value} row_id = sa.Column(sa.Integer, primary_key=True) @@ -372,13 +374,13 @@ class _MvA(MaterializedViewMixin): class _MvB(MaterializedViewMixin): __mv_name__ = "mv_b" __mv_select__ = _SELECT - __mv_dependencies__ = {"mv_a"} + __mv_dependencies__ = frozenset({"mv_a"}) class _MvC(MaterializedViewMixin): __mv_name__ = "mv_c" __mv_select__ = _SELECT - __mv_dependencies__ = {"mv_a", "mv_b"} + __mv_dependencies__ = frozenset({"mv_a", "mv_b"}) def test_resolve_mv_refresh_order_linear_dependency(): @@ -395,7 +397,7 @@ def test_resolve_mv_refresh_order_ignores_dependencies_outside_the_given_set(): class _StandaloneMv(MaterializedViewMixin): __mv_name__ = "mv_standalone" __mv_select__ = _SELECT - __mv_dependencies__ = {"some_other_table_not_in_this_registry"} + __mv_dependencies__ = frozenset({"some_other_table_not_in_this_registry"}) assert resolve_mv_refresh_order([_StandaloneMv]) == [_StandaloneMv] @@ -404,12 +406,12 @@ def test_resolve_mv_refresh_order_raises_on_cycle(): class _CycleA(MaterializedViewMixin): __mv_name__ = "mv_cycle_a" __mv_select__ = _SELECT - __mv_dependencies__ = {"mv_cycle_b"} + __mv_dependencies__ = frozenset({"mv_cycle_b"}) class _CycleB(MaterializedViewMixin): __mv_name__ = "mv_cycle_b" __mv_select__ = _SELECT - __mv_dependencies__ = {"mv_cycle_a"} + __mv_dependencies__ = frozenset({"mv_cycle_a"}) with pytest.raises(RuntimeError, match="Cycle detected"): resolve_mv_refresh_order([_CycleA, _CycleB]) diff --git a/tests/models.py b/tests/models.py index 25f39d2..9495260 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,4 +1,5 @@ from enum import Enum +from typing import ClassVar import sqlalchemy as sa import sqlalchemy.orm as so @@ -22,7 +23,7 @@ class Flag(str, Enum): class PandasLoaderTable(CSVLoadableTableInterface, Base): __tablename__ = "test_pandas_loader" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id = sa.Column(sa.Integer, primary_key=True) value = sa.Column(sa.String, nullable=False) @@ -40,7 +41,7 @@ class SimpleTable(Base, CSVLoadableTableInterface): class RequiredTable(Base, CSVLoadableTableInterface): __tablename__ = "required_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) name: so.Mapped[str] = so.mapped_column(sa.String, nullable=False) @@ -48,7 +49,7 @@ class RequiredTable(Base, CSVLoadableTableInterface): class CompositeTable(Base, CSVLoadableTableInterface): __tablename__ = "composite_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} a: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) b: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) @@ -64,7 +65,7 @@ class EnumTable(Base, CSVLoadableTableInterface): """ __tablename__ = "enum_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) role: so.Mapped[Role | None] = so.mapped_column(sa.Enum(Role), nullable=True) @@ -77,7 +78,7 @@ class ComputedColumnTable(Base, CSVLoadableTableInterface): CSVLoadableTableInterface.""" __tablename__ = "computed_column_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) name: so.Mapped[str] = so.mapped_column(sa.String) @@ -91,7 +92,7 @@ class ImpliedEnumTable(Base, CSVLoadableTableInterface): """ __tablename__ = "implied_enum_table" - __table_args__ = {"schema": SchemaRole.PRIMARY.value} + __table_args__: ClassVar[dict[str, str]] = {"schema": SchemaRole.PRIMARY.value} id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) flag: so.Mapped[str | None] = so.mapped_column(sa.String(1), nullable=True) From 7bca3693df9de352b29058be95310620d7931375 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 22 Oct 2026 03:24:16 +0000 Subject: [PATCH 6/8] Fix ty errors in tests --- tests/backends/test_base_backend.py | 39 ++++++------- tests/backends/test_postgres_backend.py | 55 +++++++++++-------- tests/backends/test_reserved_schema.py | 5 +- tests/conftest.py | 4 +- tests/loaders/test_dedupe.py | 4 +- tests/loaders/test_loader_e2e.py | 3 +- tests/loaders/test_pandas_loader.py | 16 ++++-- tests/loaders/test_parquet_loader.py | 8 +-- tests/loaders/test_split_connection.py | 9 ++- tests/mappers/test_materialised_view_mixin.py | 2 +- tests/registry/test_model_descriptor.py | 6 +- 11 files changed, 86 insertions(+), 65 deletions(-) diff --git a/tests/backends/test_base_backend.py b/tests/backends/test_base_backend.py index 87be4da..ffb9d5d 100644 --- a/tests/backends/test_base_backend.py +++ b/tests/backends/test_base_backend.py @@ -66,43 +66,38 @@ def capabilities(self) -> BackendCapabilities: def staging_name_for_table(tablename: str) -> str: return f"_staging_{tablename}" - def create_staging_table( - self, _table_cls: type[CSVTableProtocol], _session: so.Session - ) -> None: + def create_staging_table(self, table_cls: type[CSVTableProtocol], session: so.Session) -> None: return None - def drop_staging_table(self, _table_cls: type[CSVTableProtocol], _session: so.Session) -> None: + def drop_staging_table(self, table_cls: type[CSVTableProtocol], session: so.Session) -> None: return None def merge_replace( self, - _table_cls: type[CSVTableProtocol], - _session: so.Session, - _target_name: str, - _pk_cols: list[str], + table_cls: type[CSVTableProtocol], + session: so.Session, + pk_cols: list[str], *, - _merge_batch_size: int | None = None, + merge_batch_size: int | None = None, ) -> None: return None def merge_upsert( self, - _table_cls: type[CSVTableProtocol], - _session: so.Session, - _target_name: str, - _pk_cols: list[str], + table_cls: type[CSVTableProtocol], + session: so.Session, + pk_cols: list[str], *, - _merge_batch_size: int | None = None, + merge_batch_size: int | None = None, ) -> None: return None def merge_insert( self, - _table_cls: type[CSVTableProtocol], - _session: so.Session, - _target_name: str, + table_cls: type[CSVTableProtocol], + session: so.Session, *, - _merge_batch_size: int | None = None, + merge_batch_size: int | None = None, ) -> None: return None @@ -110,15 +105,17 @@ def merge_insert( def _normalize_fk_check_state(previous_state: str | int) -> str | int: return previous_state - def disable_fk_check(self, session: so.Session) -> str | int: + def disable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> str | int: self.calls.append(("disable_fk_check", session)) return "enabled" - def enable_fk_check(self, session: so.Session) -> str | int: + def enable_fk_check(self, session: so.Session, *, mapper: Any | None = None) -> str | int: self.calls.append(("enable_fk_check", session)) return "disabled" - def restore_fk_check(self, session: so.Session, previous_state: str | int) -> None: + def restore_fk_check( + self, session: so.Session, previous_state: str | int, *, mapper: Any | None = None + ) -> None: self.calls.append(("restore_fk_check", previous_state)) def create_materialized_view( diff --git a/tests/backends/test_postgres_backend.py b/tests/backends/test_postgres_backend.py index 018cd48..a452015 100644 --- a/tests/backends/test_postgres_backend.py +++ b/tests/backends/test_postgres_backend.py @@ -8,6 +8,7 @@ import sqlalchemy.orm as so from oa_configurator import SCHEMA_TRANSLATE_MAP_KEY, Role, qualified from oa_configurator.testing import isolated_test_schema +from sqlalchemy import exc as sa_exc from sqlalchemy.dialects import postgresql from sqlalchemy.engine import Engine @@ -30,7 +31,7 @@ class _FakeSession: - dialect = postgresql.dialect() + dialect: sa.engine.Dialect = postgresql.dialect() def __init__( self, @@ -81,6 +82,10 @@ def _sess(s: _FakeSession) -> so.Session: return cast(so.Session, s) +def _bind(s: _FakeSession) -> sa.engine.Connection: + return cast(sa.engine.Connection, s) + + def test_postgres_backend_identity_and_capabilities(): backend = PostgresBackend() @@ -202,8 +207,8 @@ def test_postgres_backend_materialized_view_methods_emit_expected_sql(): session = _FakeSession() selectable = sa.select(sa.literal(1).label("n")) - backend.create_materialized_view(_sess(session), "mv_test", selectable) - backend.refresh_materialized_view(_sess(session), "mv_test") + backend.create_materialized_view(_bind(session), "mv_test", selectable) + backend.refresh_materialized_view(_bind(session), "mv_test") assert any( "CREATE MATERIALIZED VIEW IF NOT EXISTS mv_test as SELECT" in sql @@ -221,7 +226,7 @@ def test_postgres_backend_rejects_mismatched_dialect_bind(): selectable = sa.select(sa.literal(1).label("n")) with pytest.raises(TypeError, match="received a 'sqlite' connection; expected 'postgresql'"): - backend.create_materialized_view(_sess(session), "mv_test", selectable) + backend.create_materialized_view(_bind(session), "mv_test", selectable) assert session.statements == [] @@ -242,7 +247,7 @@ def test_postgres_backend_rejects_dialect_that_drifted_after_resolve_backend(): selectable = sa.select(sa.literal(1).label("n")) with pytest.raises(TypeError, match="received a 'sqlite' connection; expected 'postgresql'"): - backend.create_materialized_view(_sess(sqlite_session), "mv_test", selectable) + backend.create_materialized_view(_bind(sqlite_session), "mv_test", selectable) def test_postgres_backend_quotes_unqualified_materialized_view_name(): @@ -252,9 +257,9 @@ def test_postgres_backend_quotes_unqualified_materialized_view_name(): session = _FakeSession() selectable = sa.select(sa.literal(1).label("n")) - backend.create_materialized_view(_sess(session), "mv name", selectable) + backend.create_materialized_view(_bind(session), "mv name", selectable) backend.create_materialized_view_index( - _sess(session), "mv name", MaterializedViewIndex(name="mv_name_idx", columns=("n",)) + _bind(session), "mv name", MaterializedViewIndex(name="mv_name_idx", columns=("n",)) ) assert any( @@ -268,7 +273,7 @@ def test_postgres_backend_create_mv_quotes_name_for_legacy_search_path_resolutio backend = PostgresBackend() session = _FakeSession() backend.create_materialized_view( - _sess(session), "MixedCaseMv", sa.select(sa.literal(1).label("n")) + _bind(session), "MixedCaseMv", sa.select(sa.literal(1).label("n")) ) assert any( @@ -284,7 +289,7 @@ def test_postgres_backend_create_materialized_view_index_emits_expected_sql(): session = _FakeSession(schema_translate_map={Role.PRIMARY.value: "reporting"}) index = MaterializedViewIndex(name="mv_test_row_id_uq", columns=("row_id",), unique=True) - backend.create_materialized_view_index(_sess(session), "mv_test", index, schema="reporting") + backend.create_materialized_view_index(_bind(session), "mv_test", index, schema="reporting") assert session.statements == [ 'CREATE UNIQUE INDEX IF NOT EXISTS "mv_test_row_id_uq" ON reporting.mv_test ("row_id")' @@ -301,14 +306,14 @@ def test_postgres_backend_create_materialized_view_index_failure_mentions_index_ index = MaterializedViewIndex(name="mv_test_row_id_uq", columns=("row_id",)) with pytest.raises(MaterializationError, match="mv_test_row_id_uq"): - backend.create_materialized_view_index(_sess(session), "mv_test", index) + backend.create_materialized_view_index(_bind(session), "mv_test", index) def test_postgres_backend_drop_materialized_view_default_args(): backend = PostgresBackend() session = _FakeSession(schema_translate_map={Role.PRIMARY.value: "reporting"}) - backend.drop_materialized_view(_sess(session), "mv_test", schema="reporting") + backend.drop_materialized_view(_bind(session), "mv_test", schema="reporting") assert session.statements == ["DROP MATERIALIZED VIEW IF EXISTS reporting.mv_test"] @@ -318,7 +323,7 @@ def test_postgres_backend_drop_materialized_view_cascade_and_if_exists_false(): session = _FakeSession(schema_translate_map={Role.PRIMARY.value: "reporting"}) backend.drop_materialized_view( - _sess(session), "mv_test", schema="reporting", if_exists=False, cascade=True + _bind(session), "mv_test", schema="reporting", if_exists=False, cascade=True ) assert session.statements == ["DROP MATERIALIZED VIEW reporting.mv_test CASCADE"] @@ -337,7 +342,7 @@ def test_postgres_backend_drop_materialized_view_failure_preserves_cause(): ) with pytest.raises(MaterializationError) as exc_info: - backend.drop_materialized_view(_sess(session), "mv_test") + backend.drop_materialized_view(_bind(session), "mv_test") assert exc_info.value.__cause__ is original assert exc_info.value.failure.cause is original @@ -356,7 +361,7 @@ def test_postgres_backend_create_materialized_view_failure_preserves_cause(): with pytest.raises(MaterializationError) as exc_info: backend.create_materialized_view( - _sess(session), "mv_test", sa.select(sa.literal(1).label("n")) + _bind(session), "mv_test", sa.select(sa.literal(1).label("n")) ) assert exc_info.value.__cause__ is original @@ -371,7 +376,7 @@ def test_postgres_backend_refresh_concurrently_without_declared_unique_index_rai session = _FakeSession(schema_translate_map={Role.PRIMARY.value: "reporting"}) with pytest.raises(ConcurrentRefreshNotEligibleError, match="no simple unique index"): - backend.refresh_materialized_view(_sess(session), "mv_test", concurrently=True) + backend.refresh_materialized_view(_bind(session), "mv_test", concurrently=True) assert session.statements == [] @@ -389,21 +394,23 @@ def test_postgres_backend_refresh_concurrently_declared_but_database_rejects_it_ ) backend = PostgresBackend() session = _FakeSession( - raise_on_execute=sa.exc.OperationalError("REFRESH ...", {}, orig), + raise_on_execute=sa_exc.OperationalError("REFRESH ...", {}, orig), schema_translate_map={Role.PRIMARY.value: "reporting"}, ) index = MaterializedViewIndex(name="mv_test_row_id_uq", columns=("row_id",), unique=True) with pytest.raises(ConcurrentRefreshNotEligibleError, match="cannot refresh") as exc_info: backend.refresh_materialized_view( - _sess(session), + _bind(session), "mv_test", schema="reporting", concurrently=True, declared_indexes=(index,), ) - assert exc_info.value.failure.cause.orig is orig + cause = exc_info.value.failure.cause + assert isinstance(cause, sa_exc.OperationalError) + assert cause.orig is orig assert session.statements == ["REFRESH MATERIALIZED VIEW CONCURRENTLY reporting.mv_test;"] @@ -415,14 +422,14 @@ def test_postgres_backend_refresh_concurrently_unrelated_operational_error_propa orig = psycopg.errors.QueryCanceled("canceling statement due to statement timeout") backend = PostgresBackend() session = _FakeSession( - raise_on_execute=sa.exc.OperationalError("REFRESH ...", {}, orig), + raise_on_execute=sa_exc.OperationalError("REFRESH ...", {}, orig), schema_translate_map={Role.PRIMARY.value: "reporting"}, ) index = MaterializedViewIndex(name="mv_test_row_id_uq", columns=("row_id",), unique=True) - with pytest.raises(sa.exc.OperationalError) as exc_info: + with pytest.raises(sa_exc.OperationalError) as exc_info: backend.refresh_materialized_view( - _sess(session), + _bind(session), "mv_test", concurrently=True, declared_indexes=(index,), @@ -439,7 +446,7 @@ def test_postgres_backend_refresh_concurrently_with_declared_index_emits_concurr index = MaterializedViewIndex(name="mv_test_row_id_uq", columns=("row_id",), unique=True) backend.refresh_materialized_view( - _sess(session), + _bind(session), "mv_test", schema="reporting", concurrently=True, @@ -531,7 +538,7 @@ def test_postgres_backend_refresh_concurrently_raises_when_declared_index_was_ne ) assert "concurrently" in str(exc_info.value).lower() - assert isinstance(exc_info.value.__cause__, sa.exc.OperationalError) + assert isinstance(exc_info.value.__cause__, sa_exc.OperationalError) def test_postgres_backend_materialized_view_legacy_unqualified_path_still_round_trips(pg_db): @@ -543,7 +550,7 @@ def test_postgres_backend_materialized_view_legacy_unqualified_path_still_round_ assert conn.execute(sa.text("SELECT n FROM mv_legacy_test")).scalar() == 1 backend.drop_materialized_view(conn, "mv_legacy_test") - with pytest.raises(sa.exc.ProgrammingError): + with pytest.raises(sa_exc.ProgrammingError): conn.execute(sa.text("SELECT n FROM mv_legacy_test")) diff --git a/tests/backends/test_reserved_schema.py b/tests/backends/test_reserved_schema.py index 9e85312..58d233d 100644 --- a/tests/backends/test_reserved_schema.py +++ b/tests/backends/test_reserved_schema.py @@ -21,6 +21,7 @@ from oa_configurator import ( CDMDatabaseConfig, GenericDatabaseConfig, + ResolvedCDMDatabase, Resolver, SchemaOwnershipError, ) @@ -49,8 +50,10 @@ def test_resolving_cdm_database_with_staging_schema_name_raises(pg_db, cleanup_a engine = resolver.resolve_database("loader").create_engine( schema_claims=[staging_schema_claim()] ) + cdm = resolver.resolve_database(cdm_name) + assert isinstance(cdm, ResolvedCDMDatabase) try: with pytest.raises(SchemaOwnershipError, match=f"{STAGING_SCHEMA!r}.*orm_loader"): - resolver.resolve_database(cdm_name).create_engines() + cdm.create_engines() finally: engine.dispose() diff --git a/tests/conftest.py b/tests/conftest.py index 9255295..9dfba8a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -from collections.abc import Iterable, Iterator +from collections.abc import Generator, Iterable from contextlib import contextmanager from pathlib import Path @@ -67,7 +67,7 @@ def pg_session(pg_db): @contextmanager def schema_scoped_session( pg_db, table: sa.Table, *, prefix: str, split_roles: Iterable[Role] = () -) -> Iterator[tuple[ScopedTestSchema, so.Session]]: +) -> Generator[tuple[ScopedTestSchema, so.Session]]: """A Session on committed scoped test schemas, with the staging schema claimed and *table* already created.""" with scoped_test_schema( diff --git a/tests/loaders/test_dedupe.py b/tests/loaders/test_dedupe.py index 8499318..944f775 100644 --- a/tests/loaders/test_dedupe.py +++ b/tests/loaders/test_dedupe.py @@ -1,4 +1,4 @@ -from typing import ClassVar, cast +from typing import cast import pandas as pd import pyarrow as pa @@ -19,7 +19,7 @@ class Base(DeclarativeBase): class DedupTable(Base, CSVLoadableTableInterface): __tablename__ = "dedup_table" - __table_args__: ClassVar[dict[str, str]] = {"schema": Role.PRIMARY.value} + __table_args__ = ({"schema": Role.PRIMARY.value},) id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) value: so.Mapped[str] = so.mapped_column(sa.String, nullable=False) diff --git a/tests/loaders/test_loader_e2e.py b/tests/loaders/test_loader_e2e.py index c9926ab..acc575f 100644 --- a/tests/loaders/test_loader_e2e.py +++ b/tests/loaders/test_loader_e2e.py @@ -8,6 +8,7 @@ import sqlalchemy.event as sae import sqlalchemy.orm as so from oa_configurator import Role as SchemaRole +from sqlalchemy.exc import IntegrityError from orm_loader.backends import resolve_backend from orm_loader.helpers import IngestError @@ -828,7 +829,7 @@ def test_sqlite_failed_batched_replace_keeps_original_target_rows(session, tmp_p ] ).to_csv(csv_path, index=False, sep="\t") - with pytest.raises(sa.exc.IntegrityError): + with pytest.raises(IntegrityError): SimpleTable.load_csv( session, csv_path, diff --git a/tests/loaders/test_pandas_loader.py b/tests/loaders/test_pandas_loader.py index 48821de..10e6c8a 100644 --- a/tests/loaders/test_pandas_loader.py +++ b/tests/loaders/test_pandas_loader.py @@ -1,17 +1,21 @@ from pathlib import Path +from typing import cast import pandas as pd import sqlalchemy as sa from orm_loader.loaders.data_classes import LoaderContext from orm_loader.loaders.loader_interface import PandasLoader +from orm_loader.tables.typing import CSVTableProtocol from tests.models import PandasLoaderTable +_PandasLoaderTableCls = cast(type[CSVTableProtocol], PandasLoaderTable) + def test_loader_context_fields(session): ctx = LoaderContext( - tableclass=PandasLoaderTable, + tableclass=_PandasLoaderTableCls, session=session, path=Path("file.csv"), staging_table=PandasLoaderTable.__table__, @@ -35,7 +39,7 @@ def test_pandas_loader_case_insensitive_headers(tmp_path, session): csv.write_text("ID,Value\n1,alpha\n2,beta\n") ctx = LoaderContext( - tableclass=PandasLoaderTable, + tableclass=_PandasLoaderTableCls, session=session, path=csv, staging_table=PandasLoaderTable.__table__, @@ -63,7 +67,7 @@ def test_pandas_loader_literal_mode_preserves_quotes(tmp_path, session): csv.write_text('id,value\n1,"hello"\n2,"world"\n') ctx = LoaderContext( - tableclass=PandasLoaderTable, + tableclass=_PandasLoaderTableCls, session=session, path=csv, staging_table=PandasLoaderTable.__table__, @@ -92,7 +96,7 @@ def test_pandas_loader_csv_mode_strips_quotes(tmp_path, session): csv.write_text('id,value\n1,"hello"\n2,"world"\n') ctx = LoaderContext( - tableclass=PandasLoaderTable, + tableclass=_PandasLoaderTableCls, session=session, path=csv, staging_table=PandasLoaderTable.__table__, @@ -120,7 +124,7 @@ def test_pandas_loader_by_delimiter_tab_preserves_quotes(tmp_path, session): tsv.write_text('id\tvalue\n1\t"hi"\n2\tbye\n') ctx = LoaderContext( - tableclass=PandasLoaderTable, + tableclass=_PandasLoaderTableCls, session=session, path=tsv, staging_table=PandasLoaderTable.__table__, @@ -147,7 +151,7 @@ def test_pandas_dedupe_internal(session, tmp_path): ) ctx = LoaderContext( - tableclass=PandasLoaderTable, + tableclass=_PandasLoaderTableCls, session=session, path=Path("file.csv"), staging_table=PandasLoaderTable.__table__, diff --git a/tests/loaders/test_parquet_loader.py b/tests/loaders/test_parquet_loader.py index 8fffa8e..375d7e6 100644 --- a/tests/loaders/test_parquet_loader.py +++ b/tests/loaders/test_parquet_loader.py @@ -1,4 +1,4 @@ -from typing import ClassVar, cast +from typing import cast import pandas as pd import pyarrow as pa @@ -19,7 +19,7 @@ class Base(DeclarativeBase): class ParquetTable(Base, CSVLoadableTableInterface): __tablename__ = "parquet_table" - __table_args__: ClassVar[dict[str, str]] = {"schema": Role.PRIMARY.value} + __table_args__ = ({"schema": Role.PRIMARY.value},) id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) value: so.Mapped[int] = so.mapped_column(sa.Integer, nullable=False) @@ -57,7 +57,7 @@ class NullSentinelTable(Base, CSVLoadableTableInterface): treats text null sentinels the same way the scalar path does.""" __tablename__ = "null_sentinel_table" - __table_args__: ClassVar[dict[str, str]] = {"schema": Role.PRIMARY.value} + __table_args__ = ({"schema": Role.PRIMARY.value},) id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) name: so.Mapped[str | None] = so.mapped_column(sa.String, nullable=True) @@ -134,7 +134,7 @@ def test_arrow_path_nulls_float_nan(session, engine, tmp_path): class FloatTable(Base, CSVLoadableTableInterface): __tablename__ = "float_sentinel_table" - __table_args__: ClassVar[dict[str, str]] = {"schema": Role.PRIMARY.value} + __table_args__ = ({"schema": Role.PRIMARY.value},) id: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True) score: so.Mapped[float | None] = so.mapped_column(sa.Float, nullable=True) diff --git a/tests/loaders/test_split_connection.py b/tests/loaders/test_split_connection.py index d6ee2b5..ecfb86e 100644 --- a/tests/loaders/test_split_connection.py +++ b/tests/loaders/test_split_connection.py @@ -5,7 +5,13 @@ import pandas as pd import sqlalchemy as sa import sqlalchemy.orm as so -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Resolver, StackConfig +from oa_configurator import ( + CDMDatabaseConfig, + ConnectionConfig, + ResolvedCDMDatabase, + Resolver, + StackConfig, +) from oa_configurator import Role as SchemaRole from sqlalchemy.exc import UnboundExecutionError @@ -32,6 +38,7 @@ def test_vocab_csv_load_routes_every_session_bind_to_vocab_file(tmp_path): ) ) resolved = resolver.resolve_database("test_cdm") + assert isinstance(resolved, ResolvedCDMDatabase) primary_engine, vocab_engine = resolved.create_engines(schema_claims=[staging_schema_claim()]) VocabSchemaTable.__table__.create(vocab_engine) diff --git a/tests/mappers/test_materialised_view_mixin.py b/tests/mappers/test_materialised_view_mixin.py index 14fa3c1..df5e16b 100644 --- a/tests/mappers/test_materialised_view_mixin.py +++ b/tests/mappers/test_materialised_view_mixin.py @@ -169,7 +169,7 @@ def test_create_mv_forwards_default_args_to_backend(fake_backend: _FakeBackend, def test_create_mv_rejects_invalid_bind(fake_backend: _FakeBackend): with pytest.raises(TypeError, match="bind must be a SQLAlchemy Engine or Connection"): - _NoIndexMv.create_mv("bind") + _NoIndexMv.create_mv("bind") # ty: ignore[invalid-argument-type] assert fake_backend.calls == [] diff --git a/tests/registry/test_model_descriptor.py b/tests/registry/test_model_descriptor.py index b22b270..5b8d136 100644 --- a/tests/registry/test_model_descriptor.py +++ b/tests/registry/test_model_descriptor.py @@ -1,3 +1,5 @@ +from typing import ClassVar + import pytest import sqlalchemy as sa import sqlalchemy.orm as so @@ -11,7 +13,7 @@ def test_model_descriptor_from_model_basic(): class T(ORMTableBase, Base): - __tablename__ = "t" + __tablename__: ClassVar[str] = "t" id = sa.Column(sa.Integer, primary_key=True) a = sa.Column(sa.String) @@ -30,7 +32,7 @@ class Parent(ORMTableBase, Base): id = sa.Column(sa.Integer, primary_key=True) class Child(ORMTableBase, Base): - __tablename__ = "child" + __tablename__: ClassVar[str] = "child" id = sa.Column(sa.Integer, primary_key=True) parent_id = sa.Column(sa.Integer, sa.ForeignKey("parent.id")) From cb30708afb70bd639213c6d91ccf6b8901c25e18 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Fri, 23 Oct 2026 00:14:59 +0000 Subject: [PATCH 7/8] Remove ty suppressions and casts in orm-loader --- pyproject.toml | 1 + src/orm_loader/backends/base.py | 19 +++------ src/orm_loader/backends/resolve.py | 8 ++-- src/orm_loader/helpers/bootstrap.py | 39 +++++++++++-------- src/orm_loader/loaders/data/__init__.py | 3 +- src/orm_loader/loaders/data/converters.py | 24 ++++++------ src/orm_loader/loaders/loader_interface.py | 29 +++++++------- src/orm_loader/loaders/loading_helpers.py | 26 ++++++------- .../mappers/materialised_view_mixin.py | 6 ++- src/orm_loader/tables/loadable_table.py | 2 +- tests/backends/test_base_backend.py | 26 +++++++------ tests/loaders/test_parquet_loader.py | 2 +- tests/mappers/test_materialised_view_mixin.py | 3 +- tests/registry/test_model_descriptor.py | 5 ++- tests/registry/test_registry.py | 18 ++++++--- uv.lock | 14 +++++++ 16 files changed, 126 insertions(+), 99 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e9fba20..444c5c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ dev = [ "pytest>=9.0.3", "ty==0.0.90", "ruff==0.17.1", + "pyarrow-stubs==20.0.0.20260819", "mkdocs-material>=9.7.7", "mkdocstrings-python>=2.0.1", "requests>=2.33.0", diff --git a/src/orm_loader/backends/base.py b/src/orm_loader/backends/base.py index 42d425c..d65e940 100644 --- a/src/orm_loader/backends/base.py +++ b/src/orm_loader/backends/base.py @@ -5,7 +5,7 @@ from contextlib import AbstractContextManager, contextmanager from dataclasses import dataclass from functools import wraps -from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast +from typing import TYPE_CHECKING, Any, Concatenate import sqlalchemy as sa import sqlalchemy.orm as so @@ -64,28 +64,21 @@ def staging_schema_claim( ) -P = ParamSpec("P") -R = TypeVar("R") - - -def requires_capability( +def requires_capability[B: DatabaseBackend, **P, R]( capability_name: str, feature_name: str, -) -> Callable[[Callable[P, R]], Callable[P, R]]: +) -> Callable[[Callable[Concatenate[B, P], R]], Callable[Concatenate[B, P], R]]: """Guard a concrete backend method with a capability requirement. Subclass overrides must apply this decorator themselves; Python does not automatically retain decorators when a method is overridden. """ - def decorator(method: Callable[P, R]) -> Callable[P, R]: + def decorator(method: Callable[Concatenate[B, P], R]) -> Callable[Concatenate[B, P], R]: @wraps(method) - def guarded(*args: P.args, **kwargs: P.kwargs) -> R: - # ParamSpec keeps the complete method signature, but type checkers - # cannot infer that args[0] is the backend instance here. - self = cast("DatabaseBackend", args[0]) + def guarded(self: B, *args: P.args, **kwargs: P.kwargs) -> R: self._require_capability(capability_name, feature_name) - return method(*args, **kwargs) + return method(self, *args, **kwargs) return guarded diff --git a/src/orm_loader/backends/resolve.py b/src/orm_loader/backends/resolve.py index d33e826..160aa4e 100644 --- a/src/orm_loader/backends/resolve.py +++ b/src/orm_loader/backends/resolve.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, cast +from typing import Any import sqlalchemy.orm as so from oa_configurator import Bindable, UnregisteredSchemaTagError, physical_schema_of @@ -16,10 +16,10 @@ def _dialect(bindable: Bindable) -> Dialect: - if hasattr(bindable, "dialect"): - dialect_name = cast(Any, bindable).dialect.name - else: + dialect = getattr(bindable, "dialect", None) + if dialect is None: raise TypeError(f"Unsupported bindable type: {type(bindable)!r}") + dialect_name = dialect.name try: return Dialect(dialect_name) diff --git a/src/orm_loader/helpers/bootstrap.py b/src/orm_loader/helpers/bootstrap.py index 0b44d00..6126c84 100644 --- a/src/orm_loader/helpers/bootstrap.py +++ b/src/orm_loader/helpers/bootstrap.py @@ -1,6 +1,6 @@ import logging from collections.abc import Iterable -from typing import cast +from typing import overload import sqlalchemy as sa from oa_configurator import ( @@ -21,6 +21,10 @@ logger = logging.getLogger(__name__) +@overload +def _resolve_binds(resolved: ResolvedDatabase, bindable: None) -> tuple[sa.Engine, sa.Engine]: ... +@overload +def _resolve_binds(resolved: ResolvedDatabase, bindable: Bindable) -> tuple[Bindable, Bindable]: ... def _resolve_binds( resolved: ResolvedDatabase, bindable: Bindable | None ) -> tuple[Bindable, Bindable]: @@ -81,22 +85,20 @@ def create_tables( If a table has no schema tag. """ tables = list(tables) - tagged = [(table, table.schema) for table in tables] - if not tagged: + if not tables: return - untagged = sorted(table.name for table, tag in tagged if tag is None) + untagged = sorted(table.name for table in tables if table.schema is None) if untagged: raise ValueError(f"create_tables() needs schema-tagged tables, got untagged {untagged}.") - schema_tags = {tag for _, tag in tagged if tag is not None} + tag_of = {table: table.schema for table in tables if table.schema is not None} + schema_tags = set(tag_of.values()) if not resolved.tags_share_a_transaction(*schema_tags): raise ValueError( "create_tables() requires all tables to share one database transaction; " f"got schema tags {sorted(schema_tags)}." ) crossing = any( - not resolved.foreign_key_can_span( - cast(str, table.schema), referred_schema_tag(fk, cast(str, table.schema)) - ) + not resolved.foreign_key_can_span(tag_of[table], referred_schema_tag(fk, tag_of[table])) for table in tables for fk in table.foreign_keys ) @@ -117,7 +119,7 @@ def create_tables( candidate for candidate in metadata.tables.values() if candidate.schema is not None - and resolved.tags_share_a_transaction(candidate.schema, cast(str, table.schema)) + and resolved.tags_share_a_transaction(candidate.schema, tag_of[table]) ) for metadata, related in related_by_metadata.items(): @@ -162,8 +164,12 @@ def create_db(resolved: ResolvedDatabase, *, bindable: Bindable | None = None) - "the module(s) defining your ORM models likely weren't imported yet. " "SQLAlchemy only registers a model on Base when its module runs." ) - owned = bindable is None - primary_bind, vocab_bind = _resolve_binds(resolved, bindable) + owned_engines: tuple[sa.Engine, sa.Engine] | None = None + if bindable is None: + owned_engines = _resolve_binds(resolved, None) + primary_bind, vocab_bind = owned_engines + else: + primary_bind, vocab_bind = _resolve_binds(resolved, bindable) tables_by_bind: dict[int, tuple[Bindable, dict[str, list[sa.Table]]]] = {} untagged: list[str] = [] @@ -196,12 +202,11 @@ def create_db(resolved: ResolvedDatabase, *, bindable: Bindable | None = None) - ) create_tables(connection, tables, resolved=resolved) finally: - if owned: - # _resolve_binds() only returns Engine instances on this path - # (bindable is None): create_engines()/create_engine(). - cast(sa.engine.Engine, primary_bind).dispose() - if vocab_bind is not primary_bind: - cast(sa.engine.Engine, vocab_bind).dispose() + if owned_engines is not None: + primary_engine, vocab_engine = owned_engines + primary_engine.dispose() + if vocab_engine is not primary_engine: + vocab_engine.dispose() def bootstrap( diff --git a/src/orm_loader/loaders/data/__init__.py b/src/orm_loader/loaders/data/__init__.py index 6df6e71..69b63c9 100644 --- a/src/orm_loader/loaders/data/__init__.py +++ b/src/orm_loader/loaders/data/__init__.py @@ -1,6 +1,7 @@ -from .converters import cast_arrow_column, perform_cast +from .converters import ArrowArray, cast_arrow_column, perform_cast __all__ = [ + "ArrowArray", "cast_arrow_column", "perform_cast", ] diff --git a/src/orm_loader/loaders/data/converters.py b/src/orm_loader/loaders/data/converters.py index f571995..54bc96a 100644 --- a/src/orm_loader/loaders/data/converters.py +++ b/src/orm_loader/loaders/data/converters.py @@ -78,20 +78,23 @@ def _normalise_null(value: Any) -> Any | None: _ARROW_NULL_STRINGS = pa.array(sorted(_NULL_STRINGS | {""}), type=pa.string()) -def _normalise_null_arrow(arr: pa.Array | pa.ChunkedArray) -> pa.Array | pa.ChunkedArray: +type ArrowArray = pa.Array | pa.ChunkedArray + + +def _normalise_null_arrow(arr: ArrowArray) -> ArrowArray: arrow_type = arr.type if pa.types.is_string(arrow_type) or pa.types.is_large_string(arrow_type): - probe = pc.utf8_lower(pc.utf8_trim_whitespace(arr)) # type: ignore - return pc.if_else( # type: ignore - pc.is_in(probe, value_set=_ARROW_NULL_STRINGS), # type: ignore + probe = pc.utf8_lower(pc.utf8_trim_whitespace(arr)) + return pc.if_else( + pc.is_in(probe, value_set=_ARROW_NULL_STRINGS), pa.nulls(len(arr), arrow_type), arr, ) if pa.types.is_floating(arrow_type): - return pc.if_else( # type: ignore - pc.is_nan(arr), # type: ignore + return pc.if_else( + pc.is_nan(arr), pa.nulls(len(arr), arrow_type), arr, ) @@ -425,8 +428,8 @@ def perform_cast( def cast_arrow_column( - arr: pa.Array, sa_col: ColumnElement[Any], stats: TableCastingStats | None = None -) -> pa.Array: + arr: ArrowArray, sa_col: ColumnElement[Any], stats: TableCastingStats | None = None +) -> ArrowArray: arr = _normalise_null_arrow(arr) column_rule = _COLUMN_CAST_RULES.get((sa_col.table.name, sa_col.name)) @@ -450,9 +453,8 @@ def cast_arrow_column( try: return pc.cast(arr, arrow_type) except pa.ArrowInvalid: - validity = pc.is_valid(arr) # type: ignore - invalid_mask = pc.invert(validity) # type: ignore - invalid_count = pc.sum(invalid_mask).as_py() # type: ignore + invalid_mask = pc.is_null(arr) + invalid_count = arr.null_count if invalid_count == 0: return arr diff --git a/src/orm_loader/loaders/loader_interface.py b/src/orm_loader/loaders/loader_interface.py index 9980738..e521e46 100644 --- a/src/orm_loader/loaders/loader_interface.py +++ b/src/orm_loader/loaders/loader_interface.py @@ -12,7 +12,7 @@ import pyarrow.dataset as ds from ..helpers import IngestError -from .data import cast_arrow_column, perform_cast +from .data import ArrowArray, cast_arrow_column, perform_cast from .data_classes import LoaderContext, LoaderInterface, TableCastingStats from .loading_helpers import ( arrow_drop_duplicates, @@ -136,7 +136,7 @@ def _on_cast_error(value, *, _col=col_name): if required_cols: null_mask = df[required_cols].isna() for col in required_cols: - null_count = int(null_mask[col].sum()) # type: ignore[arg-type] + null_count = int(null_mask[col].sum()) if null_count > 0: logger.warning( "Found %d rows with unexpected nulls in %s.%s", @@ -226,7 +226,7 @@ def cast_to_model(cls, data: pa.Table, ctx: LoaderContext) -> pa.Table: table_name = ctx.tableclass.__tablename__ stats = TableCastingStats(table_name=table_name) model_columns = ctx.tableclass.model_columns() - arrays: dict[str, pa.Array] = {} + arrays: dict[str, ArrowArray] = {} for col_name, sa_col in model_columns.items(): if col_name not in data.schema.names: continue @@ -238,15 +238,17 @@ def cast_to_model(cls, data: pa.Table, ctx: LoaderContext) -> pa.Table: stats=stats, ) - out = pa.table(arrays) + out = pa.table(list(arrays.values()), names=list(arrays)) _require_columns_present(ctx.tableclass, out.schema.names) required_cols = list(ctx.tableclass.required_columns()) if required_cols: - masks = [pc.is_valid(out[c]) for c in required_cols] # type: ignore - valid_mask = reduce(pc.and_, masks) # type: ignore + masks = [pc.is_valid(out[c]) for c in required_cols] + valid_mask = reduce(lambda left, right: pc.and_(left, right), masks) - dropped = out.num_rows - pc.sum(valid_mask).as_py() # type: ignore + before = out.num_rows + out = out.filter(valid_mask) + dropped = before - out.num_rows if dropped > 0: logger.warning( "Dropped %d rows with unexpected nulls in %s", @@ -254,8 +256,6 @@ def cast_to_model(cls, data: pa.Table, ctx: LoaderContext) -> pa.Table: table_name, ) - out = out.filter(valid_mask) - if stats.has_failures(): for col, col_stats in stats.columns.items(): logger.warning( @@ -265,7 +265,9 @@ def cast_to_model(cls, data: pa.Table, ctx: LoaderContext) -> pa.Table: return out @classmethod - def dedupe(cls, data: pa.Table, ctx: LoaderContext) -> pa.Table: + def dedupe(cls, data: pd.DataFrame | pa.Table, ctx: LoaderContext) -> Any: + if isinstance(data, pd.DataFrame): + return super().dedupe(data, ctx) if data.num_rows == 0: return data @@ -303,16 +305,13 @@ def orm_file_load(cls, ctx: LoaderContext) -> int: for record_batch in cls._scan_batches(ctx): if record_batch.num_rows == 0: continue - data: pa.Table | pa.RecordBatch = record_batch + data = pa.Table.from_batches([record_batch]) if ctx.normalise: data = cls.cast_to_model(data, ctx=ctx) if ctx.dedupe: data = cls.dedupe(data, ctx) - if isinstance(data, (pa.RecordBatch, pa.Table)): - df = data.to_pandas() - else: - df = data + df = data.to_pandas() if df.empty: continue diff --git a/src/orm_loader/loaders/loading_helpers.py b/src/orm_loader/loaders/loading_helpers.py index fc86c31..6243b03 100644 --- a/src/orm_loader/loaders/loading_helpers.py +++ b/src/orm_loader/loaders/loading_helpers.py @@ -4,8 +4,10 @@ import io import logging import re +from collections.abc import Iterator +from functools import reduce from pathlib import Path -from typing import Any +from typing import Any, Literal import chardet import pyarrow as pa @@ -196,21 +198,17 @@ def arrow_drop_duplicates( if table.num_rows == 0: return table - sort_keys = [(name, "ascending") for name in pk_names] - sorted_idx = pc.sort_indices(table, sort_keys=sort_keys) # type: ignore + sort_keys: list[tuple[str, Literal["ascending"]]] = [(name, "ascending") for name in pk_names] + sorted_idx = pc.sort_indices(table, sort_keys=sort_keys) sorted_table = table.take(sorted_idx) - diffs = [] + # pyarrow-stubs resolve these compute calls to a union of scalar, array and + # expression results, so the intermediate masks are typed Any. + diffs: list[Any] = [] for name in pk_names: col = sorted_table[name] - previous_arr = col[:-1] - this_arr = col[1:] - diffs.append( - pc.not_equal(previous_arr, this_arr) # type: ignore - ) - keep_tail = diffs[0] - for d in diffs[1:]: - keep_tail = pc.or_(keep_tail, d) # type: ignore - keep = pc.fill_null(keep_tail, True) + diffs.append(pc.not_equal(col[:-1], col[1:])) + keep_tail: Any = reduce(pc.or_, diffs) + keep: Any = pc.fill_null(keep_tail, pa.scalar(True)) if isinstance(keep, pa.ChunkedArray): keep = keep.combine_chunks() keep = pa.concat_arrays( @@ -226,7 +224,7 @@ def arrow_drop_duplicates( def conservative_load_parquet( path: Path, wanted_cols: list[str], chunksize: int | None = None -) -> pa.Table: +) -> Iterator[pa.RecordBatch]: delimiter = infer_delim(path) encoding = infer_encoding(path)["encoding"] convert_opts = pv.ConvertOptions( diff --git a/src/orm_loader/mappers/materialised_view_mixin.py b/src/orm_loader/mappers/materialised_view_mixin.py index 643ce15..0086cf3 100644 --- a/src/orm_loader/mappers/materialised_view_mixin.py +++ b/src/orm_loader/mappers/materialised_view_mixin.py @@ -368,7 +368,9 @@ def drop_mv( ) -def resolve_mv_refresh_order(mv_classes: list[type[MaterializedViewMixin]]) -> list[type]: +def resolve_mv_refresh_order( + mv_classes: list[type[MaterializedViewMixin]], +) -> list[type[MaterializedViewMixin]]: """ `resolve_mv_refresh_order` @@ -432,7 +434,7 @@ def refresh_all_mvs( refresh_all_mvs(engine, ALL_MVS) ``` """ - ordered: list[type[MaterializedViewMixin]] = resolve_mv_refresh_order(mv_classes) # ty: ignore[invalid-assignment] + ordered = resolve_mv_refresh_order(mv_classes) for mv in ordered: mv.refresh_mv(bind) diff --git a/src/orm_loader/tables/loadable_table.py b/src/orm_loader/tables/loadable_table.py index fce4467..1954328 100644 --- a/src/orm_loader/tables/loadable_table.py +++ b/src/orm_loader/tables/loadable_table.py @@ -634,5 +634,5 @@ def csv_columns(cls) -> dict[str, sa.ColumnElement[Any]]: Mapping of input column names to SQLAlchemy columns. """ cols = cls.model_columns() - computed_names = {c.name for c in cls.__table__.columns if c.computed is not None} # type: ignore + computed_names = {c.name for c in cls.__table__.columns if c.computed is not None} # ty: ignore[unresolved-attribute] return {k: v for k, v in cols.items() if k not in computed_names} diff --git a/tests/backends/test_base_backend.py b/tests/backends/test_base_backend.py index ffb9d5d..f20243e 100644 --- a/tests/backends/test_base_backend.py +++ b/tests/backends/test_base_backend.py @@ -21,7 +21,6 @@ ) if TYPE_CHECKING: - from orm_loader.loaders.data_classes import LoaderContext from orm_loader.tables.typing import CSVTableProtocol @@ -165,8 +164,9 @@ def test_backend_capabilities_defaults(): def test_database_backend_is_abstract(): + abstract_backend: Any = DatabaseBackend with pytest.raises(TypeError): - DatabaseBackend() # type: ignore + abstract_backend() def test_fake_backend_can_implement_contract(): @@ -180,9 +180,10 @@ def test_fake_backend_can_implement_contract(): assert backend.supports_dialect(Dialect.POSTGRESQL) is False assert backend.resolve_index_strategy("auto") == "drop_rebuild" assert backend.resolve_index_strategy("keep") == "keep" - assert backend.load_staging_fast(cast("LoaderContext", None)) is None + unused: Any = None + assert backend.load_staging_fast(unused) is None - with backend.merge_context(cast("type[CSVTableProtocol]", None), cast(so.Session, None)): + with backend.merge_context(unused, unused): pass @@ -295,18 +296,20 @@ class _Unknown: class dialect: name = "unknown" + unknown_bindable: Any = _Unknown() with pytest.raises(NotImplementedError, match="Unsupported SQLAlchemy dialect"): - resolve_backend(cast(Engine, _Unknown())) + resolve_backend(unknown_bindable) def test_unsupported_fake_backend_inherits_default_materialized_view_guards(): backend = FakeBackend() + unused: Any = None with pytest.raises(NotImplementedError, match="does not support materialized views"): - backend.drop_materialized_view(cast(Engine, None), "mv_test") + backend.drop_materialized_view(unused, "mv_test") with pytest.raises(NotImplementedError, match="does not support materialized views"): - backend.create_materialized_view_index(cast(Engine, None), "mv_test", cast(Any, None)) + backend.create_materialized_view_index(unused, "mv_test", unused) def test_capable_backend_inheriting_default_materialized_view_methods_fails_loudly(): @@ -323,27 +326,28 @@ def capabilities(self) -> BackendCapabilities: ) backend = CapableBackend() + unused: Any = None with pytest.raises(NotImplementedError, match="has not implemented drop_materialized_view"): - backend.drop_materialized_view(cast(Engine, None), "mv_test") + backend.drop_materialized_view(unused, "mv_test") with pytest.raises( NotImplementedError, match="has not implemented create_materialized_view", ): - backend.create_materialized_view(cast(Engine, None), "mv_test", cast(Any, None)) + backend.create_materialized_view(unused, "mv_test", unused) with pytest.raises( NotImplementedError, match="has not implemented refresh_materialized_view", ): - backend.refresh_materialized_view(cast(Engine, None), "mv_test") + backend.refresh_materialized_view(unused, "mv_test") with pytest.raises( NotImplementedError, match="has not implemented create_materialized_view_index", ): - backend.create_materialized_view_index(cast(Engine, None), "mv_test", cast(Any, None)) + backend.create_materialized_view_index(unused, "mv_test", unused) def test_backends_import_does_not_require_psycopg(): diff --git a/tests/loaders/test_parquet_loader.py b/tests/loaders/test_parquet_loader.py index 375d7e6..b0dd574 100644 --- a/tests/loaders/test_parquet_loader.py +++ b/tests/loaders/test_parquet_loader.py @@ -93,7 +93,7 @@ def test_arrow_and_pandas_paths_agree_on_null_sentinels(session, engine, tmp_pat col = NullSentinelTable.__table__.c.name for raw in _SENTINELS + [" keep me ", "plain"]: - arrow = cast_arrow_column(pa.array([raw], type=pa.string()), col)[0].as_py() + arrow = cast_arrow_column(pa.array([raw], type=pa.string()), col).to_pylist()[0] scalar = cast_scalar(raw, col.type) assert arrow == scalar, f"{raw!r}: arrow={arrow!r} scalar={scalar!r}" diff --git a/tests/mappers/test_materialised_view_mixin.py b/tests/mappers/test_materialised_view_mixin.py index df5e16b..cefa8b8 100644 --- a/tests/mappers/test_materialised_view_mixin.py +++ b/tests/mappers/test_materialised_view_mixin.py @@ -168,8 +168,9 @@ def test_create_mv_forwards_default_args_to_backend(fake_backend: _FakeBackend, def test_create_mv_rejects_invalid_bind(fake_backend: _FakeBackend): + invalid_bind: Any = "bind" with pytest.raises(TypeError, match="bind must be a SQLAlchemy Engine or Connection"): - _NoIndexMv.create_mv("bind") # ty: ignore[invalid-argument-type] + _NoIndexMv.create_mv(invalid_bind) assert fake_backend.calls == [] diff --git a/tests/registry/test_model_descriptor.py b/tests/registry/test_model_descriptor.py index 5b8d136..99730e4 100644 --- a/tests/registry/test_model_descriptor.py +++ b/tests/registry/test_model_descriptor.py @@ -1,4 +1,4 @@ -from typing import ClassVar +from typing import Any, ClassVar import pytest import sqlalchemy as sa @@ -45,5 +45,6 @@ def test_model_descriptor_rejects_unmapped_class(): class NotMapped: pass + not_mapped: Any = NotMapped with pytest.raises(NoInspectionAvailable): - ModelDescriptor.from_model(NotMapped) # type: ignore + ModelDescriptor.from_model(not_mapped) diff --git a/tests/registry/test_registry.py b/tests/registry/test_registry.py index 74dceb9..59479f7 100644 --- a/tests/registry/test_registry.py +++ b/tests/registry/test_registry.py @@ -15,6 +15,12 @@ Base = so.declarative_base() +def _table_spec(table_name: str, *, is_required: bool) -> TableSpec: + return TableSpec( + table_name=table_name, schema="cdm", is_required=is_required, description=table_name + ) + + class FakeResource: """Minimal stand-in for importlib.resources API.""" @@ -90,9 +96,9 @@ def test_missing_required_tables(): reg = ModelRegistry(model_version="1.0") # Fake specs injected directly (unit-level test) - reg._table_specs = { # type: ignore - "person": type("S", (), {"is_required": True})(), - "visit": type("S", (), {"is_required": False})(), + reg._table_specs = { + "person": _table_spec("person", is_required=True), + "visit": _table_spec("visit", is_required=False), } assert reg.missing_required_tables() == {"person"} @@ -104,9 +110,9 @@ class R(ORMTableBase, Base): id = sa.Column(sa.Integer, primary_key=True) reg = ModelRegistry(model_version="1.0") - reg._table_specs = { # type: ignore - "r": type("S", (), {"is_required": True})(), - "x": type("S", (), {"is_required": True})(), + reg._table_specs = { + "r": _table_spec("r", is_required=True), + "x": _table_spec("x", is_required=True), } reg.register_model(R) diff --git a/uv.lock b/uv.lock index 40909c7..1e3fc0a 100644 --- a/uv.lock +++ b/uv.lock @@ -598,6 +598,7 @@ dev = [ { name = "mkdocs-mermaid2-plugin" }, { name = "mkdocstrings-python" }, { name = "oa-configurator", extra = ["postgres"] }, + { name = "pyarrow-stubs" }, { name = "pygments" }, { name = "pytest" }, { name = "python-dotenv" }, @@ -621,6 +622,7 @@ requires-dist = [ { name = "pandas", specifier = ">=2.3.3" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'", specifier = ">=3.2" }, { name = "pyarrow", specifier = ">=23.0.1" }, + { name = "pyarrow-stubs", marker = "extra == 'dev'", specifier = "==20.0.0.20260819" }, { name = "pygments", marker = "extra == 'dev'", specifier = ">=2.20.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.3" }, { name = "python-dotenv", marker = "extra == 'dev'" }, @@ -824,6 +826,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, ] +[[package]] +name = "pyarrow-stubs" +version = "20.0.0.20260819" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyarrow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/a7/8a2ca91ffe4c6576207f932de655d7d8a36485c520dccce70ce7d492b256/pyarrow_stubs-20.0.0.20260819.tar.gz", hash = "sha256:150710a72248bc834bf048d3092713f070904a4af76d40289c43afb3ee189823", size = 238222, upload-time = "2026-08-19T05:52:53.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/6c/eea1d03e475217aea95b1d52aee09c97575d05bbc592c39c085b71dab89f/pyarrow_stubs-20.0.0.20260819-py3-none-any.whl", hash = "sha256:297e60b6e5314739c082b4757d090d8be6047465510eb0684ca954ef7ea58be3", size = 235949, upload-time = "2026-08-19T05:52:54.711Z" }, +] + [[package]] name = "pydantic" version = "2.13.4" From ff8940bca717906e71b5c4c1a7157074bd300987 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Fri, 23 Oct 2026 01:12:29 +0000 Subject: [PATCH 8/8] Switch CI to ruff, ty and lint-free test workflows (temporary branch refs) --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e7a7f4..e79930d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,10 +6,14 @@ on: jobs: label-gate: uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/label-gate.yml@main + ruff: + uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/ruff.yml@ruff-ty-workflows + ty: + uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/ty.yml@ruff-ty-workflows build-test-sqlite: - uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test.yml@main + uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test-sqlite.yml@ruff-ty-workflows build-test-postgres: - uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test-postgres-v2.yml@main + uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test-postgres-v3.yml@ruff-ty-workflows with: postgres-db: orm_loader_test setup-commands: |