From 46e620a82fb3ad0e25e52ac2843b0a76632eedc0 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 26 Aug 2026 04:44:03 +0000 Subject: [PATCH 1/4] fix(kernel): preserve empty metadata filters Signed-off-by: Vu Anh Phung --- CHANGELOG.md | 1 + .../sql/backend/databricks_client.py | 27 ++++--- src/databricks/sql/backend/kernel/client.py | 44 ++++------- src/databricks/sql/client.py | 9 ++- tests/e2e/test_kernel_backend.py | 28 +++++-- tests/unit/test_kernel_client.py | 73 ++++++++++++++++--- 6 files changed, 122 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 439983c3f..518f65831 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Release History # Unreleased +- Kernel metadata filters now preserve empty strings as empty patterns, matching no catalogs, schemas, tables, or columns. Only `None` leaves a filter unset; `%` and `*` retain their existing wildcard behavior (PECOBLR-4221). - Kernel backend (`use_kernel=True`): OAuth **M2M with a JWT private-key client assertion** (RFC 7523) is now supported. Pass `oauth_client_id` + `oauth_jwt_key_file` + `oauth_jwt_kid` (with optional `oauth_jwt_passphrase` for an encrypted PKCS#8 key, `oauth_jwt_algorithm` defaulting to `RS256`, `oauth_scopes`, and `token_url` for the IdP token endpoint) and the connector routes them to the kernel's `auth_type="oauth-m2m-jwt"`, which signs a short-lived assertion with the private key instead of sending a client secret. The kernel owns the token lifecycle. A private-key file is treated as unambiguous JWT M2M intent and is mutually exclusive with `oauth_client_secret` / `credentials_provider` (both raise `NotSupportedError`). Verified end-to-end against an Azure Databricks workspace with the service principal's public certificate registered on its Entra ID app registration. Requires `databricks-sql-kernel >= 0.2.0` with JWT support. - Kernel backend (`use_kernel=True`): OAuth U2M with `auth_type="databricks-oauth"` now forwards the connector's `databricks-sql-python` OAuth-app bundle (`client_id` + `sql offline_access` scopes + redirect port) into the kernel, so a bare U2M connection authenticates as `databricks-sql-python` — parity with the Thrift path — instead of inheriting the kernel's own `databricks-sql-connector` default. A caller-supplied `oauth_client_id` (with its coupled `oauth_redirect_port`) is honored, as is a caller-supplied `oauth_scopes`; absent one, the connector default (`sql offline_access`) is forwarded. Note: the kernel binds a single U2M redirect port, so unlike the Thrift path (which tries the full `8020..8024` range) the kernel path uses only one port and does not fall back to the next port if it is already bound — pass `oauth_redirect_port` (with `oauth_client_id`) to pick a free one on a port collision (PECOBLR-4040) - Kernel backend (`use_kernel=True`): **Azure Entra (Azure AD) service-principal M2M is now supported.** `auth_type="azure-sp-m2m"` forwards `azure_client_id` / `azure_client_secret`; the kernel is the Azure-aware auth core — it builds the Entra v2.0 token endpoint and the `{app_id}/.default` scope, and **auto-discovers the tenant** from the workspace's `/aad/auth` redirect when `azure_tenant_id` is omitted (matching Thrift). The `Authorization` bearer is the Databricks-audience data token, which alone authenticates a workspace-member SP. Set `azure_workspace_resource_id` and the kernel also sends the Azure SP management token (`X-Databricks-Azure-SP-Management-Token`) + `X-Databricks-Azure-Workspace-Resource-Id` header (matching the JDBC driver), so a service principal with an Azure RBAC role but no workspace membership can authenticate; omit it and no ARM management-scope token is fetched. Azure AD **U2M** (`auth_type="azure-oauth"`) now routes to the kernel's OAuth U2M flow, identically to `auth_type="databricks-oauth"`: the kernel runs the in-house workspace-federated browser flow, which Azure workspaces support (the workspace federates login to Entra). It forwards the connector's `databricks-sql-python` OAuth app, not the Thrift Azure app (`96eecda7` / port 8030), which is registered for Thrift's direct-Entra flow the kernel does not perform (PECOBLR-4141; PECOBLR-4120) diff --git a/src/databricks/sql/backend/databricks_client.py b/src/databricks/sql/backend/databricks_client.py index b772e7ddd..5f31c1cac 100644 --- a/src/databricks/sql/backend/databricks_client.py +++ b/src/databricks/sql/backend/databricks_client.py @@ -248,8 +248,10 @@ def get_schemas( max_rows: Maximum number of rows to fetch in a single batch max_bytes: Maximum number of bytes to fetch in a single batch cursor: The cursor object that will handle the results - catalog_name: Optional catalog name pattern to filter by - schema_name: Optional schema name pattern to filter by + catalog_name: Optional catalog name pattern to filter by. ``None`` + leaves the filter unset; an empty string matches nothing. + schema_name: Optional schema name pattern to filter by. ``None`` + leaves the filter unset; an empty string matches nothing. Returns: ResultSet: An object containing the schema metadata @@ -284,10 +286,13 @@ def get_tables( max_bytes: Maximum number of bytes to fetch in a single batch cursor: The cursor object that will handle the results catalog_name: Optional catalog name pattern to filter by - if catalog_name is None, we fetch across all catalogs + if catalog_name is None, we fetch across all catalogs; an empty + string matches nothing schema_name: Optional schema name pattern to filter by - if schema_name is None, we fetch across all schemas - table_name: Optional table name pattern to filter by + if schema_name is None, we fetch across all schemas; an empty + string matches nothing + table_name: Optional table name pattern to filter by. ``None`` + leaves the filter unset; an empty string matches nothing. table_types: Optional list of table types to filter by (e.g., ['TABLE', 'VIEW']) Returns: @@ -322,11 +327,15 @@ def get_columns( max_rows: Maximum number of rows to fetch in a single batch max_bytes: Maximum number of bytes to fetch in a single batch cursor: The cursor object that will handle the results - catalog_name: Optional catalog name pattern to filter by - schema_name: Optional schema name pattern to filter by + catalog_name: Optional catalog name pattern to filter by. ``None`` + leaves the filter unset; an empty string matches nothing. + schema_name: Optional schema name pattern to filter by. ``None`` + leaves the filter unset; an empty string matches nothing. table_name: Optional table name pattern to filter by - if table_name is None, we fetch across all tables - column_name: Optional column name pattern to filter by + if table_name is None, we fetch across all tables; an empty + string matches nothing + column_name: Optional column name pattern to filter by. ``None`` + leaves the filter unset; an empty string matches nothing. Returns: ResultSet: An object containing the column metadata diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 93b0a98a4..ac74e9e8d 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -117,35 +117,17 @@ def _is_not_found(exc: BaseException) -> bool: ) -def _none_if_blank(value: Optional[str]) -> Optional[str]: - """Map an empty/whitespace-only metadata filter to ``None`` - ("match all"), matching the Thrift backend's effective behaviour. - - The kernel's ``Identifier`` / ``LikePattern`` reject ``""`` with - ``InvalidArgument`` (-> ``ProgrammingError``); ``None`` is the - kernel's canonical "match all". Applied to schema / table / column - *pattern* args (which otherwise keep ``%`` / ``_`` as real LIKE - wildcards).""" - if value is None: - return None - return value if value.strip() else None - - def _catalog_or_none(value: Optional[str]) -> Optional[str]: - """Normalise a catalog filter: ``None`` / blank / ``'%'`` / ``'*'`` - all mean "all catalogs" -> ``None``. + """Map all-catalog wildcards to the kernel's unfiltered representation. This makes ``columns(catalog='%')`` behave like ``tables(catalog='%')`` / ``schemas(catalog='%')`` — the kernel - already treats blank/``%``/``*`` as "all catalogs" for SHOW SCHEMAS - / SHOW TABLES (``is_null_or_wildcard``) but treats the catalog as an - exact identifier for SHOW COLUMNS, so the three diverged. Normalising - connector-side makes them symmetric. This intentionally diverges from - raw-Thrift literalness (Thrift treats ``%`` as a literal catalog - name) in favour of JDBC "catalog is exact-or-all, not a pattern" + - internal consistency. Catalog is the only arg normalised this way; - schema/table/column patterns keep ``%`` / ``*`` as LIKE wildcards.""" - if value is None or not value.strip() or value in ("%", "*"): + treats the catalog as an exact identifier for SHOW COLUMNS. ``None`` + remains the only absent filter. Other strings must be preserved as real + filters; in particular, an empty string matches nothing just as it does + on the Thrift backend. + """ + if value is None or value in ("%", "*"): return None return value @@ -938,7 +920,7 @@ def get_schemas( try: stream = self._kernel_session.metadata().list_schemas( catalog=_catalog_or_none(catalog_name), - schema_pattern=_none_if_blank(schema_name), + schema_pattern=schema_name, ) return self._make_result_set(stream, cursor, self._synthetic_command_id()) except Exception as exc: @@ -965,8 +947,8 @@ def get_tables( # through preserves streaming for large schemas. stream = self._kernel_session.metadata().list_tables( catalog=_catalog_or_none(catalog_name), - schema_pattern=_none_if_blank(schema_name), - table_pattern=_none_if_blank(table_name), + schema_pattern=schema_name, + table_pattern=table_name, table_types=table_types if table_types else None, ) return self._make_result_set(stream, cursor, self._synthetic_command_id()) @@ -995,9 +977,9 @@ def get_columns( # the user's perspective. stream = self._kernel_session.metadata().list_columns( catalog=_catalog_or_none(catalog_name), - schema_pattern=_none_if_blank(schema_name), - table_pattern=_none_if_blank(table_name), - column_pattern=_none_if_blank(column_name), + schema_pattern=schema_name, + table_pattern=table_name, + column_pattern=column_name, ) return self._make_result_set(stream, cursor, self._synthetic_command_id()) except Exception as exc: diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 44895954f..634b3f966 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -1577,7 +1577,8 @@ def schemas( """ Get schemas corresponding to the catalog_name and schema_name. - Names can contain % wildcards. + ``None`` leaves a filter unset. An empty string is a real empty + pattern and matches nothing. Names can contain % wildcards. :returns self """ self._check_not_closed() @@ -1603,7 +1604,8 @@ def tables( """ Get tables corresponding to the catalog_name, schema_name and table_name. - Names can contain % wildcards. + ``None`` leaves a filter unset. An empty string is a real empty + pattern and matches nothing. Names can contain % wildcards. :returns self """ self._check_not_closed() @@ -1632,7 +1634,8 @@ def columns( """ Get columns corresponding to the catalog_name, schema_name, table_name and column_name. - Names can contain % wildcards. + ``None`` leaves a filter unset. An empty string is a real empty + pattern and matches nothing. Names can contain % wildcards. ``catalog_name=None`` is accepted on all backends and matches columns across every catalog (the kernel issues ``SHOW COLUMNS`` diff --git a/tests/e2e/test_kernel_backend.py b/tests/e2e/test_kernel_backend.py index acae90819..b45add843 100644 --- a/tests/e2e/test_kernel_backend.py +++ b/tests/e2e/test_kernel_backend.py @@ -356,17 +356,31 @@ def test_metadata_columns(conn): assert len(rows) > 0 -# ── Metadata filter normalization (batch 3) ─────────────────────── +# ── Metadata filter semantics ───────────────────────────────────── -def test_schemas_with_empty_string_filter_matches_all(conn): - """An empty-string schema pattern normalizes to match-all rather - than raising ``ProgrammingError`` (kernel rejects ``""``) — locks - ``_none_if_blank`` on the pattern args.""" +def test_schemas_with_empty_string_filter_matches_nothing(conn): + """An empty string is a real pattern, distinct from absent ``None``.""" with conn.cursor() as cur: cur.schemas(catalog_name="main", schema_name="") - rows = cur.fetchall() - assert len(rows) > 0 + assert cur.fetchall() == [] + + +@pytest.mark.parametrize( + "empty_filter", ["catalog_name", "schema_name", "table_name", "column_name"] +) +def test_columns_with_empty_string_filter_matches_nothing(conn, empty_filter): + filters = { + "catalog_name": "system", + "schema_name": "information_schema", + "table_name": "tables", + "column_name": "table_catalog", + } + filters[empty_filter] = "" + + with conn.cursor() as cur: + cur.columns(**filters) + assert cur.fetchall() == [] def test_tables_table_types_filter_is_case_insensitive(conn): diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 3eb5a9006..7aa522d9f 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -1560,15 +1560,13 @@ def test_sync_execute_leaves_rowcount_default_when_num_modified_rows_none(): # --------------------------------------------------------------------------- -# Metadata filter normalization — wildcard catalog + empty-string patterns +# Metadata filter semantics # --------------------------------------------------------------------------- -@pytest.mark.parametrize("wildcard", ["%", "*", "", " "]) +@pytest.mark.parametrize("wildcard", ["%", "*"]) def test_get_columns_normalizes_wildcard_catalog_to_none(wildcard): - """``catalog_name`` of ``%``/``*``/blank → ``None`` (all-catalogs), - matching JDBC exact-or-all semantics and keeping the three metadata - methods symmetric.""" + """The kernel represents the supported all-catalog wildcards as ``None``.""" c = _make_client() c._kernel_session = MagicMock() list_columns = c._kernel_session.metadata.return_value.list_columns @@ -1596,10 +1594,8 @@ def test_get_columns_normalizes_wildcard_catalog_to_none(wildcard): ) -def test_get_schemas_normalizes_blank_pattern_to_none(): - """An empty-string schema pattern → ``None`` (match-all), mapping - the kernel's ``InvalidArgument``-on-``""`` to Thrift's effective - match-all. ``%``/``*`` stay as real LIKE wildcards on patterns.""" +def test_get_schemas_preserves_empty_pattern(): + """An empty pattern is distinct from the absent ``None`` filter.""" c = _make_client() c._kernel_session = MagicMock() list_schemas = c._kernel_session.metadata.return_value.list_schemas @@ -1617,7 +1613,64 @@ def test_get_schemas_normalizes_blank_pattern_to_none(): schema_name="", ) - list_schemas.assert_called_once_with(catalog="main", schema_pattern=None) + list_schemas.assert_called_once_with(catalog="main", schema_pattern="") + + +def test_get_tables_preserves_empty_patterns(): + c = _make_client() + c._kernel_session = MagicMock() + list_tables = c._kernel_session.metadata.return_value.list_tables + list_tables.return_value = _stream_with_schema() + cursor = MagicMock() + cursor.arraysize = 100 + cursor.buffer_size_bytes = 1024 + + c.get_tables( + session_id=MagicMock(), + max_rows=1, + max_bytes=1, + cursor=cursor, + catalog_name="", + schema_name="", + table_name="", + ) + + list_tables.assert_called_once_with( + catalog="", + schema_pattern="", + table_pattern="", + table_types=None, + ) + + +@pytest.mark.parametrize("filter_value", ["", " "]) +def test_get_columns_preserves_blank_filters(filter_value): + """Blank strings remain filters instead of becoming match-all ``None``.""" + c = _make_client() + c._kernel_session = MagicMock() + list_columns = c._kernel_session.metadata.return_value.list_columns + list_columns.return_value = _stream_with_schema() + cursor = MagicMock() + cursor.arraysize = 100 + cursor.buffer_size_bytes = 1024 + + c.get_columns( + session_id=MagicMock(), + max_rows=1, + max_bytes=1, + cursor=cursor, + catalog_name=filter_value, + schema_name=filter_value, + table_name=filter_value, + column_name=filter_value, + ) + + list_columns.assert_called_once_with( + catalog=filter_value, + schema_pattern=filter_value, + table_pattern=filter_value, + column_pattern=filter_value, + ) def test_get_schemas_keeps_wildcard_pattern(): From 74d883fe448b0355bcb65a4c982ffe2953c17e28 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 26 Aug 2026 04:56:31 +0000 Subject: [PATCH 2/4] fix(kernel): handle empty exact catalog filters Signed-off-by: Vu Anh Phung --- src/databricks/sql/backend/kernel/client.py | 35 ++++++-- tests/unit/test_kernel_client.py | 99 ++++++++++++++++++--- 2 files changed, 117 insertions(+), 17 deletions(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index ac74e9e8d..fcdcb101e 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -25,7 +25,7 @@ import logging import threading import uuid -from typing import Any, Dict, List, Optional, Set, TYPE_CHECKING, Union +from typing import Any, Dict, List, Optional, Set, Tuple, TYPE_CHECKING, Union from databricks.sql.backend.databricks_client import DatabricksClient from databricks.sql.backend.kernel._errors import ( @@ -125,13 +125,30 @@ def _catalog_or_none(value: Optional[str]) -> Optional[str]: treats the catalog as an exact identifier for SHOW COLUMNS. ``None`` remains the only absent filter. Other strings must be preserved as real filters; in particular, an empty string matches nothing just as it does - on the Thrift backend. + on the Thrift backend. Non-empty whitespace-only strings remain invalid + and are left for kernel validation. """ if value is None or value in ("%", "*"): return None return value +def _exact_catalog_and_pattern( + catalog: Optional[str], pattern: Optional[str] +) -> Tuple[Optional[str], Optional[str]]: + """Adapt an empty exact catalog to an empty subordinate pattern. + + At ``KERNEL_REV``, ``Identifier("")`` is invalid while + ``LikePattern("")`` means match-nothing. Schema and column metadata + take an exact catalog, so represent an empty catalog as all catalogs + constrained by an empty schema pattern. This preserves empty-catalog + semantics without passing an invalid identifier to the kernel. + """ + if catalog == "": + return None, "" + return _catalog_or_none(catalog), pattern + + def _is_staging_statement(operation: str) -> bool: """True iff ``operation`` is a volume/staging statement (PUT / GET / REMOVE). @@ -918,9 +935,12 @@ def get_schemas( if self._kernel_session is None: raise InterfaceError("get_schemas requires an open session.") try: + catalog, schema_pattern = _exact_catalog_and_pattern( + catalog_name, schema_name + ) stream = self._kernel_session.metadata().list_schemas( - catalog=_catalog_or_none(catalog_name), - schema_pattern=schema_name, + catalog=catalog, + schema_pattern=schema_pattern, ) return self._make_result_set(stream, cursor, self._synthetic_command_id()) except Exception as exc: @@ -975,9 +995,12 @@ def get_columns( # row's `TABLE_CAT` is correctly attributed. Matches the # Thrift backend's `getColumns(null, …)` behaviour from # the user's perspective. + catalog, schema_pattern = _exact_catalog_and_pattern( + catalog_name, schema_name + ) stream = self._kernel_session.metadata().list_columns( - catalog=_catalog_or_none(catalog_name), - schema_pattern=schema_name, + catalog=catalog, + schema_pattern=schema_pattern, table_pattern=table_name, column_pattern=column_name, ) diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 7aa522d9f..084a962b0 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -1616,6 +1616,28 @@ def test_get_schemas_preserves_empty_pattern(): list_schemas.assert_called_once_with(catalog="main", schema_pattern="") +def test_get_schemas_empty_catalog_uses_empty_pattern(): + """Avoid passing an empty exact ``Identifier`` to the kernel.""" + c = _make_client() + c._kernel_session = MagicMock() + list_schemas = c._kernel_session.metadata.return_value.list_schemas + list_schemas.return_value = _stream_with_schema() + cursor = MagicMock() + cursor.arraysize = 100 + cursor.buffer_size_bytes = 1024 + + c.get_schemas( + session_id=MagicMock(), + max_rows=1, + max_bytes=1, + cursor=cursor, + catalog_name="", + schema_name="ignored", + ) + + list_schemas.assert_called_once_with(catalog=None, schema_pattern="") + + def test_get_tables_preserves_empty_patterns(): c = _make_client() c._kernel_session = MagicMock() @@ -1643,9 +1665,64 @@ def test_get_tables_preserves_empty_patterns(): ) -@pytest.mark.parametrize("filter_value", ["", " "]) -def test_get_columns_preserves_blank_filters(filter_value): - """Blank strings remain filters instead of becoming match-all ``None``.""" +def test_get_columns_preserves_empty_patterns(): + c = _make_client() + c._kernel_session = MagicMock() + list_columns = c._kernel_session.metadata.return_value.list_columns + list_columns.return_value = _stream_with_schema() + cursor = MagicMock() + cursor.arraysize = 100 + cursor.buffer_size_bytes = 1024 + + c.get_columns( + session_id=MagicMock(), + max_rows=1, + max_bytes=1, + cursor=cursor, + catalog_name="main", + schema_name="", + table_name="", + column_name="", + ) + + list_columns.assert_called_once_with( + catalog="main", + schema_pattern="", + table_pattern="", + column_pattern="", + ) + + +def test_get_columns_empty_catalog_uses_empty_pattern(): + """An empty catalog matches nothing without constructing ``Identifier("")``.""" + c = _make_client() + c._kernel_session = MagicMock() + list_columns = c._kernel_session.metadata.return_value.list_columns + list_columns.return_value = _stream_with_schema() + cursor = MagicMock() + cursor.arraysize = 100 + cursor.buffer_size_bytes = 1024 + + c.get_columns( + session_id=MagicMock(), + max_rows=1, + max_bytes=1, + cursor=cursor, + catalog_name="", + schema_name="ignored", + table_name="table", + column_name="column", + ) + + list_columns.assert_called_once_with( + catalog=None, + schema_pattern="", + table_pattern="table", + column_pattern="column", + ) + + +def test_get_columns_preserves_whitespace_for_kernel_validation(): c = _make_client() c._kernel_session = MagicMock() list_columns = c._kernel_session.metadata.return_value.list_columns @@ -1659,17 +1736,17 @@ def test_get_columns_preserves_blank_filters(filter_value): max_rows=1, max_bytes=1, cursor=cursor, - catalog_name=filter_value, - schema_name=filter_value, - table_name=filter_value, - column_name=filter_value, + catalog_name=" ", + schema_name=" ", + table_name=" ", + column_name=" ", ) list_columns.assert_called_once_with( - catalog=filter_value, - schema_pattern=filter_value, - table_pattern=filter_value, - column_pattern=filter_value, + catalog=" ", + schema_pattern=" ", + table_pattern=" ", + column_pattern=" ", ) From 765a95b766545ae14c6d2b3d488e4d87e34b46a9 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 26 Aug 2026 05:22:03 +0000 Subject: [PATCH 3/4] refactor(kernel): simplify metadata filter forwarding --- CHANGELOG.md | 2 +- .../sql/backend/databricks_client.py | 4 +-- src/databricks/sql/backend/kernel/client.py | 29 ++----------------- src/databricks/sql/client.py | 12 ++++---- tests/unit/test_kernel_client.py | 9 +++--- 5 files changed, 16 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 518f65831..5dbbd2995 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History # Unreleased -- Kernel metadata filters now preserve empty strings as empty patterns, matching no catalogs, schemas, tables, or columns. Only `None` leaves a filter unset; `%` and `*` retain their existing wildcard behavior (PECOBLR-4221). +- Kernel metadata filters now preserve empty strings, matching no catalogs, schemas, tables, or columns. Only `None` leaves a filter unset; other values retain their exact-identifier or pattern semantics (PECOBLR-4221). - Kernel backend (`use_kernel=True`): OAuth **M2M with a JWT private-key client assertion** (RFC 7523) is now supported. Pass `oauth_client_id` + `oauth_jwt_key_file` + `oauth_jwt_kid` (with optional `oauth_jwt_passphrase` for an encrypted PKCS#8 key, `oauth_jwt_algorithm` defaulting to `RS256`, `oauth_scopes`, and `token_url` for the IdP token endpoint) and the connector routes them to the kernel's `auth_type="oauth-m2m-jwt"`, which signs a short-lived assertion with the private key instead of sending a client secret. The kernel owns the token lifecycle. A private-key file is treated as unambiguous JWT M2M intent and is mutually exclusive with `oauth_client_secret` / `credentials_provider` (both raise `NotSupportedError`). Verified end-to-end against an Azure Databricks workspace with the service principal's public certificate registered on its Entra ID app registration. Requires `databricks-sql-kernel >= 0.2.0` with JWT support. - Kernel backend (`use_kernel=True`): OAuth U2M with `auth_type="databricks-oauth"` now forwards the connector's `databricks-sql-python` OAuth-app bundle (`client_id` + `sql offline_access` scopes + redirect port) into the kernel, so a bare U2M connection authenticates as `databricks-sql-python` — parity with the Thrift path — instead of inheriting the kernel's own `databricks-sql-connector` default. A caller-supplied `oauth_client_id` (with its coupled `oauth_redirect_port`) is honored, as is a caller-supplied `oauth_scopes`; absent one, the connector default (`sql offline_access`) is forwarded. Note: the kernel binds a single U2M redirect port, so unlike the Thrift path (which tries the full `8020..8024` range) the kernel path uses only one port and does not fall back to the next port if it is already bound — pass `oauth_redirect_port` (with `oauth_client_id`) to pick a free one on a port collision (PECOBLR-4040) - Kernel backend (`use_kernel=True`): **Azure Entra (Azure AD) service-principal M2M is now supported.** `auth_type="azure-sp-m2m"` forwards `azure_client_id` / `azure_client_secret`; the kernel is the Azure-aware auth core — it builds the Entra v2.0 token endpoint and the `{app_id}/.default` scope, and **auto-discovers the tenant** from the workspace's `/aad/auth` redirect when `azure_tenant_id` is omitted (matching Thrift). The `Authorization` bearer is the Databricks-audience data token, which alone authenticates a workspace-member SP. Set `azure_workspace_resource_id` and the kernel also sends the Azure SP management token (`X-Databricks-Azure-SP-Management-Token`) + `X-Databricks-Azure-Workspace-Resource-Id` header (matching the JDBC driver), so a service principal with an Azure RBAC role but no workspace membership can authenticate; omit it and no ARM management-scope token is fetched. Azure AD **U2M** (`auth_type="azure-oauth"`) now routes to the kernel's OAuth U2M flow, identically to `auth_type="databricks-oauth"`: the kernel runs the in-house workspace-federated browser flow, which Azure workspaces support (the workspace federates login to Entra). It forwards the connector's `databricks-sql-python` OAuth app, not the Thrift Azure app (`96eecda7` / port 8030), which is registered for Thrift's direct-Entra flow the kernel does not perform (PECOBLR-4141; PECOBLR-4120) diff --git a/src/databricks/sql/backend/databricks_client.py b/src/databricks/sql/backend/databricks_client.py index 5f31c1cac..37bcd9197 100644 --- a/src/databricks/sql/backend/databricks_client.py +++ b/src/databricks/sql/backend/databricks_client.py @@ -248,7 +248,7 @@ def get_schemas( max_rows: Maximum number of rows to fetch in a single batch max_bytes: Maximum number of bytes to fetch in a single batch cursor: The cursor object that will handle the results - catalog_name: Optional catalog name pattern to filter by. ``None`` + catalog_name: Optional exact catalog name to filter by. ``None`` leaves the filter unset; an empty string matches nothing. schema_name: Optional schema name pattern to filter by. ``None`` leaves the filter unset; an empty string matches nothing. @@ -327,7 +327,7 @@ def get_columns( max_rows: Maximum number of rows to fetch in a single batch max_bytes: Maximum number of bytes to fetch in a single batch cursor: The cursor object that will handle the results - catalog_name: Optional catalog name pattern to filter by. ``None`` + catalog_name: Optional exact catalog name to filter by. ``None`` leaves the filter unset; an empty string matches nothing. schema_name: Optional schema name pattern to filter by. ``None`` leaves the filter unset; an empty string matches nothing. diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index fcdcb101e..64781ef2a 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -117,36 +117,13 @@ def _is_not_found(exc: BaseException) -> bool: ) -def _catalog_or_none(value: Optional[str]) -> Optional[str]: - """Map all-catalog wildcards to the kernel's unfiltered representation. - - This makes ``columns(catalog='%')`` behave like - ``tables(catalog='%')`` / ``schemas(catalog='%')`` — the kernel - treats the catalog as an exact identifier for SHOW COLUMNS. ``None`` - remains the only absent filter. Other strings must be preserved as real - filters; in particular, an empty string matches nothing just as it does - on the Thrift backend. Non-empty whitespace-only strings remain invalid - and are left for kernel validation. - """ - if value is None or value in ("%", "*"): - return None - return value - - def _exact_catalog_and_pattern( catalog: Optional[str], pattern: Optional[str] ) -> Tuple[Optional[str], Optional[str]]: - """Adapt an empty exact catalog to an empty subordinate pattern. - - At ``KERNEL_REV``, ``Identifier("")`` is invalid while - ``LikePattern("")`` means match-nothing. Schema and column metadata - take an exact catalog, so represent an empty catalog as all catalogs - constrained by an empty schema pattern. This preserves empty-catalog - semantics without passing an invalid identifier to the kernel. - """ + """Avoid constructing the kernel's invalid empty ``Identifier``.""" if catalog == "": return None, "" - return _catalog_or_none(catalog), pattern + return catalog, pattern def _is_staging_statement(operation: str) -> bool: @@ -966,7 +943,7 @@ def get_tables( # do the work — no connector-side drain + refilter. Passing it # through preserves streaming for large schemas. stream = self._kernel_session.metadata().list_tables( - catalog=_catalog_or_none(catalog_name), + catalog=catalog_name, schema_pattern=schema_name, table_pattern=table_name, table_types=table_types if table_types else None, diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 634b3f966..e6c88afcd 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -1577,8 +1577,8 @@ def schemas( """ Get schemas corresponding to the catalog_name and schema_name. - ``None`` leaves a filter unset. An empty string is a real empty - pattern and matches nothing. Names can contain % wildcards. + ``None`` leaves a filter unset and an empty string matches nothing. + ``catalog_name`` is exact; ``schema_name`` can contain % wildcards. :returns self """ self._check_not_closed() @@ -1604,8 +1604,8 @@ def tables( """ Get tables corresponding to the catalog_name, schema_name and table_name. - ``None`` leaves a filter unset. An empty string is a real empty - pattern and matches nothing. Names can contain % wildcards. + ``None`` leaves a filter unset and an empty string matches nothing. + Names can contain % wildcards. :returns self """ self._check_not_closed() @@ -1634,8 +1634,8 @@ def columns( """ Get columns corresponding to the catalog_name, schema_name, table_name and column_name. - ``None`` leaves a filter unset. An empty string is a real empty - pattern and matches nothing. Names can contain % wildcards. + ``None`` leaves a filter unset and an empty string matches nothing. + ``catalog_name`` is exact; other names can contain % wildcards. ``catalog_name=None`` is accepted on all backends and matches columns across every catalog (the kernel issues ``SHOW COLUMNS`` diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 084a962b0..9c36b3ae5 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -1564,9 +1564,8 @@ def test_sync_execute_leaves_rowcount_default_when_num_modified_rows_none(): # --------------------------------------------------------------------------- -@pytest.mark.parametrize("wildcard", ["%", "*"]) -def test_get_columns_normalizes_wildcard_catalog_to_none(wildcard): - """The kernel represents the supported all-catalog wildcards as ``None``.""" +@pytest.mark.parametrize("exact_catalog", ["%", "*"]) +def test_get_columns_preserves_exact_catalog(exact_catalog): c = _make_client() c._kernel_session = MagicMock() list_columns = c._kernel_session.metadata.return_value.list_columns @@ -1580,14 +1579,14 @@ def test_get_columns_normalizes_wildcard_catalog_to_none(wildcard): max_rows=1, max_bytes=1, cursor=cursor, - catalog_name=wildcard, + catalog_name=exact_catalog, schema_name="s", table_name="t", column_name="c", ) list_columns.assert_called_once_with( - catalog=None, + catalog=exact_catalog, schema_pattern="s", table_pattern="t", column_pattern="c", From 96c3ba074006fd81ce2710633429af83cecacd33 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 26 Aug 2026 05:41:13 +0000 Subject: [PATCH 4/4] refactor(kernel): forward metadata filters unchanged --- CHANGELOG.md | 2 +- .../sql/backend/databricks_client.py | 8 +++--- src/databricks/sql/backend/kernel/client.py | 25 ++++--------------- src/databricks/sql/client.py | 15 ++++++----- tests/e2e/test_kernel_backend.py | 2 +- tests/unit/test_kernel_client.py | 12 ++++----- 6 files changed, 25 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dbbd2995..743f54daa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History # Unreleased -- Kernel metadata filters now preserve empty strings, matching no catalogs, schemas, tables, or columns. Only `None` leaves a filter unset; other values retain their exact-identifier or pattern semantics (PECOBLR-4221). +- Kernel metadata filters are now forwarded unchanged instead of collapsing empty strings to `None`. Only `None` leaves a filter unset; empty pattern filters match nothing (PECOBLR-4221). - Kernel backend (`use_kernel=True`): OAuth **M2M with a JWT private-key client assertion** (RFC 7523) is now supported. Pass `oauth_client_id` + `oauth_jwt_key_file` + `oauth_jwt_kid` (with optional `oauth_jwt_passphrase` for an encrypted PKCS#8 key, `oauth_jwt_algorithm` defaulting to `RS256`, `oauth_scopes`, and `token_url` for the IdP token endpoint) and the connector routes them to the kernel's `auth_type="oauth-m2m-jwt"`, which signs a short-lived assertion with the private key instead of sending a client secret. The kernel owns the token lifecycle. A private-key file is treated as unambiguous JWT M2M intent and is mutually exclusive with `oauth_client_secret` / `credentials_provider` (both raise `NotSupportedError`). Verified end-to-end against an Azure Databricks workspace with the service principal's public certificate registered on its Entra ID app registration. Requires `databricks-sql-kernel >= 0.2.0` with JWT support. - Kernel backend (`use_kernel=True`): OAuth U2M with `auth_type="databricks-oauth"` now forwards the connector's `databricks-sql-python` OAuth-app bundle (`client_id` + `sql offline_access` scopes + redirect port) into the kernel, so a bare U2M connection authenticates as `databricks-sql-python` — parity with the Thrift path — instead of inheriting the kernel's own `databricks-sql-connector` default. A caller-supplied `oauth_client_id` (with its coupled `oauth_redirect_port`) is honored, as is a caller-supplied `oauth_scopes`; absent one, the connector default (`sql offline_access`) is forwarded. Note: the kernel binds a single U2M redirect port, so unlike the Thrift path (which tries the full `8020..8024` range) the kernel path uses only one port and does not fall back to the next port if it is already bound — pass `oauth_redirect_port` (with `oauth_client_id`) to pick a free one on a port collision (PECOBLR-4040) - Kernel backend (`use_kernel=True`): **Azure Entra (Azure AD) service-principal M2M is now supported.** `auth_type="azure-sp-m2m"` forwards `azure_client_id` / `azure_client_secret`; the kernel is the Azure-aware auth core — it builds the Entra v2.0 token endpoint and the `{app_id}/.default` scope, and **auto-discovers the tenant** from the workspace's `/aad/auth` redirect when `azure_tenant_id` is omitted (matching Thrift). The `Authorization` bearer is the Databricks-audience data token, which alone authenticates a workspace-member SP. Set `azure_workspace_resource_id` and the kernel also sends the Azure SP management token (`X-Databricks-Azure-SP-Management-Token`) + `X-Databricks-Azure-Workspace-Resource-Id` header (matching the JDBC driver), so a service principal with an Azure RBAC role but no workspace membership can authenticate; omit it and no ARM management-scope token is fetched. Azure AD **U2M** (`auth_type="azure-oauth"`) now routes to the kernel's OAuth U2M flow, identically to `auth_type="databricks-oauth"`: the kernel runs the in-house workspace-federated browser flow, which Azure workspaces support (the workspace federates login to Entra). It forwards the connector's `databricks-sql-python` OAuth app, not the Thrift Azure app (`96eecda7` / port 8030), which is registered for Thrift's direct-Entra flow the kernel does not perform (PECOBLR-4141; PECOBLR-4120) diff --git a/src/databricks/sql/backend/databricks_client.py b/src/databricks/sql/backend/databricks_client.py index 37bcd9197..523add8e6 100644 --- a/src/databricks/sql/backend/databricks_client.py +++ b/src/databricks/sql/backend/databricks_client.py @@ -248,8 +248,8 @@ def get_schemas( max_rows: Maximum number of rows to fetch in a single batch max_bytes: Maximum number of bytes to fetch in a single batch cursor: The cursor object that will handle the results - catalog_name: Optional exact catalog name to filter by. ``None`` - leaves the filter unset; an empty string matches nothing. + catalog_name: Optional exact catalog name to filter by, forwarded + unchanged. ``None`` leaves the filter unset. schema_name: Optional schema name pattern to filter by. ``None`` leaves the filter unset; an empty string matches nothing. @@ -327,8 +327,8 @@ def get_columns( max_rows: Maximum number of rows to fetch in a single batch max_bytes: Maximum number of bytes to fetch in a single batch cursor: The cursor object that will handle the results - catalog_name: Optional exact catalog name to filter by. ``None`` - leaves the filter unset; an empty string matches nothing. + catalog_name: Optional exact catalog name to filter by, forwarded + unchanged. ``None`` leaves the filter unset. schema_name: Optional schema name pattern to filter by. ``None`` leaves the filter unset; an empty string matches nothing. table_name: Optional table name pattern to filter by diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 64781ef2a..a10c2e5c0 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -25,7 +25,7 @@ import logging import threading import uuid -from typing import Any, Dict, List, Optional, Set, Tuple, TYPE_CHECKING, Union +from typing import Any, Dict, List, Optional, Set, TYPE_CHECKING, Union from databricks.sql.backend.databricks_client import DatabricksClient from databricks.sql.backend.kernel._errors import ( @@ -117,15 +117,6 @@ def _is_not_found(exc: BaseException) -> bool: ) -def _exact_catalog_and_pattern( - catalog: Optional[str], pattern: Optional[str] -) -> Tuple[Optional[str], Optional[str]]: - """Avoid constructing the kernel's invalid empty ``Identifier``.""" - if catalog == "": - return None, "" - return catalog, pattern - - def _is_staging_statement(operation: str) -> bool: """True iff ``operation`` is a volume/staging statement (PUT / GET / REMOVE). @@ -912,12 +903,9 @@ def get_schemas( if self._kernel_session is None: raise InterfaceError("get_schemas requires an open session.") try: - catalog, schema_pattern = _exact_catalog_and_pattern( - catalog_name, schema_name - ) stream = self._kernel_session.metadata().list_schemas( - catalog=catalog, - schema_pattern=schema_pattern, + catalog=catalog_name, + schema_pattern=schema_name, ) return self._make_result_set(stream, cursor, self._synthetic_command_id()) except Exception as exc: @@ -972,12 +960,9 @@ def get_columns( # row's `TABLE_CAT` is correctly attributed. Matches the # Thrift backend's `getColumns(null, …)` behaviour from # the user's perspective. - catalog, schema_pattern = _exact_catalog_and_pattern( - catalog_name, schema_name - ) stream = self._kernel_session.metadata().list_columns( - catalog=catalog, - schema_pattern=schema_pattern, + catalog=catalog_name, + schema_pattern=schema_name, table_pattern=table_name, column_pattern=column_name, ) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index e6c88afcd..9e093cef1 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -1577,8 +1577,9 @@ def schemas( """ Get schemas corresponding to the catalog_name and schema_name. - ``None`` leaves a filter unset and an empty string matches nothing. - ``catalog_name`` is exact; ``schema_name`` can contain % wildcards. + Filters are forwarded unchanged; only ``None`` leaves one unset. + ``catalog_name`` is exact. ``schema_name`` is a pattern, can contain + % wildcards, and an empty pattern matches nothing. :returns self """ self._check_not_closed() @@ -1604,8 +1605,9 @@ def tables( """ Get tables corresponding to the catalog_name, schema_name and table_name. - ``None`` leaves a filter unset and an empty string matches nothing. - Names can contain % wildcards. + Filters are forwarded unchanged; only ``None`` leaves one unset. + Names are patterns, can contain % wildcards, and empty patterns match + nothing. :returns self """ self._check_not_closed() @@ -1634,8 +1636,9 @@ def columns( """ Get columns corresponding to the catalog_name, schema_name, table_name and column_name. - ``None`` leaves a filter unset and an empty string matches nothing. - ``catalog_name`` is exact; other names can contain % wildcards. + Filters are forwarded unchanged; only ``None`` leaves one unset. + ``catalog_name`` is exact. Other names are patterns, can contain + % wildcards, and empty patterns match nothing. ``catalog_name=None`` is accepted on all backends and matches columns across every catalog (the kernel issues ``SHOW COLUMNS`` diff --git a/tests/e2e/test_kernel_backend.py b/tests/e2e/test_kernel_backend.py index b45add843..0b767e1c3 100644 --- a/tests/e2e/test_kernel_backend.py +++ b/tests/e2e/test_kernel_backend.py @@ -367,7 +367,7 @@ def test_schemas_with_empty_string_filter_matches_nothing(conn): @pytest.mark.parametrize( - "empty_filter", ["catalog_name", "schema_name", "table_name", "column_name"] + "empty_filter", ["schema_name", "table_name", "column_name"] ) def test_columns_with_empty_string_filter_matches_nothing(conn, empty_filter): filters = { diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 9c36b3ae5..5d4149d24 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -1615,8 +1615,7 @@ def test_get_schemas_preserves_empty_pattern(): list_schemas.assert_called_once_with(catalog="main", schema_pattern="") -def test_get_schemas_empty_catalog_uses_empty_pattern(): - """Avoid passing an empty exact ``Identifier`` to the kernel.""" +def test_get_schemas_preserves_empty_catalog(): c = _make_client() c._kernel_session = MagicMock() list_schemas = c._kernel_session.metadata.return_value.list_schemas @@ -1634,7 +1633,7 @@ def test_get_schemas_empty_catalog_uses_empty_pattern(): schema_name="ignored", ) - list_schemas.assert_called_once_with(catalog=None, schema_pattern="") + list_schemas.assert_called_once_with(catalog="", schema_pattern="ignored") def test_get_tables_preserves_empty_patterns(): @@ -1692,8 +1691,7 @@ def test_get_columns_preserves_empty_patterns(): ) -def test_get_columns_empty_catalog_uses_empty_pattern(): - """An empty catalog matches nothing without constructing ``Identifier("")``.""" +def test_get_columns_preserves_empty_catalog(): c = _make_client() c._kernel_session = MagicMock() list_columns = c._kernel_session.metadata.return_value.list_columns @@ -1714,8 +1712,8 @@ def test_get_columns_empty_catalog_uses_empty_pattern(): ) list_columns.assert_called_once_with( - catalog=None, - schema_pattern="", + catalog="", + schema_pattern="ignored", table_pattern="table", column_pattern="column", )