From 78099d60345ff13b1794915785367e3c470f4d0e Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Tue, 10 Nov 2026 23:37:47 +0000 Subject: [PATCH 01/13] fix(kernel): honor use_cloud_fetch when opening sessions Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/client.py | 3 ++ src/databricks/sql/session.py | 1 + tests/unit/test_kernel_client.py | 32 +++++++++++++++++++++ tests/unit/test_session.py | 2 ++ 4 files changed, 38 insertions(+) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 0ca7ba6a8..8e6422e7d 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -293,6 +293,7 @@ def __init__( # The kernel binding owns type and range validation. self._request_timeout_secs = kwargs.get("request_timeout_secs") self._max_connections = kwargs.get("max_connections") + self._use_cloud_fetch = kwargs.get("use_cloud_fetch", True) # Opt-in native GEOMETRY / GEOGRAPHY support. True requests # the canonical Arrow struct and surfaces as ``{"srid": int, "wkb": # bytes}`` through pyarrow; False requests WKT / EWKT strings. This is @@ -396,6 +397,8 @@ def open_session( session_conf: Optional[Dict[str, str]] = None if session_configuration: session_conf = {k: str(v) for k, v in session_configuration.items()} + if not self._use_cloud_fetch: + session_conf = {**(session_conf or {}), "can_cloud_download": "false"} # The kwarg builds run INSIDE the try so the ``finally`` scrub # below always fires — including when ``kernel_auth_kwargs`` # itself raises mid-build (e.g. an OAuth token-exchange failure diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index 365b06f8f..9856e6771 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -332,6 +332,7 @@ def _create_backend( retry_options=kernel_retry_options, request_timeout_secs=kwargs.get("_socket_timeout"), max_connections=kwargs.get("_pool_maxsize") or None, + use_cloud_fetch=kwargs.get("use_cloud_fetch", True), enable_geospatial_support=kwargs.get( "enable_geospatial_support", False ), diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index b285ce6e1..2e84249b0 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -344,6 +344,38 @@ def fake_session(**kw): assert captured.get("complex_types_as_json") is expected_flag +@pytest.mark.parametrize( + "kwargs, expected", + [ + ({}, None), + ({"use_cloud_fetch": True}, None), + ({"use_cloud_fetch": False}, "false"), + ], +) +def test_open_session_honors_cloudfetch_option(monkeypatch, kwargs, expected): + captured = {} + + def fake_session(**options): + captured.update(options) + session = MagicMock() + session.session_id = "sess-id" + return session + + monkeypatch.setattr(kernel_client._kernel, "Session", fake_session) + client = kernel_client.KernelDatabricksClient( + server_hostname="example.cloud.databricks.com", + http_path="/sql/1.0/warehouses/abc", + auth_provider=AccessTokenAuthProvider("dapi-test"), + ssl_options=None, + **kwargs, + ) + configuration = {"TIMEZONE": "UTC"} + client.open_session(configuration, catalog=None, schema=None) + assert captured["session_conf"].get("can_cloud_download") == expected + assert captured["session_conf"]["TIMEZONE"] == "UTC" + assert configuration == {"TIMEZONE": "UTC"} + + @pytest.mark.parametrize("timeout", [None, 0, 12.5]) def test_open_session_passes_request_timeout_to_kernel(monkeypatch, timeout): captured = {} diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index a9f774208..25d617af6 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -484,6 +484,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): _retry_stop_after_attempts_duration=600.0, _socket_timeout=12.5, _pool_maxsize=41, + use_cloud_fetch=False, enable_geospatial_support=False, ) try: @@ -495,6 +496,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): assert opts["retry_stop_after_attempts_duration"] == 600.0 assert kwargs["request_timeout_secs"] == 12.5 assert kwargs["max_connections"] == 41 + assert kwargs["use_cloud_fetch"] is False assert kwargs["enable_geospatial_support"] is False finally: conn.close() From af7fecf2918d1b6d6a62d3dd6378c69a371fdd4f Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Tue, 10 Nov 2026 23:42:10 +0000 Subject: [PATCH 02/13] test(kernel): cover CloudFetch setting precedence and document support Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 1 + CONNECTION_PARAMETERS.md | 4 +-- src/databricks/sql/backend/kernel/client.py | 7 +++- tests/unit/test_kernel_client.py | 38 ++++++++++++++++----- 4 files changed, 38 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fbb64b68..d9b43313d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Release History # Unreleased +- Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) # 4.9.0 (2026-11-04) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 37736ed1a..187bc4e06 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -149,7 +149,7 @@ to change without notice. | Option | Type | Thrift | Kernel | Default Value | Note | | ------------------------------------- | ------ | :----: | :----: | ------------- | ------------------------------------------------------------------------------------------------------------- | -| `use_cloud_fetch` | `bool` | ✅ | ❌ | `True` | Download large result sets in parallel from cloud storage. The kernel manages result transport internally. | +| `use_cloud_fetch` | `bool` | ✅ | ✅ | `True` | Set `False` to fetch complete results through the SQL connection without cloud-storage downloads. | | `max_download_threads` | `int` | ✅ | ❌ | `10` | Worker threads for cloud-fetch downloads. Not forwarded to the kernel. | | `enable_query_result_lz4_compression` | `bool` | ✅ | ❌ | `True` | LZ4-compress result payloads. Not forwarded; the kernel handles compression internally. | | `_disable_pandas` | `bool` | ✅ | ✅ | `False` | Skip the pandas-based Arrow→row deserialization and materialize rows directly with PyArrow. This is a **Python-side** result-conversion toggle, not a wire option: the kernel returns results as Arrow (`RecordBatch`es) and the connector runs the *same* `_convert_arrow_table` for both backends, so the flag is honored on the kernel path too. Affects only row fetches (`fetchone`/`fetchmany`/`fetchall`); the `fetch*_arrow` methods return the Arrow table unchanged regardless of this flag. | @@ -202,7 +202,7 @@ installed kernel binding. `force_enable_telemetry` remains Python-driver-only. 2. `experimental_oauth_persistence` (custom OAuth token store). 3. TLS-client-cert *authentication* (`_use_cert_as_auth`) — note the TLS *transport* options (`_tls_*`) themselves **are** honored on both backends. -4. Result-transport tuning: `use_cloud_fetch`, `max_download_threads`, +4. Result-transport tuning: `max_download_threads`, `enable_query_result_lz4_compression`. 5. Arrow-native rendering for `_use_arrow_native_decimals` / `_use_arrow_native_timestamps` (complex types **are** forwarded). diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 8e6422e7d..567b98450 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -398,7 +398,12 @@ def open_session( if session_configuration: session_conf = {k: str(v) for k, v in session_configuration.items()} if not self._use_cloud_fetch: - session_conf = {**(session_conf or {}), "can_cloud_download": "false"} + session_conf = { + k: v + for k, v in (session_conf or {}).items() + if k.lower() != "can_cloud_download" + } + session_conf["can_cloud_download"] = "false" # The kwarg builds run INSIDE the try so the ``finally`` scrub # below always fires — including when ``kernel_auth_kwargs`` # itself raises mid-build (e.g. an OAuth token-exchange failure diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 2e84249b0..150eeed51 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -345,14 +345,35 @@ def fake_session(**kw): @pytest.mark.parametrize( - "kwargs, expected", + "kwargs, configuration, expected", [ - ({}, None), - ({"use_cloud_fetch": True}, None), - ({"use_cloud_fetch": False}, "false"), + ({}, {"TIMEZONE": "UTC"}, {"TIMEZONE": "UTC"}), + ( + {"use_cloud_fetch": True}, + {"can_cloud_download": "false"}, + {"can_cloud_download": "false"}, + ), + ({"use_cloud_fetch": False}, None, {"can_cloud_download": "false"}), + ( + {"use_cloud_fetch": False}, + {"TIMEZONE": "UTC"}, + {"TIMEZONE": "UTC", "can_cloud_download": "false"}, + ), + ( + {"use_cloud_fetch": False}, + {"can_cloud_download": "true"}, + {"can_cloud_download": "false"}, + ), + ( + {"use_cloud_fetch": False}, + {"Can_Cloud_Download": "true"}, + {"can_cloud_download": "false"}, + ), ], ) -def test_open_session_honors_cloudfetch_option(monkeypatch, kwargs, expected): +def test_open_session_honors_cloudfetch_option( + monkeypatch, kwargs, configuration, expected +): captured = {} def fake_session(**options): @@ -369,11 +390,10 @@ def fake_session(**options): ssl_options=None, **kwargs, ) - configuration = {"TIMEZONE": "UTC"} + original = dict(configuration) if configuration is not None else None client.open_session(configuration, catalog=None, schema=None) - assert captured["session_conf"].get("can_cloud_download") == expected - assert captured["session_conf"]["TIMEZONE"] == "UTC" - assert configuration == {"TIMEZONE": "UTC"} + assert captured["session_conf"] == expected + assert configuration == original @pytest.mark.parametrize("timeout", [None, 0, 12.5]) From 3b57c6616d46ff74895144faf0970420a3a7826b Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Tue, 10 Nov 2026 23:47:34 +0000 Subject: [PATCH 03/13] docs: clarify when the CloudFetch connection option applies Signed-off-by: Aakash Saravanan --- CONNECTION_PARAMETERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 187bc4e06..f3185a73d 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -149,7 +149,7 @@ to change without notice. | Option | Type | Thrift | Kernel | Default Value | Note | | ------------------------------------- | ------ | :----: | :----: | ------------- | ------------------------------------------------------------------------------------------------------------- | -| `use_cloud_fetch` | `bool` | ✅ | ✅ | `True` | Set `False` to fetch complete results through the SQL connection without cloud-storage downloads. | +| `use_cloud_fetch` | `bool` | ✅ | ✅ | `True` | Set `False` at connection creation to fetch complete results without cloud-storage downloads. On the kernel path, changing this attribute after connecting does not change result delivery. | | `max_download_threads` | `int` | ✅ | ❌ | `10` | Worker threads for cloud-fetch downloads. Not forwarded to the kernel. | | `enable_query_result_lz4_compression` | `bool` | ✅ | ❌ | `True` | LZ4-compress result payloads. Not forwarded; the kernel handles compression internally. | | `_disable_pandas` | `bool` | ✅ | ✅ | `False` | Skip the pandas-based Arrow→row deserialization and materialize rows directly with PyArrow. This is a **Python-side** result-conversion toggle, not a wire option: the kernel returns results as Arrow (`RecordBatch`es) and the connector runs the *same* `_convert_arrow_table` for both backends, so the flag is honored on the kernel path too. Affects only row fetches (`fetchone`/`fetchmany`/`fetchall`); the `fetch*_arrow` methods return the Arrow table unchanged regardless of this flag. | From eece87e6563c31c5d3a9f3ed394ea8611798a03d Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Tue, 10 Nov 2026 23:52:06 +0000 Subject: [PATCH 04/13] test(kernel): fetch complete large results with CloudFetch disabled Signed-off-by: Aakash Saravanan --- tests/e2e/test_kernel_backend.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/tests/e2e/test_kernel_backend.py b/tests/e2e/test_kernel_backend.py index ae0301fb6..ed199662b 100644 --- a/tests/e2e/test_kernel_backend.py +++ b/tests/e2e/test_kernel_backend.py @@ -21,6 +21,7 @@ from __future__ import annotations +import hashlib import sys from uuid import uuid4 @@ -136,6 +137,33 @@ def conn(kernel_conn_params): c.close() +def test_cloudfetch_disabled_returns_complete_large_result(kernel_conn_params): + from databricks.sql.backend.kernel.client import KernelDatabricksClient + + with sql.connect(**kernel_conn_params, use_cloud_fetch=False) as connection: + assert isinstance(connection.session.backend, KernelDatabricksClient) + with connection.cursor() as cursor: + cursor.execute( + "SELECT id, sha2(CAST(id AS STRING), 512) AS payload " + "FROM range(250000) ORDER BY id" + ) + count = 0 + payload_bytes = 0 + digest = hashlib.sha256() + while rows := cursor.fetchmany(8192): + for identity, payload in rows: + assert identity == count + assert payload == hashlib.sha512(str(identity).encode()).hexdigest() + payload_bytes += len(payload) + digest.update(f"{identity}\t{payload}\n".encode()) + count += 1 + assert count == 250000 + assert payload_bytes == 32000000 + assert digest.hexdigest() == ( + "a25ada4c6c476b53e97eb6e8a945e9ce4c14cc432c7ad672dc6fb607d660c86a" + ) + + def test_connect_with_use_kernel_opens_a_session(conn): assert conn.open, "connection should report open after connect()" @@ -366,9 +394,7 @@ def test_schemas_with_empty_string_filter_matches_nothing(conn): assert cur.fetchall() == [] -@pytest.mark.parametrize( - "empty_filter", ["schema_name", "table_name", "column_name"] -) +@pytest.mark.parametrize("empty_filter", ["schema_name", "table_name", "column_name"]) def test_columns_with_empty_string_filter_matches_nothing(conn, empty_filter): filters = { "catalog_name": "system", From 0a04d94c01585b72ae2d01a4eb39828baca7e4f6 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Tue, 10 Nov 2026 23:58:06 +0000 Subject: [PATCH 05/13] test(kernel): distinguish inline delivery from CloudFetch Signed-off-by: Aakash Saravanan --- tests/e2e/test_kernel_backend.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/e2e/test_kernel_backend.py b/tests/e2e/test_kernel_backend.py index ed199662b..fa8137a19 100644 --- a/tests/e2e/test_kernel_backend.py +++ b/tests/e2e/test_kernel_backend.py @@ -137,10 +137,15 @@ def conn(kernel_conn_params): c.close() -def test_cloudfetch_disabled_returns_complete_large_result(kernel_conn_params): +@pytest.mark.parametrize("use_cloud_fetch", [False, True]) +def test_cloudfetch_disabled_returns_complete_large_result( + kernel_conn_params, caplog, use_cloud_fetch +): from databricks.sql.backend.kernel.client import KernelDatabricksClient - with sql.connect(**kernel_conn_params, use_cloud_fetch=False) as connection: + with caplog.at_level(logging.INFO, logger="databricks.sql.kernel"), sql.connect( + **kernel_conn_params, use_cloud_fetch=use_cloud_fetch + ) as connection: assert isinstance(connection.session.backend, KernelDatabricksClient) with connection.cursor() as cursor: cursor.execute( @@ -163,6 +168,19 @@ def test_cloudfetch_disabled_returns_complete_large_result(kernel_conn_params): "a25ada4c6c476b53e97eb6e8a945e9ce4c14cc432c7ad672dc6fb607d660c86a" ) + messages = [ + record.getMessage() + for record in caplog.records + if record.name.startswith("databricks.sql.kernel") + ] + assert ( + any("Using CloudFetch reader" in message for message in messages) + == use_cloud_fetch + ) + assert any( + "inline" in message.lower() and "reader" in message.lower() + for message in messages + ) == (not use_cloud_fetch) def test_connect_with_use_kernel_opens_a_session(conn): assert conn.open, "connection should report open after connect()" @@ -394,7 +412,9 @@ def test_schemas_with_empty_string_filter_matches_nothing(conn): assert cur.fetchall() == [] -@pytest.mark.parametrize("empty_filter", ["schema_name", "table_name", "column_name"]) +@pytest.mark.parametrize( + "empty_filter", ["schema_name", "table_name", "column_name"] +) def test_columns_with_empty_string_filter_matches_nothing(conn, empty_filter): filters = { "catalog_name": "system", From 0d12e2d97ed2b280d57aef50b3b1cecbb1631ffa Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 00:08:20 +0000 Subject: [PATCH 06/13] test: verify disabled CloudFetch with workspace-only network access Signed-off-by: Aakash Saravanan --- tests/e2e/test_kernel_backend.py | 76 ++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/tests/e2e/test_kernel_backend.py b/tests/e2e/test_kernel_backend.py index fa8137a19..d84f6347a 100644 --- a/tests/e2e/test_kernel_backend.py +++ b/tests/e2e/test_kernel_backend.py @@ -22,7 +22,11 @@ from __future__ import annotations import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import select +import socket import sys +import threading from uuid import uuid4 import pytest @@ -137,15 +141,61 @@ def conn(kernel_conn_params): c.close() -@pytest.mark.parametrize("use_cloud_fetch", [False, True]) +@pytest.fixture +def workspace_only_proxy(kernel_conn_params, monkeypatch): + connections = {"workspace": 0, "blocked": 0} + workspace = kernel_conn_params["server_hostname"] + ":443" + + class Handler(BaseHTTPRequestHandler): + def do_CONNECT(self): + self.close_connection = True + if self.path != workspace: + connections["blocked"] += 1 + self.send_error(403) + return + with socket.create_connection( + (kernel_conn_params["server_hostname"], 443), timeout=30 + ) as upstream: + connections["workspace"] += 1 + self.send_response(200) + self.end_headers() + while True: + ready, _, _ = select.select([self.connection, upstream], [], [], 30) + if not ready: + return + for source in ready: + data = source.recv(65536) + if not data: + return + target = ( + upstream if source is self.connection else self.connection + ) + target.sendall(data) + + def log_message(self, *_args): + pass + + proxy = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=proxy.serve_forever, daemon=True) + thread.start() + for name in ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"): + monkeypatch.setenv(name, f"http://127.0.0.1:{proxy.server_port}") + for name in ("NO_PROXY", "no_proxy"): + monkeypatch.setenv(name, "") + try: + yield connections + finally: + proxy.shutdown() + proxy.server_close() + thread.join() + + def test_cloudfetch_disabled_returns_complete_large_result( - kernel_conn_params, caplog, use_cloud_fetch + kernel_conn_params, workspace_only_proxy ): from databricks.sql.backend.kernel.client import KernelDatabricksClient - with caplog.at_level(logging.INFO, logger="databricks.sql.kernel"), sql.connect( - **kernel_conn_params, use_cloud_fetch=use_cloud_fetch - ) as connection: + with sql.connect(**kernel_conn_params, use_cloud_fetch=False) as connection: assert isinstance(connection.session.backend, KernelDatabricksClient) with connection.cursor() as cursor: cursor.execute( @@ -168,19 +218,9 @@ def test_cloudfetch_disabled_returns_complete_large_result( "a25ada4c6c476b53e97eb6e8a945e9ce4c14cc432c7ad672dc6fb607d660c86a" ) - messages = [ - record.getMessage() - for record in caplog.records - if record.name.startswith("databricks.sql.kernel") - ] - assert ( - any("Using CloudFetch reader" in message for message in messages) - == use_cloud_fetch - ) - assert any( - "inline" in message.lower() and "reader" in message.lower() - for message in messages - ) == (not use_cloud_fetch) + assert workspace_only_proxy["workspace"] > 0 + assert workspace_only_proxy["blocked"] == 0 + def test_connect_with_use_kernel_opens_a_session(conn): assert conn.open, "connection should report open after connect()" From 4ba9d746f9fcbb27bd382049b9f7d7fcaaa6806e Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 22:01:07 +0000 Subject: [PATCH 07/13] fix(kernel): delegate CloudFetch policy to native client configuration Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- src/databricks/sql/backend/kernel/client.py | 12 +++--------- tests/unit/test_kernel_client.py | 14 ++++++-------- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index 7538a248a..cbad0f58c 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -c58f62c95f3651488f6270c0b45686614486d351 +dccdcb58ecba1bf2f74066c47dd4a98dbe81cfeb diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 567b98450..fa8085de6 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -391,19 +391,13 @@ def open_session( ) -> SessionId: if self._kernel_session is not None: raise InterfaceError("KernelDatabricksClient already has an open session.") - # ``session_configuration`` flows through to the kernel's - # ``session_conf`` map verbatim; the SEA endpoint enforces - # its own allow-list and rejects unknown keys. + # The kernel consumes client settings locally before forwarding server parameters. session_conf: Optional[Dict[str, str]] = None if session_configuration: session_conf = {k: str(v) for k, v in session_configuration.items()} if not self._use_cloud_fetch: - session_conf = { - k: v - for k, v in (session_conf or {}).items() - if k.lower() != "can_cloud_download" - } - session_conf["can_cloud_download"] = "false" + session_conf = session_conf or {} + session_conf["cloudfetch_enabled"] = "false" # The kwarg builds run INSIDE the try so the ``finally`` scrub # below always fires — including when ``kernel_auth_kwargs`` # itself raises mid-build (e.g. an OAuth token-exchange failure diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 150eeed51..2a305e6ab 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -353,21 +353,21 @@ def fake_session(**kw): {"can_cloud_download": "false"}, {"can_cloud_download": "false"}, ), - ({"use_cloud_fetch": False}, None, {"can_cloud_download": "false"}), + ({"use_cloud_fetch": False}, None, {"cloudfetch_enabled": "false"}), ( {"use_cloud_fetch": False}, {"TIMEZONE": "UTC"}, - {"TIMEZONE": "UTC", "can_cloud_download": "false"}, + {"TIMEZONE": "UTC", "cloudfetch_enabled": "false"}, ), ( {"use_cloud_fetch": False}, {"can_cloud_download": "true"}, - {"can_cloud_download": "false"}, + {"can_cloud_download": "true", "cloudfetch_enabled": "false"}, ), ( {"use_cloud_fetch": False}, - {"Can_Cloud_Download": "true"}, - {"can_cloud_download": "false"}, + {"CLOUDFETCH_ENABLED": "true", "cloudfetch_enabled": "true"}, + {"CLOUDFETCH_ENABLED": "true", "cloudfetch_enabled": "false"}, ), ], ) @@ -445,9 +445,7 @@ def fake_session(**kw): @pytest.mark.parametrize("enabled", [True, False]) -def test_open_session_passes_geospatial_representation_to_kernel( - monkeypatch, enabled -): +def test_open_session_passes_geospatial_representation_to_kernel(monkeypatch, enabled): captured = {} def fake_session(*, enable_geospatial_support=True, **kw): From b324a11be17aff2b9a595aca10c53dd959ab1e04 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 22:09:47 +0000 Subject: [PATCH 08/13] build(kernel): pin native configuration diagnostics fix Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index cbad0f58c..3f2ede3c5 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -dccdcb58ecba1bf2f74066c47dd4a98dbe81cfeb +6615a0771c9cf254b16870c02203c199866cf7dc From c1e9c90c88d1f57a28e989f814020c915ecbe42f Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 22:21:08 +0000 Subject: [PATCH 09/13] docs: require the coordinated native release for disabled downloads Signed-off-by: Aakash Saravanan --- CONNECTION_PARAMETERS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index f3185a73d..fe42e891d 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -158,6 +158,12 @@ to change without notice. | `_use_arrow_native_decimals` | `bool` | ✅ | ❌ | `True` | Thrift wire encoding for `DECIMAL`: `True` → native Arrow `decimal128`, `False` → Arrow string. **No value-level effect**, though: the connector unconditionally re-casts the column back to `decimal128` (`convert_decimals_in_arrow_table`, `thrift_backend.py`), so both `fetchall()` and `fetchall_arrow()` yield `Decimal` / `decimal128(p,s)` either way (verified live). Not forwarded to the kernel, which always returns native Arrow decimals. | | `_use_arrow_native_timestamps` | `bool` | ✅ | ❌ | `True` | Thrift wire encoding for `TIMESTAMP`: `True` → native Arrow timestamp (→ Python `datetime`), `False` → Arrow string (→ Python **`str`**). **Unlike decimals there is no re-cast**, so `False` genuinely surfaces strings — and `cursor.description` still reports the type code as `'timestamp'`, a mismatch to watch for (verified live). Note the connector always also sends the `spark.thriftserver.arrowBasedRowSet.timestampAsString=false` conf, but the `timestampAsArrow=False` flag wins. Not forwarded to the kernel, which always returns native Arrow timestamps. | +Kernel support for `use_cloud_fetch=False` requires a native release containing +[databricks-sql-kernel#436](https://github.com/databricks/databricks-sql-kernel/pull/436). +`KERNEL_REV` selects that source for validation. The published native dependency +must be updated alongside the connector release; released kernel 1.4.0 does not +provide this behavior. + ## Session defaults & transactions | Option | Type | Thrift | Kernel | Default Value | Note | From 1f33ada6902742c30cf6faf05153c14a8ade4b0f Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 22:25:57 +0000 Subject: [PATCH 10/13] docs: clarify the CloudFetch option boolean input contract Signed-off-by: Aakash Saravanan --- CONNECTION_PARAMETERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index fe42e891d..21743fd47 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -163,6 +163,7 @@ Kernel support for `use_cloud_fetch=False` requires a native release containing `KERNEL_REV` selects that source for validation. The published native dependency must be updated alongside the connector release; released kernel 1.4.0 does not provide this behavior. +Pass a Python `bool` for this option; string values such as `"false"` are not parsed. ## Session defaults & transactions From 0eb26c69d26682719ddeb150a3390d47a5ceda53 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 22:42:57 +0000 Subject: [PATCH 11/13] Stop restricted proxy handlers promptly during teardown Signed-off-by: Aakash Saravanan --- tests/e2e/test_kernel_backend.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/test_kernel_backend.py b/tests/e2e/test_kernel_backend.py index d84f6347a..260c49689 100644 --- a/tests/e2e/test_kernel_backend.py +++ b/tests/e2e/test_kernel_backend.py @@ -145,6 +145,7 @@ def conn(kernel_conn_params): def workspace_only_proxy(kernel_conn_params, monkeypatch): connections = {"workspace": 0, "blocked": 0} workspace = kernel_conn_params["server_hostname"] + ":443" + stopping = threading.Event() class Handler(BaseHTTPRequestHandler): def do_CONNECT(self): @@ -159,10 +160,8 @@ def do_CONNECT(self): connections["workspace"] += 1 self.send_response(200) self.end_headers() - while True: - ready, _, _ = select.select([self.connection, upstream], [], [], 30) - if not ready: - return + while not stopping.is_set(): + ready, _, _ = select.select([self.connection, upstream], [], [], 1) for source in ready: data = source.recv(65536) if not data: @@ -185,6 +184,7 @@ def log_message(self, *_args): try: yield connections finally: + stopping.set() proxy.shutdown() proxy.server_close() thread.join() From 80820698780c4ffd5f027076695eb34618e5a9b0 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:53:14 +0000 Subject: [PATCH 12/13] docs: remove CloudFetch release coordination notes Signed-off-by: Aakash Saravanan --- CONNECTION_PARAMETERS.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 21743fd47..f3185a73d 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -158,13 +158,6 @@ to change without notice. | `_use_arrow_native_decimals` | `bool` | ✅ | ❌ | `True` | Thrift wire encoding for `DECIMAL`: `True` → native Arrow `decimal128`, `False` → Arrow string. **No value-level effect**, though: the connector unconditionally re-casts the column back to `decimal128` (`convert_decimals_in_arrow_table`, `thrift_backend.py`), so both `fetchall()` and `fetchall_arrow()` yield `Decimal` / `decimal128(p,s)` either way (verified live). Not forwarded to the kernel, which always returns native Arrow decimals. | | `_use_arrow_native_timestamps` | `bool` | ✅ | ❌ | `True` | Thrift wire encoding for `TIMESTAMP`: `True` → native Arrow timestamp (→ Python `datetime`), `False` → Arrow string (→ Python **`str`**). **Unlike decimals there is no re-cast**, so `False` genuinely surfaces strings — and `cursor.description` still reports the type code as `'timestamp'`, a mismatch to watch for (verified live). Note the connector always also sends the `spark.thriftserver.arrowBasedRowSet.timestampAsString=false` conf, but the `timestampAsArrow=False` flag wins. Not forwarded to the kernel, which always returns native Arrow timestamps. | -Kernel support for `use_cloud_fetch=False` requires a native release containing -[databricks-sql-kernel#436](https://github.com/databricks/databricks-sql-kernel/pull/436). -`KERNEL_REV` selects that source for validation. The published native dependency -must be updated alongside the connector release; released kernel 1.4.0 does not -provide this behavior. -Pass a Python `bool` for this option; string values such as `"false"` are not parsed. - ## Session defaults & transactions | Option | Type | Thrift | Kernel | Default Value | Note | From c6388c71baf0e0b2f38c3e9218046656b0a592d2 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:00:57 +0000 Subject: [PATCH 13/13] style: preserve the existing geospatial test signature Signed-off-by: Aakash Saravanan --- tests/unit/test_kernel_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 2a305e6ab..ce91c3baf 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -445,7 +445,9 @@ def fake_session(**kw): @pytest.mark.parametrize("enabled", [True, False]) -def test_open_session_passes_geospatial_representation_to_kernel(monkeypatch, enabled): +def test_open_session_passes_geospatial_representation_to_kernel( + monkeypatch, enabled +): captured = {} def fake_session(*, enable_geospatial_support=True, **kw):