From 2cd056280109aea420176cde9ba1a78978679c0b Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 22:44:25 +0000 Subject: [PATCH 01/40] feat(volume): enable native SQL file transfers Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- src/databricks/sql/backend/kernel/_errors.py | 22 +++- src/databricks/sql/backend/kernel/client.py | 74 +------------ tests/e2e/test_kernel_backend.py | 76 +++++++++----- tests/unit/test_kernel_client.py | 105 +++++++++---------- 5 files changed, 122 insertions(+), 157 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index d025be44a..df63b0f3b 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -6f4b6d607a61bdde61dcda2cf82bc8e80fd32cb9 +d41cc3657748966d8b69a802bc40e8d019022ceb diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index bcf3f56b1..ac890fd23 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -39,6 +39,7 @@ from __future__ import annotations import logging +from typing import Union from databricks.sql.exc import ( DatabaseError, @@ -115,7 +116,9 @@ } -def reraise_kernel_error(exc: "_kernel.KernelError") -> "Error": +def reraise_kernel_error( + exc: "_kernel.KernelError", +) -> "Union[Error, FileNotFoundError]": """Convert a ``databricks_sql_kernel.KernelError`` to a PEP 249 exception with the kernel's structured attributes forwarded onto the new instance. @@ -126,6 +129,9 @@ def reraise_kernel_error(exc: "_kernel.KernelError") -> "Error": """ code = getattr(exc, "code", "Unknown") cls = _CODE_TO_EXCEPTION.get(code, DatabaseError) + file_transfer = getattr(exc, "error_code", None) == "FILE_TRANSFER_ERROR" + if file_transfer: + cls = ProgrammingError if code == "PermissionDenied" else OperationalError # For ServerOperationError, reproduce the Thrift backend's # ``context`` dict so callers that read @@ -141,7 +147,15 @@ def reraise_kernel_error(exc: "_kernel.KernelError") -> "Error": "operation-id": getattr(exc, "query_id", None), "diagnostic-info": getattr(exc, "diagnostic_info", None), } - new = cls(getattr(exc, "message", str(exc)), context) + message = getattr(exc, "message", str(exc)) + if ( + file_transfer + and code == "NotFound" + and getattr(exc, "http_status", None) is None + ): + new = FileNotFoundError(message) + else: + new = cls(message, context) for attr in ( "code", @@ -162,7 +176,9 @@ def reraise_kernel_error(exc: "_kernel.KernelError") -> "Error": return new -def wrap_kernel_exception(what: str, exc: BaseException) -> "Error": +def wrap_kernel_exception( + what: str, exc: BaseException +) -> "Union[Error, FileNotFoundError]": """Map any exception from a PyO3 call site to a PEP 249 exception. - ``KernelError`` → mapped class with structured attrs forwarded. diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 0ca7ba6a8..3671c744e 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -16,8 +16,6 @@ - ``query_tags`` on execute is not supported (kernel exposes ``statement_conf`` but PyO3 doesn't surface it). -- Volume PUT/GET (staging operations): kernel has no Volume API - yet. Users on Thrift-only paths. """ from __future__ import annotations @@ -74,41 +72,6 @@ # per-request skip-and-warn. _KERNEL_MANAGED_HEADERS = frozenset({"authorization", "x-databricks-org-id"}) -# Leading verbs of SQL volume/staging statements. Detected by the -# leading token (case-insensitive) so the kernel backend can fail loud -# on staging ops it can't service — see ``execute_command``. -_STAGING_VERBS = ("PUT", "GET", "REMOVE") - - -def _strip_leading_sql_comments(sql: str) -> str: - """Strip leading whitespace and SQL comments (``-- …`` line and - ``/* … */`` block, possibly several) from ``sql``, returning the - remainder. - - Needed so staging detection sees the real leading verb: a - comment-prefixed staging op (``-- upload\\nPUT …`` or - ``/* c */ PUT …``, common in ETL scripts) must still be classified - as staging, or it would slip past the guard into the silent-no-op - bug. Block comments do not nest in Databricks SQL, so a simple - scan-to-``*/`` is correct. - """ - i = 0 - n = len(sql) - while i < n: - if sql[i].isspace(): - i += 1 - elif sql.startswith("--", i): - # Line comment: skip to end of line (or string). - nl = sql.find("\n", i) - i = n if nl == -1 else nl + 1 - elif sql.startswith("/*", i): - # Block comment: skip to closing */ (or end if unterminated). - close = sql.find("*/", i + 2) - i = n if close == -1 else close + 2 - else: - break - return sql[i:] - def _is_not_found(exc: BaseException) -> bool: """True iff ``exc`` is a kernel ``NotFound`` error (HTTP 404 / @@ -145,21 +108,6 @@ def _catalog_or_none(value: Optional[str]) -> Optional[str]: return value -def _is_staging_statement(operation: str) -> bool: - """True iff ``operation`` is a volume/staging statement (PUT / GET / - REMOVE). - - Strips leading whitespace + SQL comments first (so a comment- - prefixed staging op is still caught), then matches the leading token - only — so a normal query that merely *contains* the word (e.g. - ``SELECT 'GET' AS x``) isn't misflagged. - """ - stripped = _strip_leading_sql_comments(operation) - # First whitespace-delimited token, uppercased. - verb = stripped.split(None, 1)[0].upper() if stripped.strip() else "" - return verb in _STAGING_VERBS - - def _kernel_session_accepts_kwarg(name: str) -> bool: """True iff the installed ``databricks_sql_kernel.Session`` constructor declares keyword ``name``. @@ -575,28 +523,14 @@ def execute_command( # ``_async_statements`` and closed by ``close_command``); the sync # path drops it in finally. ``close_stmt`` is the post-success # decision flag — it stays True on sync, flips to False on async. - # Volume/staging (PUT/GET/REMOVE) is not supported on the kernel - # path: the kernel returns the staging control row as a normal - # result set (``KernelResultSet.is_staging_operation`` is always - # False), so the connector's ``_handle_staging_operation`` never - # fires and NO file is transferred. Rather than silently no-op - # (the Thrift path performs the presigned-URL upload/download), - # fail loud at the call site so ETL scripts don't ingest - # stale/missing data. Detected by the leading SQL verb — the - # only signal available pre-execute, since the kernel exposes no - # staging marker today. - if _is_staging_statement(operation): - raise NotSupportedError( - "Volume / staging operations (PUT / GET / REMOVE) are not " - "supported on the kernel backend (use_kernel=True); the file " - "transfer would silently not happen. Use the Thrift backend " - "for staging operations." - ) - close_stmt = True try: try: stmt.set_sql(operation) + allowed_paths = cursor.connection.staging_allowed_local_path + if isinstance(allowed_paths, str): + allowed_paths = [allowed_paths] + stmt.set_staging_allowed_local_paths(allowed_paths or []) stmt.set_row_limit(row_limit) if query_tags: # Per-statement query tags. The kernel serialises the diff --git a/tests/e2e/test_kernel_backend.py b/tests/e2e/test_kernel_backend.py index ae0301fb6..c04162a55 100644 --- a/tests/e2e/test_kernel_backend.py +++ b/tests/e2e/test_kernel_backend.py @@ -32,6 +32,7 @@ DatabaseError, NotSupportedError, OperationalError, + ProgrammingError, ServerOperationError, ) @@ -366,9 +367,7 @@ def test_schemas_with_empty_string_filter_matches_nothing(conn): assert cur.fetchall() == [] -@pytest.mark.parametrize( - "empty_filter", ["schema_name", "table_name", "column_name"] -) +@pytest.mark.parametrize("empty_filter", ["schema_name", "table_name", "column_name"]) def test_columns_with_empty_string_filter_matches_nothing(conn, empty_filter): filters = { "catalog_name": "system", @@ -777,28 +776,55 @@ def test_scientific_notation_decimal_parameter_binds(conn): assert int(cur.fetchall()[0][0]) == 100 -# ── Staging / volume — fail loud, not silent no-op ──────────────── - - -def test_staging_put_raises_not_supported(conn): - """A PUT (volume/staging) statement fails loud on the kernel path - rather than silently no-opping (which would make ETL ingest - stale/missing data). (CUJ-gap audit)""" - with conn.cursor() as cur: - with pytest.raises(NotSupportedError, match="staging"): - cur.execute("PUT '/tmp/x.csv' INTO '/Volumes/main/default/v/x.csv'") - - -def test_comment_prefixed_staging_put_raises_not_supported(conn): - """A comment-prefixed staging op (common in ETL scripts) must also - fail loud — the leading-verb detection strips SQL comments first, so - it can't slip past into the silent-no-op bug (PR #825 review #1).""" - with conn.cursor() as cur: - with pytest.raises(NotSupportedError, match="staging"): - cur.execute( - "-- upload the daily extract\n" - "PUT '/tmp/x.csv' INTO '/Volumes/main/default/v/x.csv'" - ) +@pytest.mark.parametrize("async_op", [False, True]) +def test_volume_file_lifecycle(kernel_conn_params, tmp_path, catalog, async_op): + allowed = tmp_path / "allowed" + allowed.mkdir() + source = allowed / "input.bin" + destination = allowed / "output.bin" + payload = bytes(range(256)) * 4096 + source.write_bytes(payload) + denied = tmp_path / "denied.bin" + denied.write_bytes(payload) + catalog = catalog or "main" + namespace = f"volume_{uuid4().hex}" + qualified = f"`{catalog.replace('`', '``')}`.`{namespace}`" + remote = f"/Volumes/{catalog}/{namespace}/files/input.bin" + with sql.connect( + **kernel_conn_params, staging_allowed_local_path=str(allowed) + ) as connection: + with connection.cursor() as cur: + cur.execute(f"CREATE SCHEMA {qualified}") + try: + cur.execute(f"CREATE VOLUME {qualified}.files") + + def execute(query): + if async_op: + cur.execute_async(query) + cur.get_async_execution_result() + else: + cur.execute(query) + return cur.fetchall() + + assert ( + execute(f"-- upload\nPUT '{source}' INTO '{remote}' OVERWRITE") + == [] + ) + assert execute(f"GET '{remote}' TO '{destination}'") == [] + assert destination.read_bytes() == payload + with pytest.raises(ProgrammingError): + execute(f"PUT '{denied}' INTO '{remote}' OVERWRITE") + with pytest.raises(FileNotFoundError): + execute( + f"PUT '{allowed / 'missing.bin'}' INTO '{remote}' OVERWRITE" + ) + assert execute(f"REMOVE '{remote}'") == [] + assert execute(f"REMOVE '{remote}'") == [] + with pytest.raises(OperationalError, match="404"): + execute(f"GET '{remote}' TO '{allowed / 'deleted.bin'}'") + assert execute("SELECT 1")[0][0] == 1 + finally: + cur.execute(f"DROP SCHEMA {qualified} CASCADE") # ── Error fidelity — diagnostic-info reaches .context ───────────── diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index b285ce6e1..2d39bdd5a 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -393,9 +393,7 @@ def fake_session(**kw): @pytest.mark.parametrize("enabled", [True, False]) -def test_open_session_passes_geospatial_representation_to_kernel( - monkeypatch, enabled -): +def test_open_session_passes_geospatial_representation_to_kernel(monkeypatch, enabled): captured = {} def fake_session(*, enable_geospatial_support=True, **kw): @@ -807,70 +805,43 @@ def test_execute_command_forwards_row_limit(row_limit): stmt.set_row_limit.assert_called_once_with(row_limit) -# --------------------------------------------------------------------------- -# Staging / volume operations — fail loud (not silently no-op) -# --------------------------------------------------------------------------- - - @pytest.mark.parametrize( - "operation", + "allowed, expected", [ - "PUT '/local/f.csv' INTO '/Volumes/c/s/v/f.csv'", - " put '/local/f' into '/Volumes/...'", # leading ws + lowercase - "GET '/Volumes/c/s/v/f' TO '/local/f'", - "REMOVE '/Volumes/c/s/v/f'", + (None, []), + ("/tmp/files", ["/tmp/files"]), + (["/tmp/a", "/tmp/b"], ["/tmp/a", "/tmp/b"]), ], ) -def test_staging_operation_raises_not_supported(operation): - """Volume/staging PUT/GET/REMOVE must FAIL LOUD on the kernel path - (the kernel can't perform the presigned-URL transfer; silently - no-opping would make ETL ingest stale/missing data).""" +@pytest.mark.parametrize("async_op", [False, True]) +def test_execute_command_forwards_allowed_local_paths(allowed, expected, async_op): c = _make_client() c._kernel_session = MagicMock() cursor = MagicMock() + cursor.connection.staging_allowed_local_path = allowed cursor.arraysize = 100 cursor.buffer_size_bytes = 1024 - - with pytest.raises(NotSupportedError, match="staging"): - c.execute_command( - operation=operation, - session_id=MagicMock(), - max_rows=1, - max_bytes=1, - lz4_compression=False, - cursor=cursor, - use_cloud_fetch=False, - parameters=[], - async_op=False, - enforce_embedded_schema_correctness=False, - ) - - -@pytest.mark.parametrize( - "operation, is_staging", - [ - ("PUT '/f' INTO '/v'", True), - ("get '/v' to '/f'", True), - ("REMOVE '/v'", True), - # Comment-prefixed staging ops MUST still be caught — otherwise - # they slip into the silent-no-op bug this guard exists to close - # (regression: review #1 on PR #825). ETL scripts commonly - # prefix statements with comments. - ("-- upload the file\nPUT '/f' INTO '/v'", True), - ("/* staging */ PUT '/f' INTO '/v'", True), - ("/* c1 */\n -- c2\n get '/v' to '/f'", True), # mixed, multiple - (" \n\t PUT '/f' INTO '/v'", True), # leading whitespace only - ("SELECT 'GET' AS x", False), # word appears but not leading verb - ("SELECT * FROM puts", False), - ("-- PUT in a comment\nSELECT 1", False), # verb only in comment - ("/* PUT */ SELECT 1", False), - ("INSERT INTO t VALUES (1)", False), - ("", False), - ("-- just a comment", False), # comment only, no statement - ], -) -def test_is_staging_statement(operation, is_staging): - assert kernel_client._is_staging_statement(operation) is is_staging + stmt = c._kernel_session.statement.return_value + executed = MagicMock( + statement_id="00000000-0000-0000-0000-000000000001", + arrow_schema=MagicMock(return_value=pa.schema([("x", pa.int64())])), + ) + stmt.execute.return_value = executed + stmt.submit.return_value = executed + c.execute_command( + operation="PUT '/tmp/files/input' INTO '/Volumes/c/s/v/input'", + session_id=MagicMock(), + max_rows=1, + max_bytes=1, + lz4_compression=False, + cursor=cursor, + use_cloud_fetch=False, + parameters=[], + async_op=async_op, + enforce_embedded_schema_correctness=False, + ) + stmt.set_staging_allowed_local_paths.assert_called_once_with(expected) + (stmt.submit if async_op else stmt.execute).assert_called_once() # --------------------------------------------------------------------------- @@ -2582,3 +2553,21 @@ def test_only_reserved_headers_omits_kwarg(self, monkeypatch): monkeypatch, [("Authorization", "Bearer x"), ("x-databricks-org-id", "1")] ) assert "http_headers" not in captured + + +@pytest.mark.parametrize( + "code, http_status, expected", + [ + ("PermissionDenied", None, ProgrammingError), + ("NotFound", None, FileNotFoundError), + ("NotFound", 404, OperationalError), + ("NetworkError", None, OperationalError), + ], +) +def test_file_transfer_error_matches_thrift_exception(code, http_status, expected): + error = _FakeKernelError(code=code, message="File transfer failed") + error.error_code = "FILE_TRANSFER_ERROR" + error.http_status = http_status + result = kernel_client._reraise_kernel_error(error) + assert type(result) is expected + assert result.error_code == "FILE_TRANSFER_ERROR" From 95402453dcc9411af6c018317fc76a04d59129e7 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 22:56:18 +0000 Subject: [PATCH 02/40] fix(kernel): preserve ordinary queries with older native wheels Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 1 + CONNECTION_PARAMETERS.md | 3 +- KERNEL_REV | 2 +- src/databricks/sql/backend/kernel/_errors.py | 1 + src/databricks/sql/backend/kernel/client.py | 33 +++++++++++++++- tests/unit/test_kernel_client.py | 40 ++++++++++++++++++++ 6 files changed, 76 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fbb64b68..0ed0ac044 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Release History # Unreleased +- Support SQL PUT, GET, and REMOVE file transfers with a native kernel exposing `Statement.set_staging_allowed_local_paths`; older wheels reject transfers and continue to support ordinary queries. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) # 4.9.0 (2026-11-04) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 37736ed1a..a57255039 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -171,7 +171,7 @@ to change without notice. | `use_inline_params` | `bool` \| `"silent"` | ✅ | ✅ | `False` | Render parameters inline (legacy) vs. native bound params (DBR 14.1+). Shared cursor code performs the rendering before dispatch, so both backends honor it identically. | | `ignore_transactions` | `bool` | ✅ | ✅ | `True` | When `True`: `commit()` is a no-op, `rollback()` raises `NotSupportedError`, and setting `autocommit` is a no-op. | | `fetch_autocommit_from_server`| `bool` | ✅ | ✅ | `False` | Query the server (`SET AUTOCOMMIT`) for autocommit state instead of returning the cached value. | -| `staging_allowed_local_path` | `str` \| `List[str]` | ✅ | ❌ | `None` | Local path(s) permitted for Unity Catalog Volume `PUT`/`GET`. **Thrift-only** — the kernel has no Volume API yet. | +| `staging_allowed_local_path` | `str` \| `List[str]` | ✅ | ✅ | `None` | Local path(s) permitted for Unity Catalog Volume `PUT`/`GET`. Kernel transfers require a wheel exposing `Statement.set_staging_allowed_local_paths` (kernel [#439](https://github.com/databricks/databricks-sql-kernel/pull/439)); older wheels reject transfer commands. | ## Telemetry @@ -206,7 +206,6 @@ installed kernel binding. `force_enable_telemetry` remains Python-driver-only. `enable_query_result_lz4_compression`. 5. Arrow-native rendering for `_use_arrow_native_decimals` / `_use_arrow_native_timestamps` (complex types **are** forwarded). -6. `staging_allowed_local_path` (Volume `PUT`/`GET`). 7. Retry fine-tuning beyond the four forwarded knobs: `_retry_delay_default`, `_retry_dangerous_codes`, `_respect_server_retry_after_header`, `_retry_max_redirects`, `_enable_v3_retries`. diff --git a/KERNEL_REV b/KERNEL_REV index df63b0f3b..31390e272 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -d41cc3657748966d8b69a802bc40e8d019022ceb +bc39c7921acecd95e4eea0740becb883867a834b diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index ac890fd23..d2a9c2af1 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -148,6 +148,7 @@ def reraise_kernel_error( "diagnostic-info": getattr(exc, "diagnostic_info", None), } message = getattr(exc, "message", str(exc)) + # Local missing files have no HTTP status; remote missing files carry 404. if ( file_transfer and code == "NotFound" diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 3671c744e..8e79c452c 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -73,6 +73,30 @@ _KERNEL_MANAGED_HEADERS = frozenset({"authorization", "x-databricks-org-id"}) +def _strip_leading_sql_comments(sql: str) -> str: + """Skip SQL comments so unsupported transfers cannot appear successful.""" + i = 0 + n = len(sql) + while i < n: + if sql[i].isspace(): + i += 1 + elif sql.startswith("--", i): + nl = sql.find("\n", i) + i = n if nl == -1 else nl + 1 + elif sql.startswith("/*", i): + close = sql.find("*/", i + 2) + i = n if close == -1 else close + 2 + else: + break + return sql[i:] + + +def _is_staging_statement(operation: str) -> bool: + stripped = _strip_leading_sql_comments(operation) + verb = stripped.split(None, 1)[0].upper() if stripped.strip() else "" + return verb in ("PUT", "GET", "REMOVE") + + def _is_not_found(exc: BaseException) -> bool: """True iff ``exc`` is a kernel ``NotFound`` error (HTTP 404 / ``STATEMENT_NOT_FOUND``). @@ -530,7 +554,14 @@ def execute_command( allowed_paths = cursor.connection.staging_allowed_local_path if isinstance(allowed_paths, str): allowed_paths = [allowed_paths] - stmt.set_staging_allowed_local_paths(allowed_paths or []) + set_paths = getattr(stmt, "set_staging_allowed_local_paths", None) + if callable(set_paths): + set_paths(allowed_paths or []) + elif _is_staging_statement(operation): + raise NotSupportedError( + "Volume transfers require a databricks-sql-kernel wheel " + "with staging_allowed_local_paths support." + ) stmt.set_row_limit(row_limit) if query_tags: # Per-statement query tags. The kernel serialises the diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 2d39bdd5a..ca49f5035 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -844,6 +844,46 @@ def test_execute_command_forwards_allowed_local_paths(allowed, expected, async_o (stmt.submit if async_op else stmt.execute).assert_called_once() +@pytest.mark.parametrize( + "operation", + [ + "SELECT 1", + "PUT 'a' INTO 'b'", + "-- comment\nGET 'a' TO 'b'", + "/* comment */ REMOVE 'a'", + ], +) +def test_execute_command_with_older_kernel(operation): + c = _make_client() + c._kernel_session = MagicMock() + stmt = c._kernel_session.statement.return_value + del stmt.set_staging_allowed_local_paths + stmt.execute.return_value = MagicMock( + statement_id="00000000-0000-0000-0000-000000000001", + arrow_schema=MagicMock(return_value=pa.schema([("x", pa.int64())])), + ) + kwargs = dict( + operation=operation, + session_id=MagicMock(), + max_rows=1, + max_bytes=1, + lz4_compression=False, + cursor=MagicMock(), + use_cloud_fetch=False, + parameters=[], + async_op=False, + enforce_embedded_schema_correctness=False, + ) + if operation == "SELECT 1": + c.execute_command(**kwargs) + stmt.execute.assert_called_once() + else: + with pytest.raises(NotSupportedError, match="Volume transfers require"): + c.execute_command(**kwargs) + stmt.execute.assert_not_called() + stmt.close.assert_called_once() + + # --------------------------------------------------------------------------- # Sync cancel wiring (cursor.cancel() during a blocking execute()) # --------------------------------------------------------------------------- From 175efa38ef9542d801e78a1ad60627ab4cca06be Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 22:59:49 +0000 Subject: [PATCH 03/40] docs(kernel): preserve SQL transfer guard explanations Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/client.py | 41 +++++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 8e79c452c..98655de17 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -73,17 +73,35 @@ _KERNEL_MANAGED_HEADERS = frozenset({"authorization", "x-databricks-org-id"}) +# Leading verbs of SQL volume/staging statements. Detected by the +# leading token (case-insensitive) so the kernel backend can fail loud +# on staging ops it can't service — see ``execute_command``. +_STAGING_VERBS = ("PUT", "GET", "REMOVE") + + def _strip_leading_sql_comments(sql: str) -> str: - """Skip SQL comments so unsupported transfers cannot appear successful.""" + """Strip leading whitespace and SQL comments (``-- …`` line and + ``/* … */`` block, possibly several) from ``sql``, returning the + remainder. + + Needed so staging detection sees the real leading verb: a + comment-prefixed staging op (``-- upload\\nPUT …`` or + ``/* c */ PUT …``, common in ETL scripts) must still be classified + as staging, or it would slip past the guard into the silent-no-op + bug. Block comments do not nest in Databricks SQL, so a simple + scan-to-``*/`` is correct. + """ i = 0 n = len(sql) while i < n: if sql[i].isspace(): i += 1 elif sql.startswith("--", i): + # Line comment: skip to end of line (or string). nl = sql.find("\n", i) i = n if nl == -1 else nl + 1 elif sql.startswith("/*", i): + # Block comment: skip to closing */ (or end if unterminated). close = sql.find("*/", i + 2) i = n if close == -1 else close + 2 else: @@ -91,12 +109,6 @@ def _strip_leading_sql_comments(sql: str) -> str: return sql[i:] -def _is_staging_statement(operation: str) -> bool: - stripped = _strip_leading_sql_comments(operation) - verb = stripped.split(None, 1)[0].upper() if stripped.strip() else "" - return verb in ("PUT", "GET", "REMOVE") - - def _is_not_found(exc: BaseException) -> bool: """True iff ``exc`` is a kernel ``NotFound`` error (HTTP 404 / ``STATEMENT_NOT_FOUND``). @@ -132,6 +144,21 @@ def _catalog_or_none(value: Optional[str]) -> Optional[str]: return value +def _is_staging_statement(operation: str) -> bool: + """True iff ``operation`` is a volume/staging statement (PUT / GET / + REMOVE). + + Strips leading whitespace + SQL comments first (so a comment- + prefixed staging op is still caught), then matches the leading token + only — so a normal query that merely *contains* the word (e.g. + ``SELECT 'GET' AS x``) isn't misflagged. + """ + stripped = _strip_leading_sql_comments(operation) + # First whitespace-delimited token, uppercased. + verb = stripped.split(None, 1)[0].upper() if stripped.strip() else "" + return verb in _STAGING_VERBS + + def _kernel_session_accepts_kwarg(name: str) -> bool: """True iff the installed ``databricks_sql_kernel.Session`` constructor declares keyword ``name``. From a82dba087b14dcfa7e6b23a7fc74f5a09e4138c2 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:09:09 +0000 Subject: [PATCH 04/40] fix(volume): reject stream markers and classify remote failures Signed-off-by: Aakash Saravanan --- CONNECTION_PARAMETERS.md | 8 ++++---- KERNEL_REV | 2 +- src/databricks/sql/backend/kernel/_errors.py | 6 +++++- src/databricks/sql/backend/kernel/client.py | 9 +++++++++ tests/unit/test_kernel_client.py | 20 ++++++++++++++++++++ 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index a57255039..4806ae342 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -206,12 +206,12 @@ installed kernel binding. `force_enable_telemetry` remains Python-driver-only. `enable_query_result_lz4_compression`. 5. Arrow-native rendering for `_use_arrow_native_decimals` / `_use_arrow_native_timestamps` (complex types **are** forwarded). -7. Retry fine-tuning beyond the four forwarded knobs: `_retry_delay_default`, +6. Retry fine-tuning beyond the four forwarded knobs: `_retry_delay_default`, `_retry_dangerous_codes`, `_respect_server_retry_after_header`, `_retry_max_redirects`, `_enable_v3_retries`. -8. `force_enable_telemetry`. -9. `_port` and `_connection_uri`. -10. `_pool_connections` and `_proxy_auth_method` do not configure the +7. `force_enable_telemetry`. +8. `_port` and `_connection_uri`. +9. `_pool_connections` and `_proxy_auth_method` do not configure the kernel-owned network stack. ### Supported on Kernel, missing / ignored on Thrift diff --git a/KERNEL_REV b/KERNEL_REV index 31390e272..a3d86de0a 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -bc39c7921acecd95e4eea0740becb883867a834b +ebb8204b36951b751dd837b2c41ced8c821b2d94 diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index d2a9c2af1..43beaf7ba 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -131,7 +131,11 @@ def reraise_kernel_error( cls = _CODE_TO_EXCEPTION.get(code, DatabaseError) file_transfer = getattr(exc, "error_code", None) == "FILE_TRANSFER_ERROR" if file_transfer: - cls = ProgrammingError if code == "PermissionDenied" else OperationalError + cls = ( + ProgrammingError + if code == "PermissionDenied" and getattr(exc, "http_status", None) is None + else OperationalError + ) # For ServerOperationError, reproduce the Thrift backend's # ``context`` dict so callers that read diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 98655de17..5e4e7abbc 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -22,6 +22,7 @@ import inspect import logging +import re import threading import uuid from typing import Any, Dict, List, Optional, Set, TYPE_CHECKING, Union @@ -577,6 +578,14 @@ def execute_command( close_stmt = True try: try: + if re.match( + r"""PUT\s+(['"])__input_stream__\1(?:\s|$)""", + _strip_leading_sql_comments(operation), + re.IGNORECASE, + ): + raise NotSupportedError( + "Streaming PUT is not supported by the kernel backend." + ) stmt.set_sql(operation) allowed_paths = cursor.connection.staging_allowed_local_path if isinstance(allowed_paths, str): diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index ca49f5035..5e2b685dc 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -844,6 +844,25 @@ def test_execute_command_forwards_allowed_local_paths(allowed, expected, async_o (stmt.submit if async_op else stmt.execute).assert_called_once() +@pytest.mark.parametrize( + "query", + [ + "PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", + "/* upload */ PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", + ], +) +def test_kernel_rejects_streaming_put(query): + c = _make_client() + c._kernel_session = MagicMock() + with pytest.raises(NotSupportedError, match="Streaming PUT"): + c.execute_command( + query, MagicMock(), 1, 1, False, MagicMock(), False, [], False, False + ) + stmt = c._kernel_session.statement.return_value + stmt.execute.assert_not_called() + stmt.close.assert_called_once() + + @pytest.mark.parametrize( "operation", [ @@ -2599,6 +2618,7 @@ def test_only_reserved_headers_omits_kwarg(self, monkeypatch): "code, http_status, expected", [ ("PermissionDenied", None, ProgrammingError), + ("PermissionDenied", 403, OperationalError), ("NotFound", None, FileNotFoundError), ("NotFound", 404, OperationalError), ("NetworkError", None, OperationalError), From c09ff986ebc2da817c379a126e364b7df264c2f2 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:11:51 +0000 Subject: [PATCH 05/40] fix(volume): normalize path-like roots and error union typing Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/_errors.py | 1 + src/databricks/sql/backend/kernel/client.py | 5 +++-- tests/unit/test_kernel_client.py | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index 43beaf7ba..f6fa8496c 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -152,6 +152,7 @@ def reraise_kernel_error( "diagnostic-info": getattr(exc, "diagnostic_info", None), } message = getattr(exc, "message", str(exc)) + new: Union[Error, FileNotFoundError] # Local missing files have no HTTP status; remote missing files carry 404. if ( file_transfer diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 5e4e7abbc..57141a07b 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -22,6 +22,7 @@ import inspect import logging +import os import re import threading import uuid @@ -588,11 +589,11 @@ def execute_command( ) stmt.set_sql(operation) allowed_paths = cursor.connection.staging_allowed_local_path - if isinstance(allowed_paths, str): + if isinstance(allowed_paths, (str, os.PathLike)): allowed_paths = [allowed_paths] set_paths = getattr(stmt, "set_staging_allowed_local_paths", None) if callable(set_paths): - set_paths(allowed_paths or []) + set_paths([os.fsdecode(path) for path in allowed_paths or []]) elif _is_staging_statement(operation): raise NotSupportedError( "Volume transfers require a databricks-sql-kernel wheel " diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 5e2b685dc..b4cf9f2e1 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -15,6 +15,7 @@ import sys import types +from pathlib import Path from typing import Optional from unittest.mock import MagicMock @@ -810,6 +811,8 @@ def test_execute_command_forwards_row_limit(row_limit): [ (None, []), ("/tmp/files", ["/tmp/files"]), + (Path("/tmp/files"), ["/tmp/files"]), + ([Path("/tmp/a"), Path("/tmp/b")], ["/tmp/a", "/tmp/b"]), (["/tmp/a", "/tmp/b"], ["/tmp/a", "/tmp/b"]), ], ) From 3305d43c9d2378f8824d4a1d78d3a362f4046b7d Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:24:05 +0000 Subject: [PATCH 06/40] fix(volume): preserve local validation errors and document release gate Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- pyproject.toml | 1 + src/databricks/sql/backend/kernel/_errors.py | 9 +++++++-- src/databricks/sql/client.py | 2 +- tests/e2e/test_kernel_backend.py | 11 ++++++++++- tests/unit/test_kernel_client.py | 1 + 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index a3d86de0a..7b10abd33 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -ebb8204b36951b751dd837b2c41ced8c821b2d94 +02283500d50e1bcd61007e0cf04a7e1e3d16ea86 diff --git a/pyproject.toml b/pyproject.toml index 947b1c996..5c1a30767 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ requests-kerberos = {version = "^0.15.0", optional = true} # Floor is 1.4.0 (``^1.4.0`` == ``>=1.4.0,<2.0.0``), allowing compatible # 1.x releases while keeping the next major upgrade deliberate. The kernel's # pyarrow constraint remains >=23.0.1,<24. +# TODO: raise the release floor after databricks/databricks-sql-kernel#439 is published. databricks-sql-kernel = {version = "^1.4.0", optional = true, python = ">=3.10"} diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index f6fa8496c..908e9ce77 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -17,6 +17,9 @@ "only PEP 249 exception types cross the boundary". Wrapped in ``OperationalError`` here. +Local missing transfer files deliberately retain the built-in +``FileNotFoundError`` contract. + These primitives live in their own module so both ``client.py`` (which orchestrates PyO3 calls) and ``result_set.py`` (which calls ``fetch_next_batch`` on the same kernel handles) can share them @@ -133,7 +136,8 @@ def reraise_kernel_error( if file_transfer: cls = ( ProgrammingError - if code == "PermissionDenied" and getattr(exc, "http_status", None) is None + if code in ("PermissionDenied", "InvalidArgument") + and getattr(exc, "http_status", None) is None else OperationalError ) @@ -185,9 +189,10 @@ def reraise_kernel_error( def wrap_kernel_exception( what: str, exc: BaseException ) -> "Union[Error, FileNotFoundError]": - """Map any exception from a PyO3 call site to a PEP 249 exception. + """Map a PyO3 exception to the connector's public error contract. - ``KernelError`` → mapped class with structured attrs forwarded. + A local missing transfer file maps to ``FileNotFoundError``. - Already-PEP-249 ``Error`` (e.g. raised by an inner caller that already mapped) → passed through unchanged. - Anything else (``TypeError`` / ``ValueError`` / etc. from PyO3 diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 613b62386..3dd5b73a7 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -1394,7 +1394,7 @@ def _handle_staging_put( session_id_hex=self.connection.get_session_id_hex(), ) - # Kernel backends reject staging operations before reaching these handlers. + # The kernel completes configured transfers before reaching these handlers. assert self.connection.http_client is not None with open(local_file, "rb") as fh: r = self.connection.http_client.request( diff --git a/tests/e2e/test_kernel_backend.py b/tests/e2e/test_kernel_backend.py index c04162a55..b4d935a3b 100644 --- a/tests/e2e/test_kernel_backend.py +++ b/tests/e2e/test_kernel_backend.py @@ -367,7 +367,9 @@ def test_schemas_with_empty_string_filter_matches_nothing(conn): assert cur.fetchall() == [] -@pytest.mark.parametrize("empty_filter", ["schema_name", "table_name", "column_name"]) +@pytest.mark.parametrize( + "empty_filter", ["schema_name", "table_name", "column_name"] +) def test_columns_with_empty_string_filter_matches_nothing(conn, empty_filter): filters = { "catalog_name": "system", @@ -798,6 +800,13 @@ def test_volume_file_lifecycle(kernel_conn_params, tmp_path, catalog, async_op): try: cur.execute(f"CREATE VOLUME {qualified}.files") + with sql.connect(**kernel_conn_params) as no_paths: + with no_paths.cursor() as denied_cursor: + with pytest.raises(ProgrammingError): + denied_cursor.execute( + f"PUT '{source}' INTO '{remote}' OVERWRITE" + ) + def execute(query): if async_op: cur.execute_async(query) diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index b4cf9f2e1..3330ad5ed 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -2621,6 +2621,7 @@ def test_only_reserved_headers_omits_kwarg(self, monkeypatch): "code, http_status, expected", [ ("PermissionDenied", None, ProgrammingError), + ("InvalidArgument", None, ProgrammingError), ("PermissionDenied", 403, OperationalError), ("NotFound", None, FileNotFoundError), ("NotFound", 404, OperationalError), From 67efcd90521702ba818c13e65b65731905306cba Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:36:40 +0000 Subject: [PATCH 07/40] fix(kernel): keep local path validation scoped to transfers Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 2 +- CONNECTION_PARAMETERS.md | 2 +- KERNEL_REV | 2 +- src/databricks/sql/backend/kernel/_errors.py | 7 ++++--- src/databricks/sql/backend/kernel/client.py | 6 +++++- src/databricks/sql/client.py | 2 +- tests/e2e/test_kernel_backend.py | 4 +--- tests/unit/test_kernel_client.py | 18 ++++++++++++++++++ 8 files changed, 32 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ed0ac044..a0cd27a43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History # Unreleased -- Support SQL PUT, GET, and REMOVE file transfers with a native kernel exposing `Statement.set_staging_allowed_local_paths`; older wheels reject transfers and continue to support ordinary queries. +- Support SQL PUT, GET, and REMOVE file transfers with a native kernel exposing `Statement.set_staging_allowed_local_paths`; older wheels reject transfers and continue to support ordinary queries. Streaming PUT remains unsupported in kernel mode. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) # 4.9.0 (2026-11-04) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 4806ae342..86fa9f84c 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -171,7 +171,7 @@ to change without notice. | `use_inline_params` | `bool` \| `"silent"` | ✅ | ✅ | `False` | Render parameters inline (legacy) vs. native bound params (DBR 14.1+). Shared cursor code performs the rendering before dispatch, so both backends honor it identically. | | `ignore_transactions` | `bool` | ✅ | ✅ | `True` | When `True`: `commit()` is a no-op, `rollback()` raises `NotSupportedError`, and setting `autocommit` is a no-op. | | `fetch_autocommit_from_server`| `bool` | ✅ | ✅ | `False` | Query the server (`SET AUTOCOMMIT`) for autocommit state instead of returning the cached value. | -| `staging_allowed_local_path` | `str` \| `List[str]` | ✅ | ✅ | `None` | Local path(s) permitted for Unity Catalog Volume `PUT`/`GET`. Kernel transfers require a wheel exposing `Statement.set_staging_allowed_local_paths` (kernel [#439](https://github.com/databricks/databricks-sql-kernel/pull/439)); older wheels reject transfer commands. | +| `staging_allowed_local_path` | `str` \| `List[str]` | ✅ | ✅ | `None` | Local path(s) permitted for Unity Catalog Volume `PUT`/`GET`. Kernel transfers require a wheel exposing `Statement.set_staging_allowed_local_paths` (kernel [#439](https://github.com/databricks/databricks-sql-kernel/pull/439)); older wheels reject transfer commands. Kernel mode does not support streaming PUT. | ## Telemetry diff --git a/KERNEL_REV b/KERNEL_REV index 7b10abd33..ef0cbe608 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -02283500d50e1bcd61007e0cf04a7e1e3d16ea86 +2f317d97212f00d73b89d947f5261a2145ea7a10 diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index 908e9ce77..b5e020a40 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -17,8 +17,8 @@ "only PEP 249 exception types cross the boundary". Wrapped in ``OperationalError`` here. -Local missing transfer files deliberately retain the built-in -``FileNotFoundError`` contract. +Local missing transfer files use the built-in ``FileNotFoundError`` class; +the OS ``errno`` and ``filename`` attributes are unavailable. These primitives live in their own module so both ``client.py`` (which orchestrates PyO3 calls) and ``result_set.py`` (which calls @@ -192,7 +192,8 @@ def wrap_kernel_exception( """Map a PyO3 exception to the connector's public error contract. - ``KernelError`` → mapped class with structured attrs forwarded. - A local missing transfer file maps to ``FileNotFoundError``. + A local missing transfer file maps to ``FileNotFoundError`` without + the OS ``errno`` or ``filename`` attributes. - Already-PEP-249 ``Error`` (e.g. raised by an inner caller that already mapped) → passed through unchanged. - Anything else (``TypeError`` / ``ValueError`` / etc. from PyO3 diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 57141a07b..3dcca615f 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -588,7 +588,11 @@ def execute_command( "Streaming PUT is not supported by the kernel backend." ) stmt.set_sql(operation) - allowed_paths = cursor.connection.staging_allowed_local_path + allowed_paths = ( + cursor.connection.staging_allowed_local_path + if _is_staging_statement(operation) + else None + ) if isinstance(allowed_paths, (str, os.PathLike)): allowed_paths = [allowed_paths] set_paths = getattr(stmt, "set_staging_allowed_local_paths", None) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 3dd5b73a7..4e7eedb3a 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -1394,7 +1394,7 @@ def _handle_staging_put( session_id_hex=self.connection.get_session_id_hex(), ) - # The kernel completes configured transfers before reaching these handlers. + # Kernel backends transfer files natively and never dispatch to these handlers. assert self.connection.http_client is not None with open(local_file, "rb") as fh: r = self.connection.http_client.request( diff --git a/tests/e2e/test_kernel_backend.py b/tests/e2e/test_kernel_backend.py index b4d935a3b..5e1a1c90f 100644 --- a/tests/e2e/test_kernel_backend.py +++ b/tests/e2e/test_kernel_backend.py @@ -367,9 +367,7 @@ def test_schemas_with_empty_string_filter_matches_nothing(conn): assert cur.fetchall() == [] -@pytest.mark.parametrize( - "empty_filter", ["schema_name", "table_name", "column_name"] -) +@pytest.mark.parametrize("empty_filter", ["schema_name", "table_name", "column_name"]) def test_columns_with_empty_string_filter_matches_nothing(conn, empty_filter): filters = { "catalog_name": "system", diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 3330ad5ed..e001ffcef 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -847,6 +847,24 @@ def test_execute_command_forwards_allowed_local_paths(allowed, expected, async_o (stmt.submit if async_op else stmt.execute).assert_called_once() +@pytest.mark.parametrize("allowed", [123, [object()]]) +def test_non_transfer_query_ignores_malformed_local_paths(allowed): + c = _make_client() + c._kernel_session = MagicMock() + cursor = MagicMock() + cursor.connection.staging_allowed_local_path = allowed + stmt = c._kernel_session.statement.return_value + stmt.execute.return_value = MagicMock( + statement_id="00000000-0000-0000-0000-000000000001", + arrow_schema=MagicMock(return_value=pa.schema([("x", pa.int64())])), + ) + c.execute_command( + "SELECT 1", MagicMock(), 1, 1, False, cursor, False, [], False, False + ) + stmt.set_staging_allowed_local_paths.assert_called_once_with([]) + stmt.execute.assert_called_once() + + @pytest.mark.parametrize( "query", [ From b47f25e23021610e11727aeb3919e696d76956d2 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:46:22 +0000 Subject: [PATCH 08/40] fix(kernel): classify invalid transfer paths and track release gate Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- pyproject.toml | 2 +- src/databricks/sql/backend/kernel/client.py | 15 ++++++++----- tests/unit/test_kernel_client.py | 25 ++++++++++++++++----- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index ef0cbe608..913fb888c 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -2f317d97212f00d73b89d947f5261a2145ea7a10 +95f294c8c2c825440191af9f58a9b6071aa68408 diff --git a/pyproject.toml b/pyproject.toml index 5c1a30767..633016790 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ requests-kerberos = {version = "^0.15.0", optional = true} # Floor is 1.4.0 (``^1.4.0`` == ``>=1.4.0,<2.0.0``), allowing compatible # 1.x releases while keeping the next major upgrade deliberate. The kernel's # pyarrow constraint remains >=23.0.1,<24. -# TODO: raise the release floor after databricks/databricks-sql-kernel#439 is published. +# TODO: gate merge/release on https://github.com/databricks/databricks-sql-kernel/issues/443. databricks-sql-kernel = {version = "^1.4.0", optional = true, python = ">=3.10"} diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 3dcca615f..dbf197319 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -74,10 +74,7 @@ # per-request skip-and-warn. _KERNEL_MANAGED_HEADERS = frozenset({"authorization", "x-databricks-org-id"}) - -# Leading verbs of SQL volume/staging statements. Detected by the -# leading token (case-insensitive) so the kernel backend can fail loud -# on staging ops it can't service — see ``execute_command``. +# Transfer detection controls local-path forwarding and the older-wheel guard. _STAGING_VERBS = ("PUT", "GET", "REMOVE") @@ -597,7 +594,15 @@ def execute_command( allowed_paths = [allowed_paths] set_paths = getattr(stmt, "set_staging_allowed_local_paths", None) if callable(set_paths): - set_paths([os.fsdecode(path) for path in allowed_paths or []]) + try: + normalized_paths = [ + os.fsdecode(path) for path in allowed_paths or [] + ] + except (TypeError, ValueError) as exc: + raise ProgrammingError( + "staging_allowed_local_path must be a path or a list of paths" + ) from exc + set_paths(normalized_paths) elif _is_staging_statement(operation): raise NotSupportedError( "Volume transfers require a databricks-sql-kernel wheel " diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index e001ffcef..e2da884a3 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -848,7 +848,10 @@ def test_execute_command_forwards_allowed_local_paths(allowed, expected, async_o @pytest.mark.parametrize("allowed", [123, [object()]]) -def test_non_transfer_query_ignores_malformed_local_paths(allowed): +@pytest.mark.parametrize( + "operation", ["SELECT 1", "PUT 'input' INTO '/Volumes/c/s/v/file'"] +) +def test_malformed_local_paths_only_reject_transfers(allowed, operation): c = _make_client() c._kernel_session = MagicMock() cursor = MagicMock() @@ -858,11 +861,21 @@ def test_non_transfer_query_ignores_malformed_local_paths(allowed): statement_id="00000000-0000-0000-0000-000000000001", arrow_schema=MagicMock(return_value=pa.schema([("x", pa.int64())])), ) - c.execute_command( - "SELECT 1", MagicMock(), 1, 1, False, cursor, False, [], False, False - ) - stmt.set_staging_allowed_local_paths.assert_called_once_with([]) - stmt.execute.assert_called_once() + + def execute(): + c.execute_command( + operation, MagicMock(), 1, 1, False, cursor, False, [], False, False + ) + + if operation.startswith("PUT"): + with pytest.raises(ProgrammingError, match="staging_allowed_local_path"): + execute() + stmt.execute.assert_not_called() + stmt.set_staging_allowed_local_paths.assert_not_called() + else: + execute() + stmt.set_staging_allowed_local_paths.assert_called_once_with([]) + stmt.execute.assert_called_once() @pytest.mark.parametrize( From 37fa359e61cc0e0fa0ae8fb51f4fff149e82c995 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:51:59 +0000 Subject: [PATCH 09/40] docs(kernel): clarify missing-file exception contract Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/_errors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index b5e020a40..a09e806f6 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -13,8 +13,7 @@ - Anything else — ``TypeError`` / ``OverflowError`` / ``ValueError`` from PyO3 argument conversion, or arbitrary extension-internal Python errors. These would otherwise propagate - raw to connector callers, breaking the DB-API contract that says - "only PEP 249 exception types cross the boundary". Wrapped in + raw to connector callers. Wrapped in ``OperationalError`` here. Local missing transfer files use the built-in ``FileNotFoundError`` class; From 964533b0558e27fb9188a03b905810ee3a3d3259 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:52:31 +0000 Subject: [PATCH 10/40] docs(kernel): describe local missing-file mapping Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/_errors.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index a09e806f6..547ea3972 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -121,9 +121,11 @@ def reraise_kernel_error( exc: "_kernel.KernelError", ) -> "Union[Error, FileNotFoundError]": - """Convert a ``databricks_sql_kernel.KernelError`` to a PEP 249 - exception with the kernel's structured attributes forwarded onto - the new instance. + """Map a kernel error and forward its structured attributes. + + Local ``FILE_TRANSFER_ERROR`` errors with ``NotFound`` and no HTTP + status use the built-in ``FileNotFoundError``; other errors use + PEP 249 exception classes. The returned exception is raised by callers with ``raise ... from exc``; the ``from`` clause is what sets ``__cause__``, so we don't From 4bfd4b9abdeb7bb15fb32bcdc65ea47dbb29b3df Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:59:30 +0000 Subject: [PATCH 11/40] build(kernel): pin hardened file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 913fb888c..f164365a1 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -95f294c8c2c825440191af9f58a9b6071aa68408 +f68e4e5d1600e987187cf84f58ddf6a61e6a5ba6 From e1fc61575bb15c4fba19cbde2019e999a7faaf2c Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:04:59 +0000 Subject: [PATCH 12/40] fix(kernel): match documented local path option types Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/client.py | 18 ++++++++++++++---- tests/unit/test_kernel_client.py | 16 ++++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index dbf197319..1f86ba204 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -587,20 +587,30 @@ def execute_command( stmt.set_sql(operation) allowed_paths = ( cursor.connection.staging_allowed_local_path - if _is_staging_statement(operation) + if re.match( + r"(?:PUT|GET)\b", + _strip_leading_sql_comments(operation), + re.IGNORECASE, + ) else None ) - if isinstance(allowed_paths, (str, os.PathLike)): - allowed_paths = [allowed_paths] set_paths = getattr(stmt, "set_staging_allowed_local_paths", None) if callable(set_paths): + if allowed_paths is not None and not isinstance( + allowed_paths, (str, list) + ): + raise ProgrammingError( + "staging_allowed_local_path must be a string or a list of paths" + ) + if isinstance(allowed_paths, str): + allowed_paths = [allowed_paths] try: normalized_paths = [ os.fsdecode(path) for path in allowed_paths or [] ] except (TypeError, ValueError) as exc: raise ProgrammingError( - "staging_allowed_local_path must be a path or a list of paths" + "staging_allowed_local_path must be a string or a list of paths" ) from exc set_paths(normalized_paths) elif _is_staging_statement(operation): diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index e2da884a3..5fc583b22 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -811,7 +811,6 @@ def test_execute_command_forwards_row_limit(row_limit): [ (None, []), ("/tmp/files", ["/tmp/files"]), - (Path("/tmp/files"), ["/tmp/files"]), ([Path("/tmp/a"), Path("/tmp/b")], ["/tmp/a", "/tmp/b"]), (["/tmp/a", "/tmp/b"], ["/tmp/a", "/tmp/b"]), ], @@ -847,9 +846,18 @@ def test_execute_command_forwards_allowed_local_paths(allowed, expected, async_o (stmt.submit if async_op else stmt.execute).assert_called_once() -@pytest.mark.parametrize("allowed", [123, [object()]]) @pytest.mark.parametrize( - "operation", ["SELECT 1", "PUT 'input' INTO '/Volumes/c/s/v/file'"] + "allowed", + [123, [object()], Path("/tmp/files"), ("/tmp/files",), (), {"/tmp/files": True}], +) +@pytest.mark.parametrize( + "operation", + [ + "SELECT 1", + "PUT 'input' INTO '/Volumes/c/s/v/file'", + "GET '/Volumes/c/s/v/file' TO 'output'", + "REMOVE '/Volumes/c/s/v/file'", + ], ) def test_malformed_local_paths_only_reject_transfers(allowed, operation): c = _make_client() @@ -867,7 +875,7 @@ def execute(): operation, MagicMock(), 1, 1, False, cursor, False, [], False, False ) - if operation.startswith("PUT"): + if operation.startswith(("PUT", "GET")): with pytest.raises(ProgrammingError, match="staging_allowed_local_path"): execute() stmt.execute.assert_not_called() From 317bb1ad914b8a12fcbb2cd2d35154ff9de17a09 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:10:07 +0000 Subject: [PATCH 13/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index f164365a1..098e99438 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -f68e4e5d1600e987187cf84f58ddf6a61e6a5ba6 +c3e7cf1ba339155c7932c77e39e2821fd918cddf From 0f391f79ec5ffd35de2c30cfaf8ebd73449d1d77 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:19:07 +0000 Subject: [PATCH 14/40] test(volume): verify option and error boundaries Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 2 +- CONNECTION_PARAMETERS.md | 2 +- tests/unit/test_kernel_client.py | 47 +++++++++++++++++++++++--------- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0cd27a43..e8710805a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History # Unreleased -- Support SQL PUT, GET, and REMOVE file transfers with a native kernel exposing `Statement.set_staging_allowed_local_paths`; older wheels reject transfers and continue to support ordinary queries. Streaming PUT remains unsupported in kernel mode. +- Support SQL PUT, GET, and REMOVE file transfers with a native kernel built from `KERNEL_REV`. Published kernel 1.4.0 does not support these transfers; the release version and dependency floor are pending [kernel #443](https://github.com/databricks/databricks-sql-kernel/issues/443). Older wheels reject transfers and continue to support ordinary queries. Streaming PUT remains unsupported in kernel mode. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) # 4.9.0 (2026-11-04) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 86fa9f84c..048834a36 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -171,7 +171,7 @@ to change without notice. | `use_inline_params` | `bool` \| `"silent"` | ✅ | ✅ | `False` | Render parameters inline (legacy) vs. native bound params (DBR 14.1+). Shared cursor code performs the rendering before dispatch, so both backends honor it identically. | | `ignore_transactions` | `bool` | ✅ | ✅ | `True` | When `True`: `commit()` is a no-op, `rollback()` raises `NotSupportedError`, and setting `autocommit` is a no-op. | | `fetch_autocommit_from_server`| `bool` | ✅ | ✅ | `False` | Query the server (`SET AUTOCOMMIT`) for autocommit state instead of returning the cached value. | -| `staging_allowed_local_path` | `str` \| `List[str]` | ✅ | ✅ | `None` | Local path(s) permitted for Unity Catalog Volume `PUT`/`GET`. Kernel transfers require a wheel exposing `Statement.set_staging_allowed_local_paths` (kernel [#439](https://github.com/databricks/databricks-sql-kernel/pull/439)); older wheels reject transfer commands. Kernel mode does not support streaming PUT. | +| `staging_allowed_local_path` | `str` \| `List[str]` | ✅ | Source build | `None` | Local path(s) permitted for Unity Catalog Volume `PUT`/`GET`. Requires a native kernel built from `KERNEL_REV` with [kernel #439](https://github.com/databricks/databricks-sql-kernel/pull/439). Published 1.4.0 lacks this capability; the release version and dependency floor are pending [kernel #443](https://github.com/databricks/databricks-sql-kernel/issues/443). Older wheels reject transfers. Kernel mode does not support streaming PUT. | ## Telemetry diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 5fc583b22..9f7101b7a 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -816,7 +816,19 @@ def test_execute_command_forwards_row_limit(row_limit): ], ) @pytest.mark.parametrize("async_op", [False, True]) -def test_execute_command_forwards_allowed_local_paths(allowed, expected, async_op): +@pytest.mark.parametrize( + "operation, uses_paths", + [ + ("PUT '/tmp/files/input' INTO '/Volumes/c/s/v/input'", True), + ("GET '/Volumes/c/s/v/input' TO '/tmp/files/output'", True), + ("/* download */ GET '/Volumes/c/s/v/input' TO '/tmp/files/output'", True), + ("REMOVE '/Volumes/c/s/v/input'", False), + ("SELECT 1", False), + ], +) +def test_execute_command_forwards_allowed_local_paths( + allowed, expected, async_op, operation, uses_paths +): c = _make_client() c._kernel_session = MagicMock() cursor = MagicMock() @@ -831,7 +843,7 @@ def test_execute_command_forwards_allowed_local_paths(allowed, expected, async_o stmt.execute.return_value = executed stmt.submit.return_value = executed c.execute_command( - operation="PUT '/tmp/files/input' INTO '/Volumes/c/s/v/input'", + operation=operation, session_id=MagicMock(), max_rows=1, max_bytes=1, @@ -842,7 +854,9 @@ def test_execute_command_forwards_allowed_local_paths(allowed, expected, async_o async_op=async_op, enforce_embedded_schema_correctness=False, ) - stmt.set_staging_allowed_local_paths.assert_called_once_with(expected) + stmt.set_staging_allowed_local_paths.assert_called_once_with( + expected if uses_paths else [] + ) (stmt.submit if async_op else stmt.execute).assert_called_once() @@ -2657,20 +2671,27 @@ def test_only_reserved_headers_omits_kwarg(self, monkeypatch): @pytest.mark.parametrize( - "code, http_status, expected", + "code, http_status, error_code, expected", [ - ("PermissionDenied", None, ProgrammingError), - ("InvalidArgument", None, ProgrammingError), - ("PermissionDenied", 403, OperationalError), - ("NotFound", None, FileNotFoundError), - ("NotFound", 404, OperationalError), - ("NetworkError", None, OperationalError), + ("PermissionDenied", None, "FILE_TRANSFER_ERROR", ProgrammingError), + ("InvalidArgument", None, "FILE_TRANSFER_ERROR", ProgrammingError), + ("InvalidArgument", 400, "FILE_TRANSFER_ERROR", OperationalError), + ("PermissionDenied", 403, "FILE_TRANSFER_ERROR", OperationalError), + ("NotFound", None, "FILE_TRANSFER_ERROR", FileNotFoundError), + ("NotFound", 404, "FILE_TRANSFER_ERROR", OperationalError), + ("NetworkError", None, "FILE_TRANSFER_ERROR", OperationalError), + ("Internal", None, "FILE_TRANSFER_ERROR", OperationalError), + ("Internal", None, None, DatabaseError), ], ) -def test_file_transfer_error_matches_thrift_exception(code, http_status, expected): +def test_file_transfer_error_matches_thrift_exception( + code, http_status, error_code, expected +): error = _FakeKernelError(code=code, message="File transfer failed") - error.error_code = "FILE_TRANSFER_ERROR" + error.error_code = error_code error.http_status = http_status result = kernel_client._reraise_kernel_error(error) assert type(result) is expected - assert result.error_code == "FILE_TRANSFER_ERROR" + assert result.error_code == error_code + assert result.code == code + assert result.http_status == http_status From 865f6a65d59c64ce9bca488e4a7434049a0508be Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:22:17 +0000 Subject: [PATCH 15/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 098e99438..fc75c3b8f 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -c3e7cf1ba339155c7932c77e39e2821fd918cddf +7a938104d5e9870f995926f135aa41925c044cf6 From 83fc6742f145eb15d6671e1eb7303bce9a5053d9 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:35:40 +0000 Subject: [PATCH 16/40] fix(volume): preserve filesystem errors and unify transfer detection Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/_errors.py | 23 ++++++++------ src/databricks/sql/backend/kernel/client.py | 33 ++++++-------------- tests/unit/test_kernel_client.py | 3 ++ 3 files changed, 25 insertions(+), 34 deletions(-) diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index 547ea3972..f99ae4f82 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -120,12 +120,11 @@ def reraise_kernel_error( exc: "_kernel.KernelError", -) -> "Union[Error, FileNotFoundError]": +) -> "Union[Error, OSError]": """Map a kernel error and forward its structured attributes. - Local ``FILE_TRANSFER_ERROR`` errors with ``NotFound`` and no HTTP - status use the built-in ``FileNotFoundError``; other errors use - PEP 249 exception classes. + Local missing files use ``FileNotFoundError``; filesystem permission + failures use ``PermissionError``. Other errors use PEP 249 classes. The returned exception is raised by callers with ``raise ... from exc``; the ``from`` clause is what sets ``__cause__``, so we don't @@ -133,7 +132,11 @@ def reraise_kernel_error( """ code = getattr(exc, "code", "Unknown") cls = _CODE_TO_EXCEPTION.get(code, DatabaseError) - file_transfer = getattr(exc, "error_code", None) == "FILE_TRANSFER_ERROR" + error_code = getattr(exc, "error_code", None) + file_transfer = error_code in ( + "FILE_TRANSFER_ERROR", + "FILE_TRANSFER_PERMISSION_ERROR", + ) if file_transfer: cls = ( ProgrammingError @@ -157,9 +160,11 @@ def reraise_kernel_error( "diagnostic-info": getattr(exc, "diagnostic_info", None), } message = getattr(exc, "message", str(exc)) - new: Union[Error, FileNotFoundError] + new: Union[Error, OSError] # Local missing files have no HTTP status; remote missing files carry 404. - if ( + if error_code == "FILE_TRANSFER_PERMISSION_ERROR": + new = PermissionError(message) + elif ( file_transfer and code == "NotFound" and getattr(exc, "http_status", None) is None @@ -187,9 +192,7 @@ def reraise_kernel_error( return new -def wrap_kernel_exception( - what: str, exc: BaseException -) -> "Union[Error, FileNotFoundError]": +def wrap_kernel_exception(what: str, exc: BaseException) -> "Union[Error, OSError]": """Map a PyO3 exception to the connector's public error contract. - ``KernelError`` → mapped class with structured attrs forwarded. diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 1f86ba204..ae8f670f4 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -74,10 +74,6 @@ # per-request skip-and-warn. _KERNEL_MANAGED_HEADERS = frozenset({"authorization", "x-databricks-org-id"}) -# Transfer detection controls local-path forwarding and the older-wheel guard. -_STAGING_VERBS = ("PUT", "GET", "REMOVE") - - def _strip_leading_sql_comments(sql: str) -> str: """Strip leading whitespace and SQL comments (``-- …`` line and ``/* … */`` block, possibly several) from ``sql``, returning the @@ -143,19 +139,10 @@ def _catalog_or_none(value: Optional[str]) -> Optional[str]: return value -def _is_staging_statement(operation: str) -> bool: - """True iff ``operation`` is a volume/staging statement (PUT / GET / - REMOVE). - - Strips leading whitespace + SQL comments first (so a comment- - prefixed staging op is still caught), then matches the leading token - only — so a normal query that merely *contains* the word (e.g. - ``SELECT 'GET' AS x``) isn't misflagged. - """ - stripped = _strip_leading_sql_comments(operation) - # First whitespace-delimited token, uppercased. - verb = stripped.split(None, 1)[0].upper() if stripped.strip() else "" - return verb in _STAGING_VERBS +def _staging_verb(stripped_sql: str) -> Optional[str]: + """Match a leading verb so SELECT 'GET' cannot enable local file access.""" + match = re.match(r"(PUT|GET|REMOVE)\b", stripped_sql, re.IGNORECASE) + return match.group(1).upper() if match else None def _kernel_session_accepts_kwarg(name: str) -> bool: @@ -576,9 +563,11 @@ def execute_command( close_stmt = True try: try: + stripped_sql = _strip_leading_sql_comments(operation) + staging_verb = _staging_verb(stripped_sql) if re.match( r"""PUT\s+(['"])__input_stream__\1(?:\s|$)""", - _strip_leading_sql_comments(operation), + stripped_sql, re.IGNORECASE, ): raise NotSupportedError( @@ -587,11 +576,7 @@ def execute_command( stmt.set_sql(operation) allowed_paths = ( cursor.connection.staging_allowed_local_path - if re.match( - r"(?:PUT|GET)\b", - _strip_leading_sql_comments(operation), - re.IGNORECASE, - ) + if staging_verb in ("PUT", "GET") else None ) set_paths = getattr(stmt, "set_staging_allowed_local_paths", None) @@ -613,7 +598,7 @@ def execute_command( "staging_allowed_local_path must be a string or a list of paths" ) from exc set_paths(normalized_paths) - elif _is_staging_statement(operation): + elif staging_verb is not None: raise NotSupportedError( "Volume transfers require a databricks-sql-kernel wheel " "with staging_allowed_local_paths support." diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 9f7101b7a..e1e9c3537 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -926,6 +926,8 @@ def test_kernel_rejects_streaming_put(query): "PUT 'a' INTO 'b'", "-- comment\nGET 'a' TO 'b'", "/* comment */ REMOVE 'a'", + "GET'/Volumes/c/s/v/file' TO '/tmp/file'", + "/* comment */ REMOVE'/Volumes/c/s/v/file'", ], ) def test_execute_command_with_older_kernel(operation): @@ -2679,6 +2681,7 @@ def test_only_reserved_headers_omits_kwarg(self, monkeypatch): ("PermissionDenied", 403, "FILE_TRANSFER_ERROR", OperationalError), ("NotFound", None, "FILE_TRANSFER_ERROR", FileNotFoundError), ("NotFound", 404, "FILE_TRANSFER_ERROR", OperationalError), + ("PermissionDenied", None, "FILE_TRANSFER_PERMISSION_ERROR", PermissionError), ("NetworkError", None, "FILE_TRANSFER_ERROR", OperationalError), ("Internal", None, "FILE_TRANSFER_ERROR", OperationalError), ("Internal", None, None, DatabaseError), From 224a044a0d493fce60e69fee8da18457bcb44f4b Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:38:10 +0000 Subject: [PATCH 17/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- src/databricks/sql/backend/kernel/client.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index fc75c3b8f..152d3f41e 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -7a938104d5e9870f995926f135aa41925c044cf6 +ab10f222493fe45c645f746cdd2d762173a70fc5 diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index ae8f670f4..658976954 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -74,6 +74,7 @@ # per-request skip-and-warn. _KERNEL_MANAGED_HEADERS = frozenset({"authorization", "x-databricks-org-id"}) + def _strip_leading_sql_comments(sql: str) -> str: """Strip leading whitespace and SQL comments (``-- …`` line and ``/* … */`` block, possibly several) from ``sql``, returning the From dc9d19ee92086d2366c550d887f3037385cd73e0 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:47:12 +0000 Subject: [PATCH 18/40] fix(kernel): reject streaming PUT without whitespace Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/client.py | 2 +- tests/unit/test_kernel_client.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 658976954..493fa62e3 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -567,7 +567,7 @@ def execute_command( stripped_sql = _strip_leading_sql_comments(operation) staging_verb = _staging_verb(stripped_sql) if re.match( - r"""PUT\s+(['"])__input_stream__\1(?:\s|$)""", + r"""PUT\s*(['"])__input_stream__\1(?:\s|$)""", stripped_sql, re.IGNORECASE, ): diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index e1e9c3537..97dede7d6 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -905,6 +905,8 @@ def execute(): [ "PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", "/* upload */ PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", + "PUT'__input_stream__' INTO '/Volumes/c/s/v/file'", + '/* upload */ PUT"__input_stream__" INTO \'/Volumes/c/s/v/file\'', ], ) def test_kernel_rejects_streaming_put(query): From 9591f9361a9cc5dedcc55c5e6686727134527e49 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:49:16 +0000 Subject: [PATCH 19/40] style: format streaming PUT regression cases Signed-off-by: Aakash Saravanan --- tests/unit/test_kernel_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 97dede7d6..725673b49 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -906,7 +906,7 @@ def execute(): "PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", "/* upload */ PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", "PUT'__input_stream__' INTO '/Volumes/c/s/v/file'", - '/* upload */ PUT"__input_stream__" INTO \'/Volumes/c/s/v/file\'', + "/* upload */ PUT\"__input_stream__\" INTO '/Volumes/c/s/v/file'", ], ) def test_kernel_rejects_streaming_put(query): From 934cf35bd460a0513fd0d086516b3e239c29dc3c Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:49:42 +0000 Subject: [PATCH 20/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 152d3f41e..731c1165e 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -ab10f222493fe45c645f746cdd2d762173a70fc5 +6dbc9a7c721e55e535b469df446251f1fc0ccaaa From 9ddfb8033c77e146b50a366f509b73ad0c5977bf Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:57:54 +0000 Subject: [PATCH 21/40] fix(kernel): distinguish HTTP and filesystem permission errors Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/_errors.py | 5 ++++- tests/unit/test_kernel_client.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index f99ae4f82..909f57e2d 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -162,7 +162,10 @@ def reraise_kernel_error( message = getattr(exc, "message", str(exc)) new: Union[Error, OSError] # Local missing files have no HTTP status; remote missing files carry 404. - if error_code == "FILE_TRANSFER_PERMISSION_ERROR": + if ( + error_code == "FILE_TRANSFER_PERMISSION_ERROR" + and getattr(exc, "http_status", None) is None + ): new = PermissionError(message) elif ( file_transfer diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 725673b49..1a4ef1c28 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -2684,6 +2684,7 @@ def test_only_reserved_headers_omits_kwarg(self, monkeypatch): ("NotFound", None, "FILE_TRANSFER_ERROR", FileNotFoundError), ("NotFound", 404, "FILE_TRANSFER_ERROR", OperationalError), ("PermissionDenied", None, "FILE_TRANSFER_PERMISSION_ERROR", PermissionError), + ("PermissionDenied", 403, "FILE_TRANSFER_PERMISSION_ERROR", OperationalError), ("NetworkError", None, "FILE_TRANSFER_ERROR", OperationalError), ("Internal", None, "FILE_TRANSFER_ERROR", OperationalError), ("Internal", None, None, DatabaseError), From b07f679ddbf702104133950719f280fb9a6def45 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:00:00 +0000 Subject: [PATCH 22/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 731c1165e..bbb429490 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -6dbc9a7c721e55e535b469df446251f1fc0ccaaa +bd8adef4f3a38cc8807e4a22b614865246ee7cc2 From 1598dc0922d65738fe9811068fa9be7f6e76a1fa Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:10:18 +0000 Subject: [PATCH 23/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index bbb429490..533b2c979 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -bd8adef4f3a38cc8807e4a22b614865246ee7cc2 +c3300aaa17d0d1109e2579c40e6f35b3c3aeaf54 From 6015540be2ac880b7f055697b2a0126df7553d1f Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:23:14 +0000 Subject: [PATCH 24/40] fix(kernel): preserve mapped transfer errors through nested calls Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/_errors.py | 8 ++- tests/unit/test_kernel_client.py | 60 ++++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index 909f57e2d..73d85e6f8 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -201,8 +201,8 @@ def wrap_kernel_exception(what: str, exc: BaseException) -> "Union[Error, OSErro - ``KernelError`` → mapped class with structured attrs forwarded. A local missing transfer file maps to ``FileNotFoundError`` without the OS ``errno`` or ``filename`` attributes. - - Already-PEP-249 ``Error`` (e.g. raised by an inner caller that - already mapped) → passed through unchanged. + - Already-mapped PEP 249 or local transfer errors → passed through + unchanged so nested callers preserve their class and attributes. - Anything else (``TypeError`` / ``ValueError`` / etc. from PyO3 argument conversion, extension-internal errors) → wrapped in ``OperationalError``. @@ -215,6 +215,10 @@ def wrap_kernel_exception(what: str, exc: BaseException) -> "Union[Error, OSErro return reraise_kernel_error(exc) if isinstance(exc, Error): return exc + if isinstance(exc, (FileNotFoundError, PermissionError)) and getattr( + exc, "error_code", None + ) in ("FILE_TRANSFER_ERROR", "FILE_TRANSFER_PERMISSION_ERROR"): + return exc return OperationalError( f"Unexpected error from databricks_sql_kernel during {what}: {exc!r}" ) diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 1a4ef1c28..06827fdb4 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -900,6 +900,34 @@ def execute(): stmt.execute.assert_called_once() +@pytest.mark.parametrize( + "operation, expected", + [ + ("PUT '/f' INTO '/v'", "PUT"), + ("get '/v' to '/f'", "GET"), + ("REMOVE '/v'", "REMOVE"), + ("-- upload the file\nPUT '/f' INTO '/v'", "PUT"), + ("/* staging */ PUT '/f' INTO '/v'", "PUT"), + ("/* c1 */\n -- c2\n get '/v' to '/f'", "GET"), + (" \n\t PUT '/f' INTO '/v'", "PUT"), + ("SELECT 'GET' AS x", None), + ("SELECT * FROM puts", None), + ("-- PUT in a comment\nSELECT 1", None), + ("/* PUT */ SELECT 1", None), + ("INSERT INTO t VALUES (1)", None), + ("", None), + ("-- just a comment", None), + ("/* just a comment */", None), + ("PUTTER", None), + ("GETTING", None), + ("REMOVED", None), + ], +) +def test_staging_verb_matches_only_the_leading_statement(operation, expected): + stripped = kernel_client._strip_leading_sql_comments(operation) + assert kernel_client._staging_verb(stripped) == expected + + @pytest.mark.parametrize( "query", [ @@ -2701,3 +2729,35 @@ def test_file_transfer_error_matches_thrift_exception( assert result.error_code == error_code assert result.code == code assert result.http_status == http_status + + +@pytest.mark.parametrize( + "code,error_code", + [ + ("NotFound", "FILE_TRANSFER_ERROR"), + ("PermissionDenied", "FILE_TRANSFER_PERMISSION_ERROR"), + ], +) +def test_nested_wrapping_preserves_mapped_local_transfer_error(code, error_code): + from databricks.sql.backend.kernel._errors import wrap_kernel_exception + + error = _FakeKernelError(code=code, sql_state="HY000", query_id="transfer-id") + error.error_code = error_code + mapped = wrap_kernel_exception("inner", error) + wrapped = wrap_kernel_exception("outer", mapped) + assert wrapped is mapped + assert wrapped.code == code + assert wrapped.error_code == error_code + assert wrapped.sql_state == "HY000" + assert wrapped.query_id == "transfer-id" + + +@pytest.mark.parametrize( + "error", [FileNotFoundError("missing"), PermissionError("denied")] +) +def test_raw_local_errors_keep_unexpected_error_wrapping(error): + from databricks.sql.backend.kernel._errors import wrap_kernel_exception + + wrapped = wrap_kernel_exception("execute_command", error) + assert type(wrapped) is OperationalError + assert "execute_command" in str(wrapped) From 91ad80a6d2d5e77cf1595d7b17bd1965448e4792 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:29:04 +0000 Subject: [PATCH 25/40] docs(kernel): describe comment-prefixed transfer path handling Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/client.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 493fa62e3..04aec85ff 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -80,12 +80,8 @@ def _strip_leading_sql_comments(sql: str) -> str: ``/* … */`` block, possibly several) from ``sql``, returning the remainder. - Needed so staging detection sees the real leading verb: a - comment-prefixed staging op (``-- upload\\nPUT …`` or - ``/* c */ PUT …``, common in ETL scripts) must still be classified - as staging, or it would slip past the guard into the silent-no-op - bug. Block comments do not nest in Databricks SQL, so a simple - scan-to-``*/`` is correct. + Comment-prefixed PUT/GET statements need the permitted local paths + just as statements that begin with the verb do. """ i = 0 n = len(sql) From f7efb411fd56848ce47046131ddda442c8c4139c Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:30:18 +0000 Subject: [PATCH 26/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 533b2c979..4d7e0fd57 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -c3300aaa17d0d1109e2579c40e6f35b3c3aeaf54 +ed829e8ac8b27bdb503beaf6576dc41876df9465 From 706c9c4df5e74cf7660f59addbecaa570c459c07 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:40:47 +0000 Subject: [PATCH 27/40] docs: correct permitted staging path types Signed-off-by: Aakash Saravanan --- examples/staging_ingestion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/staging_ingestion.py b/examples/staging_ingestion.py index a55be4778..51b6eff0a 100644 --- a/examples/staging_ingestion.py +++ b/examples/staging_ingestion.py @@ -30,7 +30,7 @@ 2. Set the FILEPATH constant to the path of a file that will be uploaded (this example assumes its a CSV file) 3. Run this file -Note: staging_allowed_local_path can be either a Pathlike object or a list of Pathlike objects. +Note: staging_allowed_local_path accepts a path string or a list of path strings. """ INGESTION_USER = "some.user@example.com" From fb2ef00a0294c33f697f8d5f4691444cca67af78 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:41:20 +0000 Subject: [PATCH 28/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 4d7e0fd57..f67811cf1 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -ed829e8ac8b27bdb503beaf6576dc41876df9465 +2006c55d1b67903ca7b6090d0b84a2be73fcb82f From a42d0214cc8a860dfe2c9f9b012b32c3fd6d1d92 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:49:02 +0000 Subject: [PATCH 29/40] fix(kernel): recognize streaming PUT after a closing quote Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/client.py | 2 +- tests/unit/test_kernel_client.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 04aec85ff..97c4caee1 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -563,7 +563,7 @@ def execute_command( stripped_sql = _strip_leading_sql_comments(operation) staging_verb = _staging_verb(stripped_sql) if re.match( - r"""PUT\s*(['"])__input_stream__\1(?:\s|$)""", + r"""PUT\s*(['"])__input_stream__\1""", stripped_sql, re.IGNORECASE, ): diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 06827fdb4..3a12d2c74 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -932,6 +932,8 @@ def test_staging_verb_matches_only_the_leading_statement(operation, expected): "query", [ "PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", + "PUT '__input_stream__'INTO '/Volumes/c/s/v/file'", + 'PUT"__input_stream__"INTO "/Volumes/c/s/v/file"', "/* upload */ PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", "PUT'__input_stream__' INTO '/Volumes/c/s/v/file'", "/* upload */ PUT\"__input_stream__\" INTO '/Volumes/c/s/v/file'", From 5e78ed690bfcc765507b5be8648fcc58d0b381a4 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:50:11 +0000 Subject: [PATCH 30/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index f67811cf1..d0a7d0678 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -2006c55d1b67903ca7b6090d0b84a2be73fcb82f +892ee5698302b4e34db5ea926baa0354e9c229da From dbf9701f3a6382e7da3e7e79f32d68ce1a9dcb81 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:59:56 +0000 Subject: [PATCH 31/40] fix(staging): reject byte-valued local path entries Signed-off-by: Aakash Saravanan --- examples/staging_ingestion.py | 3 ++- src/databricks/sql/backend/kernel/client.py | 8 +++++--- tests/unit/test_kernel_client.py | 11 ++++++++++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/examples/staging_ingestion.py b/examples/staging_ingestion.py index 51b6eff0a..4a0dce397 100644 --- a/examples/staging_ingestion.py +++ b/examples/staging_ingestion.py @@ -30,7 +30,8 @@ 2. Set the FILEPATH constant to the path of a file that will be uploaded (this example assumes its a CSV file) 3. Run this file -Note: staging_allowed_local_path accepts a path string or a list of path strings. +Note: staging_allowed_local_path accepts a path string or a list of strings or +os.PathLike objects that resolve to strings. """ INGESTION_USER = "some.user@example.com" diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 97c4caee1..3d1375e49 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -582,17 +582,19 @@ def execute_command( allowed_paths, (str, list) ): raise ProgrammingError( - "staging_allowed_local_path must be a string or a list of paths" + "staging_allowed_local_path must be a string or a list of strings or os.PathLike objects" ) if isinstance(allowed_paths, str): allowed_paths = [allowed_paths] try: normalized_paths = [ - os.fsdecode(path) for path in allowed_paths or [] + os.fspath(path) for path in allowed_paths or [] ] + if any(not isinstance(path, str) for path in normalized_paths): + raise TypeError("Local paths must resolve to strings") except (TypeError, ValueError) as exc: raise ProgrammingError( - "staging_allowed_local_path must be a string or a list of paths" + "staging_allowed_local_path must be a string or a list of strings or os.PathLike objects" ) from exc set_paths(normalized_paths) elif staging_verb is not None: diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 3a12d2c74..6c56d9df0 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -862,7 +862,16 @@ def test_execute_command_forwards_allowed_local_paths( @pytest.mark.parametrize( "allowed", - [123, [object()], Path("/tmp/files"), ("/tmp/files",), (), {"/tmp/files": True}], + [ + 123, + [object()], + [b"/tmp/files"], + ["/tmp/files", b"/tmp/other"], + Path("/tmp/files"), + ("/tmp/files",), + (), + {"/tmp/files": True}, + ], ) @pytest.mark.parametrize( "operation", From 3efbcc9e9b95d3947bda4c5ccd6460a113473c1f Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:01:59 +0000 Subject: [PATCH 32/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index d0a7d0678..e4f032405 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -892ee5698302b4e34db5ea926baa0354e9c229da +00e1701a9b14082dba9e4d4d7d8df378971e1818 From a8664be9a1078218a1718fed02db0e94cbe9be9c Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:09:52 +0000 Subject: [PATCH 33/40] refactor(staging): share local path validation message Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/client.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 3d1375e49..f97fe8983 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -73,6 +73,10 @@ # http_path). Forwarding either is redundant and trips the kernel's # per-request skip-and-warn. _KERNEL_MANAGED_HEADERS = frozenset({"authorization", "x-databricks-org-id"}) +_STAGING_PATHS_TYPE_ERROR = ( + "staging_allowed_local_path must be a string or a list of " + "strings or os.PathLike objects" +) def _strip_leading_sql_comments(sql: str) -> str: @@ -581,9 +585,7 @@ def execute_command( if allowed_paths is not None and not isinstance( allowed_paths, (str, list) ): - raise ProgrammingError( - "staging_allowed_local_path must be a string or a list of strings or os.PathLike objects" - ) + raise ProgrammingError(_STAGING_PATHS_TYPE_ERROR) if isinstance(allowed_paths, str): allowed_paths = [allowed_paths] try: @@ -593,9 +595,7 @@ def execute_command( if any(not isinstance(path, str) for path in normalized_paths): raise TypeError("Local paths must resolve to strings") except (TypeError, ValueError) as exc: - raise ProgrammingError( - "staging_allowed_local_path must be a string or a list of strings or os.PathLike objects" - ) from exc + raise ProgrammingError(_STAGING_PATHS_TYPE_ERROR) from exc set_paths(normalized_paths) elif staging_verb is not None: raise NotSupportedError( From 5df354ace25c9a184c2528b54e63ad9885916e09 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:12:00 +0000 Subject: [PATCH 34/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index e4f032405..a3bdb6dd7 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -00e1701a9b14082dba9e4d4d7d8df378971e1818 +e78a4b1e58562e03a06c9166cc76bde5842f1de0 From 04a52c5e47335da36670ab81aee368be861332d7 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:25:36 +0000 Subject: [PATCH 35/40] fix(staging): recognize transfers after nested SQL comments Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/client.py | 12 +++++++++--- tests/unit/test_kernel_client.py | 8 ++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index f97fe8983..27195ec37 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -97,9 +97,15 @@ def _strip_leading_sql_comments(sql: str) -> str: nl = sql.find("\n", i) i = n if nl == -1 else nl + 1 elif sql.startswith("/*", i): - # Block comment: skip to closing */ (or end if unterminated). - close = sql.find("*/", i + 2) - i = n if close == -1 else close + 2 + depth = 1 + i += 2 + for delimiter in re.finditer(r"/\*|\*/", sql[i:]): + depth += 1 if delimiter.group() == "/*" else -1 + if depth == 0: + i += delimiter.end() + break + else: + i = n else: break return sql[i:] diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 6c56d9df0..a67f77950 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -822,6 +822,10 @@ def test_execute_command_forwards_row_limit(row_limit): ("PUT '/tmp/files/input' INTO '/Volumes/c/s/v/input'", True), ("GET '/Volumes/c/s/v/input' TO '/tmp/files/output'", True), ("/* download */ GET '/Volumes/c/s/v/input' TO '/tmp/files/output'", True), + ( + "/* outer /* inner */ outer */ GET '/Volumes/c/s/v/input' TO '/tmp/files/output'", + True, + ), ("REMOVE '/Volumes/c/s/v/input'", False), ("SELECT 1", False), ], @@ -918,6 +922,9 @@ def execute(): ("-- upload the file\nPUT '/f' INTO '/v'", "PUT"), ("/* staging */ PUT '/f' INTO '/v'", "PUT"), ("/* c1 */\n -- c2\n get '/v' to '/f'", "GET"), + ("/* outer /* inner */ outer */ PUT '/f' INTO '/v'", "PUT"), + ("/* a /* b /* c */ b */ a */ GET '/v' TO '/f'", "GET"), + ("/* outer /* closed */ PUT '/f' INTO '/v'", None), (" \n\t PUT '/f' INTO '/v'", "PUT"), ("SELECT 'GET' AS x", None), ("SELECT * FROM puts", None), @@ -944,6 +951,7 @@ def test_staging_verb_matches_only_the_leading_statement(operation, expected): "PUT '__input_stream__'INTO '/Volumes/c/s/v/file'", 'PUT"__input_stream__"INTO "/Volumes/c/s/v/file"', "/* upload */ PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", + "/* outer /* inner */ outer */ PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", "PUT'__input_stream__' INTO '/Volumes/c/s/v/file'", "/* upload */ PUT\"__input_stream__\" INTO '/Volumes/c/s/v/file'", ], From e9da44ada9f9fc502c226894e86a3ee7a5b69199 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:34:14 +0000 Subject: [PATCH 36/40] docs: keep native release coordination in PR metadata Signed-off-by: Aakash Saravanan --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 633016790..947b1c996 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,6 @@ requests-kerberos = {version = "^0.15.0", optional = true} # Floor is 1.4.0 (``^1.4.0`` == ``>=1.4.0,<2.0.0``), allowing compatible # 1.x releases while keeping the next major upgrade deliberate. The kernel's # pyarrow constraint remains >=23.0.1,<24. -# TODO: gate merge/release on https://github.com/databricks/databricks-sql-kernel/issues/443. databricks-sql-kernel = {version = "^1.4.0", optional = true, python = ">=3.10"} From 8ab0fa6481c339e353ac448c164204c6788ef980 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:41:50 +0000 Subject: [PATCH 37/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index a3bdb6dd7..3f36a79f5 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -e78a4b1e58562e03a06c9166cc76bde5842f1de0 +f139344c9c8cf9f3224f6d3068adba08632a0c32 From 70f669549e99189b2d9f4fdc9a886ecb095688f2 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:51:02 +0000 Subject: [PATCH 38/40] fix(staging): recognize streaming input after SQL comments Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/client.py | 6 +++--- tests/unit/test_kernel_client.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 27195ec37..a8b386570 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -572,9 +572,9 @@ def execute_command( try: stripped_sql = _strip_leading_sql_comments(operation) staging_verb = _staging_verb(stripped_sql) - if re.match( - r"""PUT\s*(['"])__input_stream__\1""", - stripped_sql, + if staging_verb == "PUT" and re.match( + r"""(['"])__input_stream__\1""", + _strip_leading_sql_comments(stripped_sql[3:]), re.IGNORECASE, ): raise NotSupportedError( diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index a67f77950..7a119face 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -953,6 +953,8 @@ def test_staging_verb_matches_only_the_leading_statement(operation, expected): "/* upload */ PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", "/* outer /* inner */ outer */ PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", "PUT'__input_stream__' INTO '/Volumes/c/s/v/file'", + "PUT /* upload */ '__input_stream__' INTO '/Volumes/c/s/v/file'", + "PUT\n-- upload\n'__input_stream__' INTO '/Volumes/c/s/v/file'", "/* upload */ PUT\"__input_stream__\" INTO '/Volumes/c/s/v/file'", ], ) From 43fedf4ad32fe1bae8ece429c79a55a993b692ae Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 03:05:49 +0000 Subject: [PATCH 39/40] fix(volume): preserve OS errors and exact stream token matching Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/_errors.py | 20 +++++++--- src/databricks/sql/backend/kernel/client.py | 1 - tests/unit/test_kernel_client.py | 41 ++++++++++++++++++++ 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index 73d85e6f8..57f2b68ef 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -17,7 +17,8 @@ ``OperationalError`` here. Local missing transfer files use the built-in ``FileNotFoundError`` class; -the OS ``errno`` and ``filename`` attributes are unavailable. +the original OS error code is preserved when available. The ``filename`` +attribute is unavailable. These primitives live in their own module so both ``client.py`` (which orchestrates PyO3 calls) and ``result_set.py`` (which calls @@ -41,6 +42,7 @@ from __future__ import annotations import logging +import os from typing import Union from databricks.sql.exc import ( @@ -160,19 +162,26 @@ def reraise_kernel_error( "diagnostic-info": getattr(exc, "diagnostic_info", None), } message = getattr(exc, "message", str(exc)) + os_error_code = getattr(exc, "os_error_code", None) + if os_error_code is None: + file_error_args = (message,) + elif os.name == "nt": + file_error_args = (0, message, None, os_error_code) + else: + file_error_args = (os_error_code, message) new: Union[Error, OSError] # Local missing files have no HTTP status; remote missing files carry 404. if ( error_code == "FILE_TRANSFER_PERMISSION_ERROR" and getattr(exc, "http_status", None) is None ): - new = PermissionError(message) + new = PermissionError(*file_error_args) elif ( file_transfer and code == "NotFound" and getattr(exc, "http_status", None) is None ): - new = FileNotFoundError(message) + new = FileNotFoundError(*file_error_args) else: new = cls(message, context) @@ -181,6 +190,7 @@ def reraise_kernel_error( "sql_state", "error_code", "vendor_code", + "os_error_code", "http_status", "retryable", "query_id", @@ -199,8 +209,8 @@ def wrap_kernel_exception(what: str, exc: BaseException) -> "Union[Error, OSErro """Map a PyO3 exception to the connector's public error contract. - ``KernelError`` → mapped class with structured attrs forwarded. - A local missing transfer file maps to ``FileNotFoundError`` without - the OS ``errno`` or ``filename`` attributes. + A local missing transfer file maps to ``FileNotFoundError`` with + its original OS error code when available, without ``filename``. - Already-mapped PEP 249 or local transfer errors → passed through unchanged so nested callers preserve their class and attributes. - Anything else (``TypeError`` / ``ValueError`` / etc. from PyO3 diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index a8b386570..7680dd714 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -575,7 +575,6 @@ def execute_command( if staging_verb == "PUT" and re.match( r"""(['"])__input_stream__\1""", _strip_leading_sql_comments(stripped_sql[3:]), - re.IGNORECASE, ): raise NotSupportedError( "Streaming PUT is not supported by the kernel backend." diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 7a119face..b11e5de00 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -13,6 +13,8 @@ from __future__ import annotations +import errno +import os import sys import types from pathlib import Path @@ -820,6 +822,7 @@ def test_execute_command_forwards_row_limit(row_limit): "operation, uses_paths", [ ("PUT '/tmp/files/input' INTO '/Volumes/c/s/v/input'", True), + ("PUT '__INPUT_STREAM__' INTO '/Volumes/c/s/v/input'", True), ("GET '/Volumes/c/s/v/input' TO '/tmp/files/output'", True), ("/* download */ GET '/Volumes/c/s/v/input' TO '/tmp/files/output'", True), ( @@ -948,6 +951,7 @@ def test_staging_verb_matches_only_the_leading_statement(operation, expected): "query", [ "PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", + "put '__input_stream__' INTO '/Volumes/c/s/v/file'", "PUT '__input_stream__'INTO '/Volumes/c/s/v/file'", 'PUT"__input_stream__"INTO "/Volumes/c/s/v/file"', "/* upload */ PUT '__input_stream__' INTO '/Volumes/c/s/v/file'", @@ -2750,6 +2754,43 @@ def test_file_transfer_error_matches_thrift_exception( assert result.error_code == error_code assert result.code == code assert result.http_status == http_status + if issubclass(expected, OSError): + assert result.errno is None + + +@pytest.mark.parametrize( + "code,error_code,raw,expected", + [ + ("NotFound", "FILE_TRANSFER_ERROR", errno.ENOENT, FileNotFoundError), + ( + "PermissionDenied", + "FILE_TRANSFER_PERMISSION_ERROR", + errno.EACCES, + PermissionError, + ), + ( + "PermissionDenied", + "FILE_TRANSFER_PERMISSION_ERROR", + errno.EPERM, + PermissionError, + ), + ], +) +def test_local_transfer_error_preserves_os_code(code, error_code, raw, expected): + error = _FakeKernelError(code=code, message="File transfer failed") + error.error_code = error_code + if os.name == "nt": + raw = 2 if expected is FileNotFoundError else 5 + error.os_error_code = raw + result = kernel_client._reraise_kernel_error(error) + assert type(result) is expected + assert result.os_error_code == raw + assert result.filename is None + if os.name == "nt": + assert result.winerror == raw + assert result.errno == (errno.ENOENT if raw == 2 else errno.EACCES) + else: + assert result.errno == raw @pytest.mark.parametrize( From 73990ee48d3855daccf2262fd7e6db05715e8f35 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 03:07:59 +0000 Subject: [PATCH 40/40] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 3f36a79f5..0c7999481 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -f139344c9c8cf9f3224f6d3068adba08632a0c32 +9f7bf932aacbd76ff67c52cc1cd84082af2e5ab0