From 78e18c7c4008f486cda3a8e29afc362095c4438e Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 22:42:38 +0000 Subject: [PATCH 01/44] feat(volume): enable native file transfers through ExecContext Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- connection.go | 8 ++--- internal/backend/backend.go | 2 ++ internal/backend/kernel/backend.go | 13 ++------ .../kernel/include/databricks_kernel.h | 4 +++ internal/backend/kernel/kernel_test.go | 24 +++----------- internal/backend/kernel/operation.go | 31 +++++++++++++++---- internal/backend/kernel/staging.go | 14 ++------- 8 files changed, 46 insertions(+), 52 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index d025be44..df63b0f3 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -6f4b6d607a61bdde61dcda2cf82bc8e80fd32cb9 +d41cc3657748966d8b69a802bc40e8d019022ceb diff --git a/connection.go b/connection.go index 19de7f76..548fd589 100644 --- a/connection.go +++ b/connection.go @@ -148,7 +148,7 @@ func (c *conn) ExecContext(ctx context.Context, query string, args []driver.Name // Capture execution start time for telemetry before running the query executeStart := time.Now() - op, err := c.runQuery(ctx, query, args) + op, err := c.runQuery(ctx, query, args, true) // A nil op means execution never reached the backend (parameter conversion // failed): no server statement to close or measure. Return the already-wrapped // error directly. @@ -278,7 +278,7 @@ func (c *conn) QueryContext(ctx context.Context, query string, args []driver.Nam // Capture execution start time for telemetry before running the query executeStart := time.Now() - op, err := c.runQuery(ctx, query, args) + op, err := c.runQuery(ctx, query, args, false) // A nil op means execution never reached the backend (parameter conversion // failed): no statement id, no telemetry to emit. Log and return the // already-wrapped error directly. @@ -437,12 +437,12 @@ func (c *conn) QueryContext(ctx context.Context, query string, args []driver.Nam // which case the wrapped error is returned directly. Once the backend is called, // backend.Execute guarantees a non-nil Operation, so callers guard only for the // pre-backend nil. -func (c *conn) runQuery(ctx context.Context, query string, args []driver.NamedValue) (backend.Operation, error) { +func (c *conn) runQuery(ctx context.Context, query string, args []driver.NamedValue, transferFiles bool) (backend.Operation, error) { params, err := convertNamedValuesToParams(args) if err != nil { return nil, dbsqlerrint.NewExecutionError(ctx, dbsqlerr.ErrQueryExecution, err, nil) } - return c.backend.Execute(ctx, backend.ExecRequest{Query: query, Params: params}) + return c.backend.Execute(ctx, backend.ExecRequest{Query: query, Params: params, TransferFiles: transferFiles}) } // enrichQueryId sets the operation's statement id as the context queryId. A diff --git a/internal/backend/backend.go b/internal/backend/backend.go index 37fa1a43..ce8686fe 100644 --- a/internal/backend/backend.go +++ b/internal/backend/backend.go @@ -119,6 +119,8 @@ type ExecRequest struct { // dbsql.Parameter/SqlType types; each backend maps these neutral params to its // own wire type. Params []Param + // QueryContext exposes transfer instructions without touching local files. + TransferFiles bool } // Param is one bound query parameter in backend-neutral form: the name (empty diff --git a/internal/backend/kernel/backend.go b/internal/backend/kernel/backend.go index f8e6ebb7..9a485326 100644 --- a/internal/backend/kernel/backend.go +++ b/internal/backend/kernel/backend.go @@ -71,7 +71,7 @@ import ( "sync/atomic" "time" - dbsqlerr "github.com/databricks/databricks-sql-go/errors" + "github.com/databricks/databricks-sql-go/driverctx" "github.com/databricks/databricks-sql-go/internal/backend" ) @@ -890,15 +890,8 @@ func (k *KernelBackend) SessionID() string { return k.sessionID } // parameters (req.Params) are bound onto the statement in execute (see // bindParams). func (k *KernelBackend) Execute(ctx context.Context, req backend.ExecRequest) (backend.Operation, error) { - // Staging (Unity Catalog volume PUT/GET/REMOVE) needs a local file transfer the - // kernel path can't perform, and the kernel C ABI surfaces no IsStagingOperation - // signal to drive conn.execStagingOperation. Reject it here rather than let - // IsStaging return false and report success with no file moved (silent data loss). - // (Bound parameters, by contrast, ARE now supported — bound in execute via the - // kernel's raw-param C ABI.) - if isStagingStatement(req.Query) { - return &kernelOp{}, fmt.Errorf("databricks: staging operations (PUT/GET/REMOVE on a volume) are %w; "+ - "use the default (Thrift) backend", dbsqlerr.ErrNotSupportedByKernel) + if req.TransferFiles && isStagingStatement(req.Query) && len(driverctx.StagingPathsFromContext(ctx)) == 0 { + return &kernelOp{}, fmt.Errorf("databricks: staging ctx must be provided") } return k.execute(ctx, req) } diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index 965c09b6..a91d2af7 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -688,6 +688,10 @@ KernelStatusCode kernel_session_rollback(kernel_session_t* session); * so a reused statement handle starts each query with a fresh parameter set. */ KernelStatusCode kernel_statement_set_sql(kernel_statement_t* stmt, const char* sql); +/** Configure a JSON array of allowed local paths for SQL file transfers. */ +KernelStatusCode kernel_statement_set_staging_allowed_local_paths( + kernel_statement_t* stmt, const char* paths_json, bool retain_result); + /* Typed positional parameter binding (1-based ordinal); the kernel infers the * SEA wire type from the C type. These are the "typed params" the * kernel_statement_bind_parameter contract refers to: within one statement they diff --git a/internal/backend/kernel/kernel_test.go b/internal/backend/kernel/kernel_test.go index cd379412..4c0c5b82 100644 --- a/internal/backend/kernel/kernel_test.go +++ b/internal/backend/kernel/kernel_test.go @@ -531,30 +531,16 @@ func TestExecuteHandleLessOpContract(t *testing.T) { } } -// TestExecuteRejectsStaging drives Execute with a staging statement (not just the -// isStagingStatement detector in isolation) to pin the detector→Execute wiring: a -// refactor that dropped or reordered the check would silently reopen the -// silent-no-op data-loss path. Mirrors TestExecuteRejectsParams. -func TestExecuteRejectsStaging(t *testing.T) { +func TestExecuteRequiresStagingContext(t *testing.T) { k := &KernelBackend{} op, err := k.Execute(context.Background(), backend.ExecRequest{ - Query: "PUT '/tmp/f' INTO '/Volumes/main/s/e/f.csv'", + Query: "PUT '/tmp/f' INTO '/Volumes/main/s/v/f'", TransferFiles: true, }) - if err == nil { - t.Fatal("expected an error for a staging statement, got nil") - } - if !errors.Is(err, dbsqlerr.ErrNotSupportedByKernel) { - t.Errorf("staging rejection should wrap ErrNotSupportedByKernel, got %v", err) + if err == nil || !strings.Contains(err.Error(), "staging ctx must be provided") { + t.Fatalf("expected missing staging context error, got %v", err) } if op == nil { - t.Fatal("Execute must return a non-nil Operation per the Backend contract") - } - closed, closeErr := op.Close(context.Background()) - if closeErr != nil { - t.Errorf("Close error = %v, want nil", closeErr) - } - if closed { - t.Error("Close on a handle-less op must report closed=false (no CLOSE_STATEMENT)") + t.Fatal("expected non-nil operation") } } diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index e2f355f0..54eb0546 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -11,6 +11,7 @@ import "C" import ( "context" "database/sql/driver" + "encoding/json" "errors" "fmt" "sync" @@ -35,8 +36,7 @@ import ( // client-timeout path, let an in-flight cancel finish in the background so // it cannot extend the client deadline // -// Binds any query parameters (bindParams) before executing; staging statements -// are rejected up front by Execute, so none reach here. +// Binds query parameters and configures local file access before executing. func (k *KernelBackend) execute(ctx context.Context, req backend.ExecRequest) (backend.Operation, error) { // Log the SQL length, not the text: query bodies can carry PII/secrets in // WHERE/INSERT/SET, and this goes to stderr. Matches the driver's own @@ -78,6 +78,27 @@ func (k *KernelBackend) execute(ctx context.Context, req backend.ExecRequest) (b } sql.free() + if req.TransferFiles { + paths := driverctx.StagingPathsFromContext(ctx) + if paths == nil { + paths = []string{} + } + encoded, err := json.Marshal(paths) + if err != nil { + C.kernel_statement_close(stmt) + return &kernelOp{}, err + } + value := newCStr(string(encoded)) + err = call(func() C.KernelStatusCode { + return C.kernel_statement_set_staging_allowed_local_paths(stmt, value.c, false) + }) + value.free() + if err != nil { + C.kernel_statement_close(stmt) + return &kernelOp{}, fmt.Errorf("kernel: allowed local paths: %w", toStatementError(err)) + } + } + // Bind parameters. The driver hands us backend.Param{Name, Type, Value}: the // value is already stringified, Type is the Databricks SQL type name, and a nil // Value is SQL NULL (Type "VOID"). Each maps 1:1 onto @@ -384,10 +405,8 @@ func (o *kernelOp) Results(ctx context.Context, callbacks *dbsqlrows.TelemetryCa return newKernelRows(ctx, o, stream, callbacks) } -// IsStaging reports whether this is a staging (PUT/GET/REMOVE) operation. Always -// false on the kernel path: staging statements are rejected up front in -// Execute (isStagingStatement), so no kernelOp is ever produced for one — there -// is nothing for conn.execStagingOperation to act on here. +// IsStaging is false because the kernel completes configured file transfers +// before returning the result. func (o *kernelOp) IsStaging(ctx context.Context) (bool, error) { return false, nil } // Close best-effort closes the executed statement and its statement handle. It is diff --git a/internal/backend/kernel/staging.go b/internal/backend/kernel/staging.go index c614169c..99f5a8c5 100644 --- a/internal/backend/kernel/staging.go +++ b/internal/backend/kernel/staging.go @@ -6,18 +6,8 @@ import "strings" // Staging detection is pure Go (string inspection of the SQL), so keeping it // untagged lets its test run under CGO_ENABLED=0. -// isStagingStatement reports whether sql is a Unity Catalog volume staging -// command (PUT / GET / REMOVE). The Thrift backend learns a statement is staging -// from server result metadata (IsStagingOperation); the kernel C ABI exposes no -// such signal, and the kernel path cannot perform the local file transfer these -// commands require. We therefore detect them from the SQL and reject at execute -// time (see Execute) rather than silently returning success with no file moved — -// which is what returning false from IsStaging would do. Case-insensitive; skips -// leading whitespace AND leading SQL comments (`-- ...` and `/* ... */`), so a -// comment-prefixed staging command is still caught rather than slipping through -// as a silent no-op. Errs toward matching only a clear leading keyword — a false -// negative just yields the kernel's normal "unsupported syntax" server error, -// whereas a false positive would reject a legitimate query. +// Missing staging context must fail for volume commands even when REMOVE +// does not need a local file path. func isStagingStatement(sql string) bool { s := stripLeadingSQLComments(sql) for _, kw := range []string{"PUT", "GET", "REMOVE"} { From 04a0591bd0ca5cc15ded4fc20d12884017d89290 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 22:56:48 +0000 Subject: [PATCH 02/44] fix(volume): align missing-context errors and document native transfers Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 3 +++ CONNECTION_PARAMETERS.md | 8 +++++++ KERNEL_REV | 2 +- examples/kernel/main.go | 19 ++++++++------- internal/backend/kernel/backend.go | 3 ++- .../kernel/include/databricks_kernel.h | 2 +- internal/backend/kernel/kernel_test.go | 24 ++++++++++++------- 7 files changed, 41 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b736e9a..b2e148e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Release History +## Unreleased +- Support SQL PUT, GET, and REMOVE through kernel-backed `ExecContext` with a staging context. `QueryContext` continues to return transfer instructions. + ## v1.19.0 (2026-11-04) - Upgrade the SQL kernel bindings to v1.4.0. - Upgrade go-oidc to v3.21.0 (go-jose v4.1.6) and x/oauth2 to v0.36.0 to address go-jose/v3 security findings (databricks/databricks-sql-go#526). diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 35a4ad0e..26856058 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -237,3 +237,11 @@ unset), kernel telemetry follows its server feature flag, and `telemetry_batch_s The Go wrapper telemetry interceptor is skipped on the kernel path so it does not duplicate kernel-owned telemetry for the same connection and statements. See [`telemetry/DESIGN.md`](./telemetry/DESIGN.md). + +## Volume file transfers + +Both backends support SQL PUT, GET, and REMOVE through `ExecContext` with +`driverctx.NewContextWithStagingInfo(ctx, allowedLocalPaths)`. The context must +contain at least one allowed path, including for REMOVE. `QueryContext` returns +the transfer instructions for callers that handle file transfers themselves. +Kernel builds require the revision in `KERNEL_REV` and matching native bindings. diff --git a/KERNEL_REV b/KERNEL_REV index df63b0f3..31390e27 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -d41cc3657748966d8b69a802bc40e8d019022ceb +bc39c7921acecd95e4eea0740becb883867a834b diff --git a/examples/kernel/main.go b/examples/kernel/main.go index 51d99c31..018ff28b 100644 --- a/examples/kernel/main.go +++ b/examples/kernel/main.go @@ -18,6 +18,9 @@ // DATABRICKS_ACCESSTOKEN personal access token // DATABRICKS_HTTPPATH warehouse http path (or set DATABRICKS_WAREHOUSEID) // DATABRICKS_WAREHOUSEID bare warehouse id (kernel routes by this when set) +// +// Optional upload: VOLUME_UPLOAD_SOURCE is a local file and +// VOLUME_UPLOAD_DESTINATION is its /Volumes/catalog/schema/volume/file destination. package main import ( @@ -27,9 +30,11 @@ import ( "fmt" "log" "os" + "strings" "time" dbsql "github.com/databricks/databricks-sql-go" + "github.com/databricks/databricks-sql-go/driverctx" dbsqlerr "github.com/databricks/databricks-sql-go/errors" "github.com/joho/godotenv" ) @@ -88,13 +93,11 @@ func main() { } fmt.Printf("server version: %s\n", version) - // Options the kernel can't honor are rejected with an error wrapping - // ErrNotSupportedByKernel, never silently ignored. Staging (PUT/GET/REMOVE on a - // Unity Catalog volume) is one such feature, rejected at execute. - _, err = db.ExecContext(ctx, "PUT '/tmp/local.csv' INTO '/Volumes/main/default/vol/f.csv' OVERWRITE") - if errors.Is(err, dbsqlerr.ErrNotSupportedByKernel) { - fmt.Println("staging is not supported on the kernel backend (use Thrift for staging)") - } else if err != nil { - fmt.Printf("staging: %v\n", err) + if source, destination := os.Getenv("VOLUME_UPLOAD_SOURCE"), os.Getenv("VOLUME_UPLOAD_DESTINATION"); source != "" && destination != "" { + stagingCtx := driverctx.NewContextWithStagingInfo(ctx, []string{source}) + quote := func(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } + if _, err := db.ExecContext(stagingCtx, "PUT "+quote(source)+" INTO "+quote(destination)+" OVERWRITE"); err != nil { + log.Fatalf("upload: %v", err) + } } } diff --git a/internal/backend/kernel/backend.go b/internal/backend/kernel/backend.go index 9a485326..c82a3f00 100644 --- a/internal/backend/kernel/backend.go +++ b/internal/backend/kernel/backend.go @@ -73,6 +73,7 @@ import ( "github.com/databricks/databricks-sql-go/driverctx" "github.com/databricks/databricks-sql-go/internal/backend" + dbsqlerrint "github.com/databricks/databricks-sql-go/internal/errors" ) // kernelSessionSeq is a process-wide monotonic counter for kernel session ids. @@ -891,7 +892,7 @@ func (k *KernelBackend) SessionID() string { return k.sessionID } // bindParams). func (k *KernelBackend) Execute(ctx context.Context, req backend.ExecRequest) (backend.Operation, error) { if req.TransferFiles && isStagingStatement(req.Query) && len(driverctx.StagingPathsFromContext(ctx)) == 0 { - return &kernelOp{}, fmt.Errorf("databricks: staging ctx must be provided") + return &kernelOp{}, dbsqlerrint.NewDriverError(ctx, "staging ctx must be provided.", nil) } return k.execute(ctx, req) } diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index a91d2af7..f1a40844 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -688,7 +688,7 @@ KernelStatusCode kernel_session_rollback(kernel_session_t* session); * so a reused statement handle starts each query with a fresh parameter set. */ KernelStatusCode kernel_statement_set_sql(kernel_statement_t* stmt, const char* sql); -/** Configure a JSON array of allowed local paths for SQL file transfers. */ +/** Configure allowed local paths; retain_result keeps instructions after transfer. */ KernelStatusCode kernel_statement_set_staging_allowed_local_paths( kernel_statement_t* stmt, const char* paths_json, bool retain_result); diff --git a/internal/backend/kernel/kernel_test.go b/internal/backend/kernel/kernel_test.go index 4c0c5b82..e1fd5a90 100644 --- a/internal/backend/kernel/kernel_test.go +++ b/internal/backend/kernel/kernel_test.go @@ -532,15 +532,21 @@ func TestExecuteHandleLessOpContract(t *testing.T) { } func TestExecuteRequiresStagingContext(t *testing.T) { - k := &KernelBackend{} - op, err := k.Execute(context.Background(), backend.ExecRequest{ - Query: "PUT '/tmp/f' INTO '/Volumes/main/s/v/f'", TransferFiles: true, - }) - if err == nil || !strings.Contains(err.Error(), "staging ctx must be provided") { - t.Fatalf("expected missing staging context error, got %v", err) - } - if op == nil { - t.Fatal("expected non-nil operation") + for _, query := range []string{"PUT '/tmp/f' INTO '/Volumes/main/s/v/f'", "REMOVE '/Volumes/main/s/v/f'", "/* upload */ PUT '/tmp/f' INTO '/Volumes/main/s/v/f'"} { + t.Run(query, func(t *testing.T) { + k := &KernelBackend{} + op, err := k.Execute(context.Background(), backend.ExecRequest{Query: query, TransferFiles: true}) + if err == nil || !strings.Contains(err.Error(), "staging ctx must be provided") { + t.Fatalf("expected missing staging context error, got %v", err) + } + if op == nil { + t.Fatal("expected non-nil operation") + } + closed, closeErr := op.Close(context.Background()) + if closed || closeErr != nil { + t.Fatalf("Close = %v, %v", closed, closeErr) + } + }) } } From 39733aab7e1f2cf140ca00b4ccc272a77fe696ec Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:09:16 +0000 Subject: [PATCH 03/44] test(volume): cover execution mode and staging context Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- connection_test.go | 21 +++++++++++++++++++++ examples/kernel/main.go | 2 +- internal/backend/backend.go | 3 ++- internal/backend/kernel/kernel_test.go | 13 ++++++++++++- 5 files changed, 37 insertions(+), 4 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index 31390e27..a3d86de0 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -bc39c7921acecd95e4eea0740becb883867a834b +ebb8204b36951b751dd837b2c41ced8c821b2d94 diff --git a/connection_test.go b/connection_test.go index 59be9150..f013077f 100644 --- a/connection_test.go +++ b/connection_test.go @@ -18,6 +18,7 @@ import ( "github.com/pkg/errors" dbsqlerr "github.com/databricks/databricks-sql-go/errors" + "github.com/databricks/databricks-sql-go/internal/backend" thriftbackend "github.com/databricks/databricks-sql-go/internal/backend/thrift" "github.com/databricks/databricks-sql-go/internal/cli_service" "github.com/databricks/databricks-sql-go/internal/client" @@ -974,3 +975,23 @@ func TestConn_handleStagingRetry(t *testing.T) { assert.Less(t, elapsed, 2*time.Second, "context cancel should abort PUT retry backoff promptly") }) } + +type transferCaptureBackend struct { + backend.Backend + transferFiles []bool +} + +func (b *transferCaptureBackend) Execute(_ context.Context, req backend.ExecRequest) (backend.Operation, error) { + b.transferFiles = append(b.transferFiles, req.TransferFiles) + return nil, fmt.Errorf("capture request") +} + +func TestConnectionTransferMode(t *testing.T) { + capture := &transferCaptureBackend{} + connection := &conn{cfg: config.WithDefaults(), backend: capture} + _, execErr := connection.ExecContext(context.Background(), "REMOVE '/Volumes/c/s/v/file'", nil) + _, queryErr := connection.QueryContext(context.Background(), "GET '/Volumes/c/s/v/file' TO '/tmp/file'", nil) + assert.Error(t, execErr) + assert.Error(t, queryErr) + assert.Equal(t, []bool{true, false}, capture.transferFiles) +} diff --git a/examples/kernel/main.go b/examples/kernel/main.go index 018ff28b..37e40385 100644 --- a/examples/kernel/main.go +++ b/examples/kernel/main.go @@ -95,7 +95,7 @@ func main() { if source, destination := os.Getenv("VOLUME_UPLOAD_SOURCE"), os.Getenv("VOLUME_UPLOAD_DESTINATION"); source != "" && destination != "" { stagingCtx := driverctx.NewContextWithStagingInfo(ctx, []string{source}) - quote := func(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } + quote := func(value string) string { return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(value) + "'" } if _, err := db.ExecContext(stagingCtx, "PUT "+quote(source)+" INTO "+quote(destination)+" OVERWRITE"); err != nil { log.Fatalf("upload: %v", err) } diff --git a/internal/backend/backend.go b/internal/backend/backend.go index ce8686fe..d6667190 100644 --- a/internal/backend/backend.go +++ b/internal/backend/backend.go @@ -119,7 +119,8 @@ type ExecRequest struct { // dbsql.Parameter/SqlType types; each backend maps these neutral params to its // own wire type. Params []Param - // QueryContext exposes transfer instructions without touching local files. + // TransferFiles asks the backend to complete SQL file transfers. + // When false, transfer instructions remain available to the caller. TransferFiles bool } diff --git a/internal/backend/kernel/kernel_test.go b/internal/backend/kernel/kernel_test.go index e1fd5a90..1ec2f2c9 100644 --- a/internal/backend/kernel/kernel_test.go +++ b/internal/backend/kernel/kernel_test.go @@ -532,7 +532,7 @@ func TestExecuteHandleLessOpContract(t *testing.T) { } func TestExecuteRequiresStagingContext(t *testing.T) { - for _, query := range []string{"PUT '/tmp/f' INTO '/Volumes/main/s/v/f'", "REMOVE '/Volumes/main/s/v/f'", "/* upload */ PUT '/tmp/f' INTO '/Volumes/main/s/v/f'"} { + for _, query := range []string{"GET '/Volumes/main/s/v/f' TO '/tmp/f'", "PUT '/tmp/f' INTO '/Volumes/main/s/v/f'", "REMOVE '/Volumes/main/s/v/f'", "/* upload */ PUT '/tmp/f' INTO '/Volumes/main/s/v/f'"} { t.Run(query, func(t *testing.T) { k := &KernelBackend{} op, err := k.Execute(context.Background(), backend.ExecRequest{Query: query, TransferFiles: true}) @@ -550,6 +550,17 @@ func TestExecuteRequiresStagingContext(t *testing.T) { } } +func TestQueryTransferInstructionsNeedNoStagingContext(t *testing.T) { + k := &KernelBackend{} + op, err := k.Execute(context.Background(), backend.ExecRequest{Query: "GET '/Volumes/c/s/v/file' TO '/tmp/file'", TransferFiles: false}) + if op == nil { + t.Fatal("expected non-nil operation") + } + if err == nil || !strings.Contains(err.Error(), "new_statement") { + t.Fatalf("expected native statement creation, got %v", err) + } +} + // ExecutionError must satisfy the same public contract as the Thrift path so the // errors.Is → errors.As → SqlState()/QueryId() recipe documented in doc.go works // on the kernel backend too (it previously returned a bare *KernelError that From cec0d858a107d30488cce42cfee761ff5af08ed8 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:11:54 +0000 Subject: [PATCH 04/44] docs(volume): keep usage in connection parameter documentation Signed-off-by: Aakash Saravanan --- examples/kernel/main.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/examples/kernel/main.go b/examples/kernel/main.go index 37e40385..51d99c31 100644 --- a/examples/kernel/main.go +++ b/examples/kernel/main.go @@ -18,9 +18,6 @@ // DATABRICKS_ACCESSTOKEN personal access token // DATABRICKS_HTTPPATH warehouse http path (or set DATABRICKS_WAREHOUSEID) // DATABRICKS_WAREHOUSEID bare warehouse id (kernel routes by this when set) -// -// Optional upload: VOLUME_UPLOAD_SOURCE is a local file and -// VOLUME_UPLOAD_DESTINATION is its /Volumes/catalog/schema/volume/file destination. package main import ( @@ -30,11 +27,9 @@ import ( "fmt" "log" "os" - "strings" "time" dbsql "github.com/databricks/databricks-sql-go" - "github.com/databricks/databricks-sql-go/driverctx" dbsqlerr "github.com/databricks/databricks-sql-go/errors" "github.com/joho/godotenv" ) @@ -93,11 +88,13 @@ func main() { } fmt.Printf("server version: %s\n", version) - if source, destination := os.Getenv("VOLUME_UPLOAD_SOURCE"), os.Getenv("VOLUME_UPLOAD_DESTINATION"); source != "" && destination != "" { - stagingCtx := driverctx.NewContextWithStagingInfo(ctx, []string{source}) - quote := func(value string) string { return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(value) + "'" } - if _, err := db.ExecContext(stagingCtx, "PUT "+quote(source)+" INTO "+quote(destination)+" OVERWRITE"); err != nil { - log.Fatalf("upload: %v", err) - } + // Options the kernel can't honor are rejected with an error wrapping + // ErrNotSupportedByKernel, never silently ignored. Staging (PUT/GET/REMOVE on a + // Unity Catalog volume) is one such feature, rejected at execute. + _, err = db.ExecContext(ctx, "PUT '/tmp/local.csv' INTO '/Volumes/main/default/vol/f.csv' OVERWRITE") + if errors.Is(err, dbsqlerr.ErrNotSupportedByKernel) { + fmt.Println("staging is not supported on the kernel backend (use Thrift for staging)") + } else if err != nil { + fmt.Printf("staging: %v\n", err) } } From 484a2f64006d582624103c42e6b326337099aa56 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:14:09 +0000 Subject: [PATCH 05/44] docs(volume): remove obsolete unsupported-transfer example Signed-off-by: Aakash Saravanan --- examples/kernel/main.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/examples/kernel/main.go b/examples/kernel/main.go index 51d99c31..e3041a2d 100644 --- a/examples/kernel/main.go +++ b/examples/kernel/main.go @@ -88,13 +88,4 @@ func main() { } fmt.Printf("server version: %s\n", version) - // Options the kernel can't honor are rejected with an error wrapping - // ErrNotSupportedByKernel, never silently ignored. Staging (PUT/GET/REMOVE on a - // Unity Catalog volume) is one such feature, rejected at execute. - _, err = db.ExecContext(ctx, "PUT '/tmp/local.csv' INTO '/Volumes/main/default/vol/f.csv' OVERWRITE") - if errors.Is(err, dbsqlerr.ErrNotSupportedByKernel) { - fmt.Println("staging is not supported on the kernel backend (use Thrift for staging)") - } else if err != nil { - fmt.Printf("staging: %v\n", err) - } } From 2cbe20877600e84f16d2d4907835f41fa39dcfbb Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:24:08 +0000 Subject: [PATCH 06/44] test(volume): retain the live transfer lifecycle and error cases Signed-off-by: Aakash Saravanan --- CONNECTION_PARAMETERS.md | 2 + KERNEL_REV | 2 +- internal/backend/kernel/operation.go | 1 + kernel_e2e_test.go | 75 ++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 1 deletion(-) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 26856058..5305da10 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -245,3 +245,5 @@ Both backends support SQL PUT, GET, and REMOVE through `ExecContext` with contain at least one allowed path, including for REMOVE. `QueryContext` returns the transfer instructions for callers that handle file transfers themselves. Kernel builds require the revision in `KERNEL_REV` and matching native bindings. + +The credential-gated `TestKernelE2EVolumeFileLifecycle` uses `DATABRICKS_VOLUME_TEST_CATALOG` as a writable catalog and creates and removes an isolated schema and Volume. It covers ExecContext transfers and QueryContext raw instructions. diff --git a/KERNEL_REV b/KERNEL_REV index a3d86de0..7b10abd3 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -ebb8204b36951b751dd837b2c41ced8c821b2d94 +02283500d50e1bcd61007e0cf04a7e1e3d16ea86 diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index 54eb0546..d6d60784 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -95,6 +95,7 @@ func (k *KernelBackend) execute(ctx context.Context, req backend.ExecRequest) (b value.free() if err != nil { C.kernel_statement_close(stmt) + k.evictIfSessionFatal(err) return &kernelOp{}, fmt.Errorf("kernel: allowed local paths: %w", toStatementError(err)) } } diff --git a/kernel_e2e_test.go b/kernel_e2e_test.go index 55feacae..12d279ac 100644 --- a/kernel_e2e_test.go +++ b/kernel_e2e_test.go @@ -8,8 +8,11 @@ import ( "database/sql" "database/sql/driver" "errors" + "fmt" "io" "os" + "path/filepath" + "strings" "sync" "sync/atomic" "testing" @@ -671,3 +674,75 @@ func TestKernelE2EGetArrowBatches(t *testing.T) { t.Errorf("GetArrowBatches row count = %d, want %d", rowCount, want) } } + +func TestKernelE2EVolumeFileLifecycle(t *testing.T) { + catalog := os.Getenv("DATABRICKS_VOLUME_TEST_CATALOG") + if catalog == "" { + t.Skip("set DATABRICKS_VOLUME_TEST_CATALOG to a writable catalog") + } + db := kernelTestDB(t) + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + schema := fmt.Sprintf("driver_volume_%d", time.Now().UnixNano()) + identifier := func(value string) string { return "`" + strings.ReplaceAll(value, "`", "``") + "`" } + literal := func(value string) string { return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(value) + "'" } + namespace := identifier(catalog) + "." + identifier(schema) + if _, err := db.ExecContext(ctx, "CREATE SCHEMA "+namespace); err != nil { + t.Fatal(err) + } + defer func() { + if _, err := db.ExecContext(context.Background(), "DROP SCHEMA "+namespace+" CASCADE"); err != nil { + t.Error(err) + } + }() + if _, err := db.ExecContext(ctx, "CREATE VOLUME "+namespace+".files"); err != nil { + t.Fatal(err) + } + directory := t.TempDir() + source, destination := filepath.Join(directory, "input.bin"), filepath.Join(directory, "output.bin") + payload := bytes.Repeat([]byte{0, 255, 1, 128}, 262144) + if err := os.WriteFile(source, payload, 0600); err != nil { + t.Fatal(err) + } + remote := "/Volumes/" + catalog + "/" + schema + "/files/input.bin" + stagingCtx := driverctx.NewContextWithStagingInfo(ctx, []string{directory}) + if _, err := db.ExecContext(stagingCtx, "PUT "+literal(source)+" INTO "+literal(remote)+" OVERWRITE"); err != nil { + t.Fatal(err) + } + get := "GET " + literal(remote) + " TO " + literal(destination) + rows, err := db.QueryContext(ctx, get) + if err != nil { + t.Fatal(err) + } + if err := rows.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(destination); !os.IsNotExist(err) { + t.Fatalf("raw instructions changed local destination: %v", err) + } + if _, err := db.ExecContext(stagingCtx, get); err != nil { + t.Fatal(err) + } + actual, err := os.ReadFile(destination) + if err != nil || !bytes.Equal(actual, payload) { + t.Fatalf("download differs: %v", err) + } + denied := filepath.Join(t.TempDir(), "denied.bin") + if err := os.WriteFile(denied, payload, 0600); err != nil { + t.Fatal(err) + } + for _, local := range []string{denied, filepath.Join(directory, "missing.bin")} { + if _, err := db.ExecContext(stagingCtx, "PUT "+literal(local)+" INTO "+literal(remote)+" OVERWRITE"); err == nil { + t.Fatalf("PUT accepted unavailable local path %q", local) + } + } + for range 2 { + if _, err := db.ExecContext(stagingCtx, "REMOVE "+literal(remote)); err != nil { + t.Fatal(err) + } + } + if _, err := db.ExecContext(stagingCtx, get); err == nil { + t.Fatal("GET succeeded after REMOVE") + } +} From 28f724d340d6373f8117b2c12a5229516aa53b75 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:36:41 +0000 Subject: [PATCH 07/44] fix(kernel): align native pin and explain transfer opt-in Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- internal/backend/kernel/include/databricks_kernel.h | 3 ++- internal/backend/kernel/operation.go | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index 7b10abd3..ef0cbe60 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -02283500d50e1bcd61007e0cf04a7e1e3d16ea86 +2f317d97212f00d73b89d947f5261a2145ea7a10 diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index f1a40844..0fe272f7 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -688,7 +688,8 @@ KernelStatusCode kernel_session_rollback(kernel_session_t* session); * so a reused statement handle starts each query with a fresh parameter set. */ KernelStatusCode kernel_statement_set_sql(kernel_statement_t* stmt, const char* sql); -/** Configure allowed local paths; retain_result keeps instructions after transfer. */ +/** Configure allowed local paths; null restores caller-managed transfers. + * retain_result keeps instructions after transfer. */ KernelStatusCode kernel_statement_set_staging_allowed_local_paths( kernel_statement_t* stmt, const char* paths_json, bool retain_result); diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index d6d60784..391d6317 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -78,6 +78,7 @@ func (k *KernelBackend) execute(ctx context.Context, req backend.ExecRequest) (b } sql.free() + // An empty list denies local transfers; leaving the option unset returns raw instructions. if req.TransferFiles { paths := driverctx.StagingPathsFromContext(ctx) if paths == nil { From 6cda8ed28d33c9d6a530e7b02ae054c12e9a507c Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:47:06 +0000 Subject: [PATCH 08/44] test(volume): assert local transfer errors and clarify backend contract Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- examples/kernel/main.go | 1 - internal/backend/backend.go | 4 ++-- internal/backend/kernel/staging.go | 4 ++-- kernel_e2e_test.go | 9 ++++++--- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index ef0cbe60..913fb888 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -2f317d97212f00d73b89d947f5261a2145ea7a10 +95f294c8c2c825440191af9f58a9b6071aa68408 diff --git a/examples/kernel/main.go b/examples/kernel/main.go index e3041a2d..e3f19775 100644 --- a/examples/kernel/main.go +++ b/examples/kernel/main.go @@ -87,5 +87,4 @@ func main() { log.Fatalf("query: %v", err) } fmt.Printf("server version: %s\n", version) - } diff --git a/internal/backend/backend.go b/internal/backend/backend.go index d6667190..7bfc239c 100644 --- a/internal/backend/backend.go +++ b/internal/backend/backend.go @@ -119,8 +119,8 @@ type ExecRequest struct { // dbsql.Parameter/SqlType types; each backend maps these neutral params to its // own wire type. Params []Param - // TransferFiles asks the backend to complete SQL file transfers. - // When false, transfer instructions remain available to the caller. + // TransferFiles requests native completion where the backend supports it. + // Other backends leave transfers to the connection layer. TransferFiles bool } diff --git a/internal/backend/kernel/staging.go b/internal/backend/kernel/staging.go index 99f5a8c5..f4dd1499 100644 --- a/internal/backend/kernel/staging.go +++ b/internal/backend/kernel/staging.go @@ -6,8 +6,8 @@ import "strings" // Staging detection is pure Go (string inspection of the SQL), so keeping it // untagged lets its test run under CGO_ENABLED=0. -// Missing staging context must fail for volume commands even when REMOVE -// does not need a local file path. +// isStagingStatement detects PUT, GET, or REMOVE after leading SQL comments, +// ignoring keyword case. func isStagingStatement(sql string) bool { s := stripLeadingSQLComments(sql) for _, kw := range []string{"PUT", "GET", "REMOVE"} { diff --git a/kernel_e2e_test.go b/kernel_e2e_test.go index 12d279ac..ed278c99 100644 --- a/kernel_e2e_test.go +++ b/kernel_e2e_test.go @@ -732,9 +732,12 @@ func TestKernelE2EVolumeFileLifecycle(t *testing.T) { if err := os.WriteFile(denied, payload, 0600); err != nil { t.Fatal(err) } - for _, local := range []string{denied, filepath.Join(directory, "missing.bin")} { - if _, err := db.ExecContext(stagingCtx, "PUT "+literal(local)+" INTO "+literal(remote)+" OVERWRITE"); err == nil { - t.Fatalf("PUT accepted unavailable local path %q", local) + for _, test := range []struct{ local, message string }{ + {denied, "restricted"}, + {filepath.Join(directory, "missing.bin"), "File transfer failed"}, + } { + if _, err := db.ExecContext(stagingCtx, "PUT "+literal(test.local)+" INTO "+literal(remote)+" OVERWRITE"); err == nil || !strings.Contains(err.Error(), test.message) { + t.Fatalf("PUT %q: expected %q, got %v", test.local, test.message, err) } } for range 2 { From 6b45c4c4064747047310e67180d76be77162aa6f Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 11 Nov 2026 23:59:08 +0000 Subject: [PATCH 09/44] test(kernel): verify raw file transfer instruction contents Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- kernel_e2e_test.go | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 913fb888..f164365a 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -95f294c8c2c825440191af9f58a9b6071aa68408 +f68e4e5d1600e987187cf84f58ddf6a61e6a5ba6 diff --git a/kernel_e2e_test.go b/kernel_e2e_test.go index ed278c99..ff93a783 100644 --- a/kernel_e2e_test.go +++ b/kernel_e2e_test.go @@ -715,6 +715,25 @@ func TestKernelE2EVolumeFileLifecycle(t *testing.T) { if err != nil { t.Fatal(err) } + defer rows.Close() + columns, err := rows.Columns() + if err != nil || len(columns) != 4 { + t.Fatalf("raw instruction columns: %v, %v", columns, err) + } + if !rows.Next() { + t.Fatalf("missing raw instruction: %v", rows.Err()) + } + var operation, storageURL, local string + var headers any + if err := rows.Scan(&operation, &storageURL, &headers, &local); err != nil { + t.Fatal(err) + } + if operation != "GET" || local != destination || storageURL == "" { + t.Fatalf("unexpected raw instruction: operation=%q local=%q", operation, local) + } + if rows.Next() || rows.Err() != nil { + t.Fatalf("expected one raw instruction: %v", rows.Err()) + } if err := rows.Close(); err != nil { t.Fatal(err) } From 4c00ab8bbff05d461e9fb2755f026f05fa9cc04e Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:10:10 +0000 Subject: [PATCH 10/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index f164365a..098e9943 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -f68e4e5d1600e987187cf84f58ddf6a61e6a5ba6 +c3e7cf1ba339155c7932c77e39e2821fd918cddf From e540d7a41885e36ed214d9ac4f5a6c524012b5c6 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:19:10 +0000 Subject: [PATCH 11/44] test(kernel): name the staging context guard precisely Signed-off-by: Aakash Saravanan --- internal/backend/kernel/kernel_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backend/kernel/kernel_test.go b/internal/backend/kernel/kernel_test.go index 1ec2f2c9..f6e10f3a 100644 --- a/internal/backend/kernel/kernel_test.go +++ b/internal/backend/kernel/kernel_test.go @@ -550,7 +550,7 @@ func TestExecuteRequiresStagingContext(t *testing.T) { } } -func TestQueryTransferInstructionsNeedNoStagingContext(t *testing.T) { +func TestQuerySkipsStagingContextGuard(t *testing.T) { k := &KernelBackend{} op, err := k.Execute(context.Background(), backend.ExecRequest{Query: "GET '/Volumes/c/s/v/file' TO '/tmp/file'", TransferFiles: false}) if op == nil { From 8479abd0bd917c4e993fea0eb9337e6b3c7d9bd3 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:22:20 +0000 Subject: [PATCH 12/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 098e9943..fc75c3b8 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -c3e7cf1ba339155c7932c77e39e2821fd918cddf +7a938104d5e9870f995926f135aa41925c044cf6 From b93ca567ac5781eaf1add9f5e27b5d5fab148bf3 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:38:12 +0000 Subject: [PATCH 13/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index fc75c3b8..152d3f41 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -7a938104d5e9870f995926f135aa41925c044cf6 +ab10f222493fe45c645f746cdd2d762173a70fc5 From 2c4a296d189bcb28671676aaabc371b1856464aa Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:47:14 +0000 Subject: [PATCH 14/44] test(kernel): rely on live raw-result contract coverage Signed-off-by: Aakash Saravanan --- internal/backend/kernel/kernel_test.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/internal/backend/kernel/kernel_test.go b/internal/backend/kernel/kernel_test.go index f6e10f3a..959b87e9 100644 --- a/internal/backend/kernel/kernel_test.go +++ b/internal/backend/kernel/kernel_test.go @@ -550,17 +550,6 @@ func TestExecuteRequiresStagingContext(t *testing.T) { } } -func TestQuerySkipsStagingContextGuard(t *testing.T) { - k := &KernelBackend{} - op, err := k.Execute(context.Background(), backend.ExecRequest{Query: "GET '/Volumes/c/s/v/file' TO '/tmp/file'", TransferFiles: false}) - if op == nil { - t.Fatal("expected non-nil operation") - } - if err == nil || !strings.Contains(err.Error(), "new_statement") { - t.Fatalf("expected native statement creation, got %v", err) - } -} - // ExecutionError must satisfy the same public contract as the Thrift path so the // errors.Is → errors.As → SqlState()/QueryId() recipe documented in doc.go works // on the kernel backend too (it previously returned a bare *KernelError that From 05c00b7821f56044df1f1aab38e8f57553e49525 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:49:45 +0000 Subject: [PATCH 15/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 152d3f41..731c1165 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -ab10f222493fe45c645f746cdd2d762173a70fc5 +6dbc9a7c721e55e535b469df446251f1fc0ccaaa From e5f6e8fe8daaf07184ab81fe0bdd7c246be90b33 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 00:57:56 +0000 Subject: [PATCH 16/44] test: exercise transfer routing with a valid operation Signed-off-by: Aakash Saravanan --- connection_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/connection_test.go b/connection_test.go index f013077f..8a9f9fe1 100644 --- a/connection_test.go +++ b/connection_test.go @@ -981,13 +981,18 @@ type transferCaptureBackend struct { transferFiles []bool } -func (b *transferCaptureBackend) Execute(_ context.Context, req backend.ExecRequest) (backend.Operation, error) { +func (b *transferCaptureBackend) Execute(ctx context.Context, req backend.ExecRequest) (backend.Operation, error) { b.transferFiles = append(b.transferFiles, req.TransferFiles) - return nil, fmt.Errorf("capture request") + return b.Backend.Execute(ctx, req) } func TestConnectionTransferMode(t *testing.T) { - capture := &transferCaptureBackend{} + testClient := &client.TestClient{ + FnExecuteStatement: func(context.Context, *cli_service.TExecuteStatementReq) (*cli_service.TExecuteStatementResp, error) { + return &cli_service.TExecuteStatementResp{}, fmt.Errorf("capture request") + }, + } + capture := &transferCaptureBackend{Backend: thriftbackend.NewForTest(testClient, getTestSession(), config.WithDefaults())} connection := &conn{cfg: config.WithDefaults(), backend: capture} _, execErr := connection.ExecContext(context.Background(), "REMOVE '/Volumes/c/s/v/file'", nil) _, queryErr := connection.QueryContext(context.Background(), "GET '/Volumes/c/s/v/file' TO '/tmp/file'", nil) From 0beb47234ddb1e59577b984762512d8b35d3b528 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:00:03 +0000 Subject: [PATCH 17/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 731c1165..bbb42949 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -6dbc9a7c721e55e535b469df446251f1fc0ccaaa +bd8adef4f3a38cc8807e4a22b614865246ee7cc2 From b47ceda0202bf7716d59aa211e24f0f6427e1eaf Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:10:21 +0000 Subject: [PATCH 18/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index bbb42949..533b2c97 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -bd8adef4f3a38cc8807e4a22b614865246ee7cc2 +c3300aaa17d0d1109e2579c40e6f35b3c3aeaf54 From 092917d4e823b8c41a0c124bf6e9f1f9070f5332 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:25:01 +0000 Subject: [PATCH 19/44] docs: explain native transfer fallback behavior Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2e148e6..78c265f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History ## Unreleased -- Support SQL PUT, GET, and REMOVE through kernel-backed `ExecContext` with a staging context. `QueryContext` continues to return transfer instructions. +- Support SQL PUT, GET, and REMOVE through kernel-backed `ExecContext` with a staging context. `QueryContext` continues to return transfer instructions. Kernel-backed `ExecContext` performs supported transfers instead of returning `ErrNotSupportedByKernel`; callers should remove fallback handling for that error on these operations. ## v1.19.0 (2026-11-04) - Upgrade the SQL kernel bindings to v1.4.0. From c965c7ff763e9fe1c12f1601a7cd6b5e4188f2b2 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:30:21 +0000 Subject: [PATCH 20/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 533b2c97..4d7e0fd5 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -c3300aaa17d0d1109e2579c40e6f35b3c3aeaf54 +ed829e8ac8b27bdb503beaf6576dc41876df9465 From a4ef5cf0675de8ca49d0a3d9613702c583fa216a Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:40:13 +0000 Subject: [PATCH 21/44] docs: include native volume transfers in package support Signed-off-by: Aakash Saravanan --- doc.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc.go b/doc.go index e78f5a61..9d43ddac 100644 --- a/doc.go +++ b/doc.go @@ -220,16 +220,17 @@ RUST_LOG. Filter on the target databricks::sql::kernel (note the colons): Supported on the kernel backend: PAT and OAuth (M2M via WithClientCredentials, U2M via the authType=oauthU2M DSN param); reading scalar, nested, and complex-typed results (CloudFetch is transparent); bound query parameters (positional and named); -context cancellation during execute; the initial namespace (WithInitialNamespace, +context cancellation during execute; SQL PUT/GET/REMOVE on Unity Catalog Volumes +through ExecContext with a staging context (QueryContext returns raw instructions); +the initial namespace (WithInitialNamespace, applied post-connect via USE CATALOG / USE SCHEMA); metric-view metadata (WithEnableMetricViewMetadata); the retry / backoff policy (WithRetries: RetryWaitMin / RetryWaitMax / RetryMax, including the disable form, forwarded to the kernel's HTTP retry config); and the TLS, proxy, and session-conf (query tags, statement timeout, time zone) options. Federated token providers use one token snapshot during setup. Nothing is silently ignored: WithTimeout and token-provider / external / -static authenticators are rejected at connect; staging (PUT/GET/REMOVE on a -Unity Catalog volume) is rejected at execute. WithMaxRows is accepted but inert (the -kernel manages fetching below the C ABI). +static authenticators are rejected at connect. WithMaxRows is accepted but inert +(the kernel manages fetching below the C ABI). Kernel HTTP requests inherit the driver's ClientTimeout (900s by default). If that value is zero, the kernel uses its own 120s default; zero is neither From c65fde1564179b61e52b6e03c9edbcf91051a4a0 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:41:22 +0000 Subject: [PATCH 22/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 4d7e0fd5..f67811cf 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -ed829e8ac8b27bdb503beaf6576dc41876df9465 +2006c55d1b67903ca7b6090d0b84a2be73fcb82f From a0c5358e91d8dc5bab98b80d7b21aa637550ab13 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 01:50:14 +0000 Subject: [PATCH 23/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index f67811cf..d0a7d067 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -2006c55d1b67903ca7b6090d0b84a2be73fcb82f +892ee5698302b4e34db5ea926baa0354e9c229da From 55fb9795658f95aeb8c1f7f64faeee55109b6120 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:02:02 +0000 Subject: [PATCH 24/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index d0a7d067..e4f03240 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -892ee5698302b4e34db5ea926baa0354e9c229da +00e1701a9b14082dba9e4d4d7d8df378971e1818 From aa8695c4ece8216b0c45985e6e6180c4427431d8 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:09:55 +0000 Subject: [PATCH 25/44] ci: verify transfers link against published native modules Signed-off-by: Aakash Saravanan --- .github/workflows/go.yml | 25 +++++++++++++++++++++++++ internal/backend/kernel/operation.go | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 92e32bb6..ed9c95e1 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -105,6 +105,31 @@ jobs: - name: Build run: make linux + build-published-kernel: + name: Build (published kernel) + timeout-minutes: 15 + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.27.3' + cache: false + + - name: Link against published native modules + run: go test -mod=readonly -tags databricks_kernel -run '^$' ./... + env: + CGO_ENABLED: 1 + GOWORK: off + build-and-test-kernel: name: Test (kernel backend) # Exercises the opt-in SEA-via-kernel backend: builds the Rust kernel static diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index 391d6317..1fec9cc6 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -87,7 +87,7 @@ func (k *KernelBackend) execute(ctx context.Context, req backend.ExecRequest) (b encoded, err := json.Marshal(paths) if err != nil { C.kernel_statement_close(stmt) - return &kernelOp{}, err + return &kernelOp{}, fmt.Errorf("kernel: encode allowed local paths: %w", err) } value := newCStr(string(encoded)) err = call(func() C.KernelStatusCode { From e02b74140f68cadd37cdc28099203b62b24efda6 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:12:03 +0000 Subject: [PATCH 26/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index e4f03240..a3bdb6dd 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -00e1701a9b14082dba9e4d4d7d8df378971e1818 +e78a4b1e58562e03a06c9166cc76bde5842f1de0 From 235ad3e39b995706af2747294a72780beeac7585 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:41:53 +0000 Subject: [PATCH 27/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index a3bdb6dd..3f36a79f 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -e78a4b1e58562e03a06c9166cc76bde5842f1de0 +f139344c9c8cf9f3224f6d3068adba08632a0c32 From 684d44c2225d0b6b43303b5e966e8d13d8dfa595 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 02:51:35 +0000 Subject: [PATCH 28/44] test(staging): bound volume fixture cleanup Signed-off-by: Aakash Saravanan --- kernel_e2e_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel_e2e_test.go b/kernel_e2e_test.go index ff93a783..70126e1e 100644 --- a/kernel_e2e_test.go +++ b/kernel_e2e_test.go @@ -692,7 +692,9 @@ func TestKernelE2EVolumeFileLifecycle(t *testing.T) { t.Fatal(err) } defer func() { - if _, err := db.ExecContext(context.Background(), "DROP SCHEMA "+namespace+" CASCADE"); err != nil { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), time.Minute) + defer cancelCleanup() + if _, err := db.ExecContext(cleanupCtx, "DROP SCHEMA "+namespace+" CASCADE"); err != nil { t.Error(err) } }() From 76d50d4d3d3df5f93e57326e7abdef3137e29a15 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 03:08:01 +0000 Subject: [PATCH 29/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 3f36a79f..0c799948 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -f139344c9c8cf9f3224f6d3068adba08632a0c32 +9f7bf932aacbd76ff67c52cc1cd84082af2e5ab0 From 1c9771929b8d46ff45655a2dc8d6e3e663c20b75 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 03:16:22 +0000 Subject: [PATCH 30/44] test(volume): assert structured local transfer error categories Signed-off-by: Aakash Saravanan --- kernel_e2e_test.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/kernel_e2e_test.go b/kernel_e2e_test.go index 70126e1e..9e3ab0c4 100644 --- a/kernel_e2e_test.go +++ b/kernel_e2e_test.go @@ -20,6 +20,7 @@ import ( "github.com/databricks/databricks-sql-go/driverctx" dbsqlerr "github.com/databricks/databricks-sql-go/errors" + dbsqlerrint "github.com/databricks/databricks-sql-go/internal/errors" dbsqlrows "github.com/databricks/databricks-sql-go/rows" ) @@ -753,12 +754,15 @@ func TestKernelE2EVolumeFileLifecycle(t *testing.T) { if err := os.WriteFile(denied, payload, 0600); err != nil { t.Fatal(err) } - for _, test := range []struct{ local, message string }{ - {denied, "restricted"}, - {filepath.Join(directory, "missing.bin"), "File transfer failed"}, + for _, test := range []struct { + local string + category dbsqlerrint.ErrorCategory + }{ + {denied, dbsqlerrint.CategoryPermissionError}, + {filepath.Join(directory, "missing.bin"), dbsqlerrint.CategoryNotFound}, } { - if _, err := db.ExecContext(stagingCtx, "PUT "+literal(test.local)+" INTO "+literal(remote)+" OVERWRITE"); err == nil || !strings.Contains(err.Error(), test.message) { - t.Fatalf("PUT %q: expected %q, got %v", test.local, test.message, err) + if _, err := db.ExecContext(stagingCtx, "PUT "+literal(test.local)+" INTO "+literal(remote)+" OVERWRITE"); err == nil || dbsqlerrint.CategoryFromError(err) != test.category { + t.Fatalf("PUT %q: expected category %q, got %v", test.local, test.category, err) } } for range 2 { From 16e43e9a765554ab65aad338b3a6603156193810 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 03:18:13 +0000 Subject: [PATCH 31/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 0c799948..679cb150 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -9f7bf932aacbd76ff67c52cc1cd84082af2e5ab0 +be474276cbffa2aa89eeee593c957cacc7b43c81 From 43ecee4bbedf270edeae68bf57f45e779123dd67 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 03:44:14 +0000 Subject: [PATCH 32/44] test(kernel): cover transfer cancellation and raw execution Signed-off-by: Aakash Saravanan --- .../backend/kernel/staging_cancel_test.go | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 internal/backend/kernel/staging_cancel_test.go diff --git a/internal/backend/kernel/staging_cancel_test.go b/internal/backend/kernel/staging_cancel_test.go new file mode 100644 index 00000000..5ed6565f --- /dev/null +++ b/internal/backend/kernel/staging_cancel_test.go @@ -0,0 +1,172 @@ +//go:build cgo && databricks_kernel + +package kernel + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/arrow/array" + "github.com/apache/arrow/go/v12/arrow/ipc" + "github.com/apache/arrow/go/v12/arrow/memory" + "github.com/databricks/databricks-sql-go/driverctx" + "github.com/databricks/databricks-sql-go/internal/backend" +) + +func TestExecuteCancelsFileTransfer(t *testing.T) { + for _, verb := range []string{"PUT", "GET"} { + t.Run(verb, func(t *testing.T) { + directory := t.TempDir() + local := filepath.Join(directory, "file") + original := bytes.Repeat([]byte("a"), 8*1024*1024) + if err := os.WriteFile(local, original, 0600); err != nil { + t.Fatal(err) + } + started := make(chan struct{}) + release := make(chan struct{}) + var instruction []byte + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/storage": + if verb == "PUT" { + if _, err := io.CopyN(io.Discard, r.Body, 1); err != nil { + return + } + } else { + _, _ = w.Write([]byte("partial")) + w.(http.Flusher).Flush() + } + close(started) + select { + case <-r.Context().Done(): + case <-release: + } + case r.Method == "POST" && r.URL.Path == "/api/2.0/sql/statements": + _, _ = w.Write(instruction) + case strings.HasSuffix(r.URL.Path, "/sessions"): + _, _ = io.WriteString(w, `{"session_id":"session","warehouse_id":"test"}`) + default: + _, _ = io.WriteString(w, `{}`) + } + })) + defer server.Close() + instruction = fileTransferInstruction(t, verb, server.URL+"/storage", local) + k := New(Config{Host: server.URL, WarehouseID: "test", Auth: Auth{Mode: AuthPAT, Token: "test"}}) + if err := k.OpenSession(context.Background()); err != nil { + t.Fatal(err) + } + defer k.CloseSession(context.Background()) + raw, err := k.Execute(context.Background(), backend.ExecRequest{Query: "get '/Volumes/c/s/v/file' to '" + local + "'"}) + if err != nil { + t.Fatalf("raw execution without staging context: %v", err) + } + if _, err := raw.Close(context.Background()); err != nil { + t.Fatal(err) + } + select { + case <-started: + t.Fatal("raw execution started a file transfer") + default: + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ctx = driverctx.NewContextWithStagingInfo(ctx, []string{directory}) + finished := make(chan error, 1) + stopped := make(chan struct{}) + query := "PUT '" + local + "' INTO '/Volumes/c/s/v/file' OVERWRITE" + if verb == "GET" { + query = "GET '/Volumes/c/s/v/file' TO '" + local + "'" + } + go func() { + defer close(stopped) + op, err := k.Execute(ctx, backend.ExecRequest{Query: query, TransferFiles: true}) + if op != nil { + _, _ = op.Close(context.Background()) + } + finished <- err + }() + defer func() { + close(release) + <-stopped + }() + select { + case <-started: + case err := <-finished: + t.Fatalf("execute returned before transfer: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("transfer did not start") + } + cancel() + select { + case err := <-finished: + if !errors.Is(err, context.Canceled) { + t.Fatalf("execute error = %v, want context.Canceled", err) + } + case <-time.After(2 * time.Second): + t.Fatal("cancelled execute is still waiting for the transfer") + } + if verb == "GET" { + data, err := os.ReadFile(local) + if err != nil || !bytes.Equal(data, original) { + t.Fatalf("cancelled GET changed the destination: %v", err) + } + } + }) + } +} + +func fileTransferInstruction(t *testing.T, verb, url, local string) []byte { + t.Helper() + names := []string{"operation", "url", "headers", "local"} + fields := make([]arrow.Field, len(names)) + columns := make([]map[string]any, len(names)) + for i, name := range names { + fields[i] = arrow.Field{Name: name, Type: arrow.BinaryTypes.String} + columns[i] = map[string]any{"name": name, "type_name": "STRING", "type_text": "STRING", "position": i} + } + schema := arrow.NewSchema(fields, nil) + builder := array.NewRecordBuilder(memory.NewGoAllocator(), schema) + defer builder.Release() + for i, value := range []string{verb, url, "{}", local} { + builder.Field(i).(*array.StringBuilder).Append(value) + } + record := builder.NewRecord() + defer record.Release() + var buffer bytes.Buffer + writer := ipc.NewWriter(&buffer, ipc.WithSchema(schema)) + if err := writer.Write(record); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + body, err := json.Marshal(map[string]any{ + "statement_id": "transfer", + "status": map[string]any{"state": "SUCCEEDED"}, + "manifest": map[string]any{ + "format": "ARROW_STREAM", "is_volume_operation": true, + "schema": map[string]any{"column_count": len(names), "columns": columns}, + "total_chunk_count": 1, "total_row_count": 1, "total_byte_count": buffer.Len(), "truncated": false, + }, + "result": map[string]any{ + "chunk_index": 0, "row_offset": 0, "row_count": 1, "byte_count": buffer.Len(), + "attachment": base64.StdEncoding.EncodeToString(buffer.Bytes()), + }, + }) + if err != nil { + t.Fatal(err) + } + return body +} From 3f3d591d82e8ac96938dd6a85641e71479d1a533 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 03:46:18 +0000 Subject: [PATCH 33/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 679cb150..029e66f9 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -be474276cbffa2aa89eeee593c957cacc7b43c81 +23cd28e82876a75d0ecbaa487018906bc0dca924 From f95ffe093bdb0a2742030c31ce2541fca5a379a2 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 03:58:10 +0000 Subject: [PATCH 34/44] test(kernel): tolerate repeated storage request notifications Signed-off-by: Aakash Saravanan --- internal/backend/kernel/staging_cancel_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/backend/kernel/staging_cancel_test.go b/internal/backend/kernel/staging_cancel_test.go index 5ed6565f..73daa452 100644 --- a/internal/backend/kernel/staging_cancel_test.go +++ b/internal/backend/kernel/staging_cancel_test.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -35,6 +36,7 @@ func TestExecuteCancelsFileTransfer(t *testing.T) { t.Fatal(err) } started := make(chan struct{}) + var startedOnce sync.Once release := make(chan struct{}) var instruction []byte server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -48,7 +50,7 @@ func TestExecuteCancelsFileTransfer(t *testing.T) { _, _ = w.Write([]byte("partial")) w.(http.Flusher).Flush() } - close(started) + startedOnce.Do(func() { close(started) }) select { case <-r.Context().Done(): case <-release: From fdae998dcfab0156034fd1f2269603c5f6082664 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 04:03:32 +0000 Subject: [PATCH 35/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 029e66f9..b9db4871 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -23cd28e82876a75d0ecbaa487018906bc0dca924 +4f12d122d435c03ca489f77138deebd2ebe17081 From 94b7221d23dcd6214f25c8f66f196a2c93c134bb Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 04:16:38 +0000 Subject: [PATCH 36/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index b9db4871..0d8e1b74 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -4f12d122d435c03ca489f77138deebd2ebe17081 +b92a86716b49f48f577e2299989f235c20b50ee7 From 9980687198db856bf530fe3abb10315146511a79 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 04:30:05 +0000 Subject: [PATCH 37/44] test(kernel): synchronize transfer fixtures and check raw contexts Signed-off-by: Aakash Saravanan --- .../backend/kernel/staging_cancel_test.go | 20 +++++++++++++------ kernel_e2e_test.go | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/internal/backend/kernel/staging_cancel_test.go b/internal/backend/kernel/staging_cancel_test.go index 73daa452..c4c174c8 100644 --- a/internal/backend/kernel/staging_cancel_test.go +++ b/internal/backend/kernel/staging_cancel_test.go @@ -39,6 +39,7 @@ func TestExecuteCancelsFileTransfer(t *testing.T) { var startedOnce sync.Once release := make(chan struct{}) var instruction []byte + instructionReady := make(chan struct{}) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.URL.Path == "/storage": @@ -56,6 +57,7 @@ func TestExecuteCancelsFileTransfer(t *testing.T) { case <-release: } case r.Method == "POST" && r.URL.Path == "/api/2.0/sql/statements": + <-instructionReady _, _ = w.Write(instruction) case strings.HasSuffix(r.URL.Path, "/sessions"): _, _ = io.WriteString(w, `{"session_id":"session","warehouse_id":"test"}`) @@ -65,17 +67,23 @@ func TestExecuteCancelsFileTransfer(t *testing.T) { })) defer server.Close() instruction = fileTransferInstruction(t, verb, server.URL+"/storage", local) + close(instructionReady) k := New(Config{Host: server.URL, WarehouseID: "test", Auth: Auth{Mode: AuthPAT, Token: "test"}}) if err := k.OpenSession(context.Background()); err != nil { t.Fatal(err) } defer k.CloseSession(context.Background()) - raw, err := k.Execute(context.Background(), backend.ExecRequest{Query: "get '/Volumes/c/s/v/file' to '" + local + "'"}) - if err != nil { - t.Fatalf("raw execution without staging context: %v", err) - } - if _, err := raw.Close(context.Background()); err != nil { - t.Fatal(err) + for _, rawCtx := range []context.Context{ + context.Background(), + driverctx.NewContextWithStagingInfo(context.Background(), []string{directory}), + } { + raw, err := k.Execute(rawCtx, backend.ExecRequest{Query: "get '/Volumes/c/s/v/file' to '" + local + "'"}) + if err != nil { + t.Fatalf("raw execution: %v", err) + } + if _, err := raw.Close(context.Background()); err != nil { + t.Fatal(err) + } } select { case <-started: diff --git a/kernel_e2e_test.go b/kernel_e2e_test.go index 9e3ab0c4..e7877e76 100644 --- a/kernel_e2e_test.go +++ b/kernel_e2e_test.go @@ -714,7 +714,7 @@ func TestKernelE2EVolumeFileLifecycle(t *testing.T) { t.Fatal(err) } get := "GET " + literal(remote) + " TO " + literal(destination) - rows, err := db.QueryContext(ctx, get) + rows, err := db.QueryContext(stagingCtx, get) if err != nil { t.Fatal(err) } From 5345fb05bdea226b8db066aeaaa63fd5348f4253 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 04:35:52 +0000 Subject: [PATCH 38/44] docs: clarify native QueryContext transfer behavior Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78c265f9..f4bd5356 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History ## Unreleased -- Support SQL PUT, GET, and REMOVE through kernel-backed `ExecContext` with a staging context. `QueryContext` continues to return transfer instructions. Kernel-backed `ExecContext` performs supported transfers instead of returning `ErrNotSupportedByKernel`; callers should remove fallback handling for that error on these operations. +- Support SQL PUT, GET, and REMOVE through kernel-backed `ExecContext` with a staging context. Kernel-backed `QueryContext` now returns raw transfer instructions, matching Thrift, instead of `ErrNotSupportedByKernel`; it does not transfer files. Callers should review fallback handling that depends on that error and use `ExecContext` to complete transfers. ## v1.19.0 (2026-11-04) - Upgrade the SQL kernel bindings to v1.4.0. From 5bcd046a43663f2fd08c8ed2453ad86578014450 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 04:45:08 +0000 Subject: [PATCH 39/44] docs: explain completion for every ExecContext Signed-off-by: Aakash Saravanan --- internal/backend/kernel/operation.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index 1fec9cc6..01a4331b 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -78,7 +78,8 @@ func (k *KernelBackend) execute(ctx context.Context, req backend.ExecRequest) (b } sql.free() - // An empty list denies local transfers; leaving the option unset returns raw instructions. + // Every ExecContext opts into completion, including transfers missed by the SQL recognizer. + // An empty list denies local transfers instead of reporting raw instructions as success. if req.TransferFiles { paths := driverctx.StagingPathsFromContext(ctx) if paths == nil { From 55686d03b81d2a0d9ce73a5f6cabb6884c26dbcd Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 04:47:34 +0000 Subject: [PATCH 40/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 0d8e1b74..c034890e 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -b92a86716b49f48f577e2299989f235c20b50ee7 +ea26771a5122c8558066638c292cff01df1ebb82 From 3856db24095a72abcf7056ae8dbf116c46415873 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 05:06:43 +0000 Subject: [PATCH 41/44] Recognize commented volume commands before checking staging context Signed-off-by: Aakash Saravanan --- internal/backend/kernel/staging.go | 30 +++++++++++++++++-------- internal/backend/kernel/staging_test.go | 6 +++++ kernel_e2e_test.go | 11 +++++++++ 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/internal/backend/kernel/staging.go b/internal/backend/kernel/staging.go index f4dd1499..52cc59f7 100644 --- a/internal/backend/kernel/staging.go +++ b/internal/backend/kernel/staging.go @@ -13,7 +13,7 @@ func isStagingStatement(sql string) bool { for _, kw := range []string{"PUT", "GET", "REMOVE"} { if len(s) > len(kw) && strings.EqualFold(s[:len(kw)], kw) && - isSQLTokenBreak(s[len(kw)]) { + (isSQLTokenBreak(s[len(kw)]) || strings.HasPrefix(s[len(kw):], "/*") || strings.HasPrefix(s[len(kw):], "--")) { return true } } @@ -23,9 +23,9 @@ func isStagingStatement(sql string) bool { // stripLeadingSQLComments removes leading whitespace and leading SQL comments // (both `-- ...` line comments and `/* ... */` block comments), repeatedly, so // the first real token is exposed. An unterminated block comment consumes the -// rest of the string (yielding ""). Comment bodies are not scanned for nested -// markers — SQL block comments do not nest — and this only touches the LEADING -// run, so a comment later in the statement is untouched. +// rest of the string (yielding ""). Nested block comments must be balanced so +// their trailing text cannot hide a staging keyword. Only the leading run is +// removed; comments later in the statement are untouched. func stripLeadingSQLComments(sql string) string { s := strings.TrimSpace(sql) for { @@ -37,11 +37,23 @@ func stripLeadingSQLComments(sql string) string { return "" // line comment to end of input } case strings.HasPrefix(s, "/*"): - if i := strings.Index(s[2:], "*/"); i >= 0 { - s = s[2+i+2:] - } else { + depth, i := 1, 2 + for i < len(s) && depth > 0 { + switch { + case strings.HasPrefix(s[i:], "/*"): + depth++ + i += 2 + case strings.HasPrefix(s[i:], "*/"): + depth-- + i += 2 + default: + i++ + } + } + if depth != 0 { return "" // unterminated block comment } + s = s[i:] default: return s } @@ -49,9 +61,9 @@ func stripLeadingSQLComments(sql string) string { } } -// isSQLTokenBreak reports whether b ends a leading keyword — a space or a quote +// isSQLTokenBreak reports whether b ends a leading keyword, whitespace or a quote // (staging commands take a quoted path, e.g. PUT '' INTO ''), so // "PUT '..." matches but an identifier like "PUTS" or "GETTER" does not. func isSQLTokenBreak(b byte) bool { - return b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\'' || b == '"' + return b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\f' || b == '\v' || b == '\'' || b == '"' } diff --git a/internal/backend/kernel/staging_test.go b/internal/backend/kernel/staging_test.go index 91d69794..8a29843c 100644 --- a/internal/backend/kernel/staging_test.go +++ b/internal/backend/kernel/staging_test.go @@ -13,6 +13,11 @@ func TestIsStagingStatement(t *testing.T) { "/* a */ /* b */ PUT '/l' INTO '/Volumes/x'", // multiple block comments "-- upload the file\nPUT '/l' INTO '/Volumes/x'", // leading line comment, then staging " /* c */\n -- d\n REMOVE '/Volumes/x'", // mixed comments + whitespace + "/* outer /* inner */ outer */ REMOVE '/Volumes/x'", + "REMOVE/* note */ '/Volumes/x'", + "REMOVE-- note\n'/Volumes/x'", + "REMOVE\f'/Volumes/x'", + "REMOVE\v'/Volumes/x'", } for _, sql := range staging { if !isStagingStatement(sql) { @@ -32,6 +37,7 @@ func TestIsStagingStatement(t *testing.T) { "SELECT 'PUT ' AS col", // keyword only inside a literal "/* unterminated PUT '/x'", // unterminated block comment consumes the rest "/* c */ SELECT 1", // real leading comment, but not a staging command + "/* outer /* inner */ REMOVE '/Volumes/x'", } for _, sql := range notStaging { if isStagingStatement(sql) { diff --git a/kernel_e2e_test.go b/kernel_e2e_test.go index e7877e76..94e7df8c 100644 --- a/kernel_e2e_test.go +++ b/kernel_e2e_test.go @@ -713,6 +713,17 @@ func TestKernelE2EVolumeFileLifecycle(t *testing.T) { if _, err := db.ExecContext(stagingCtx, "PUT "+literal(source)+" INTO "+literal(remote)+" OVERWRITE"); err != nil { t.Fatal(err) } + for _, remove := range []string{ + "/* outer /* inner */ outer */ REMOVE " + literal(remote), + "REMOVE/* note */ " + literal(remote), + "REMOVE-- note\n" + literal(remote), + "REMOVE\f" + literal(remote), + "REMOVE\v" + literal(remote), + } { + if _, err := db.ExecContext(ctx, remove); err == nil || !strings.Contains(err.Error(), "staging ctx must be provided") { + t.Fatalf("REMOVE without staging context: %v", err) + } + } get := "GET " + literal(remote) + " TO " + literal(destination) rows, err := db.QueryContext(stagingCtx, get) if err != nil { From 60ca0f2cf178a704d561f8faec88d55f7ddb1b5f Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 05:18:58 +0000 Subject: [PATCH 42/44] build(kernel): pin reviewed file transfer source Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index c034890e..b1078d37 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -ea26771a5122c8558066638c292cff01df1ebb82 +37c7a3ec1b54f4690d9325d65f2220e0bb64b83e From c0413355532146eb8640c7743ae4135fa84e1f94 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 19:07:06 +0000 Subject: [PATCH 43/44] Keep volume transfer documentation focused on usage Signed-off-by: Aakash Saravanan --- CONNECTION_PARAMETERS.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 5305da10..9469e29a 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -244,6 +244,5 @@ Both backends support SQL PUT, GET, and REMOVE through `ExecContext` with `driverctx.NewContextWithStagingInfo(ctx, allowedLocalPaths)`. The context must contain at least one allowed path, including for REMOVE. `QueryContext` returns the transfer instructions for callers that handle file transfers themselves. -Kernel builds require the revision in `KERNEL_REV` and matching native bindings. The credential-gated `TestKernelE2EVolumeFileLifecycle` uses `DATABRICKS_VOLUME_TEST_CATALOG` as a writable catalog and creates and removes an isolated schema and Volume. It covers ExecContext transfers and QueryContext raw instructions. From ad9e40c11fcea38e127a947ffc9fa57c4616619e Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 12 Nov 2026 19:23:18 +0000 Subject: [PATCH 44/44] Simplify path encoding and transfer validation Signed-off-by: Aakash Saravanan --- internal/backend/kernel/operation.go | 8 ++------ kernel_e2e_test.go | 12 ++---------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index 01a4331b..ce4efa0d 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -85,13 +85,9 @@ func (k *KernelBackend) execute(ctx context.Context, req backend.ExecRequest) (b if paths == nil { paths = []string{} } - encoded, err := json.Marshal(paths) - if err != nil { - C.kernel_statement_close(stmt) - return &kernelOp{}, fmt.Errorf("kernel: encode allowed local paths: %w", err) - } + encoded, _ := json.Marshal(paths) value := newCStr(string(encoded)) - err = call(func() C.KernelStatusCode { + err := call(func() C.KernelStatusCode { return C.kernel_statement_set_staging_allowed_local_paths(stmt, value.c, false) }) value.free() diff --git a/kernel_e2e_test.go b/kernel_e2e_test.go index 94e7df8c..51c3cfdd 100644 --- a/kernel_e2e_test.go +++ b/kernel_e2e_test.go @@ -713,16 +713,8 @@ func TestKernelE2EVolumeFileLifecycle(t *testing.T) { if _, err := db.ExecContext(stagingCtx, "PUT "+literal(source)+" INTO "+literal(remote)+" OVERWRITE"); err != nil { t.Fatal(err) } - for _, remove := range []string{ - "/* outer /* inner */ outer */ REMOVE " + literal(remote), - "REMOVE/* note */ " + literal(remote), - "REMOVE-- note\n" + literal(remote), - "REMOVE\f" + literal(remote), - "REMOVE\v" + literal(remote), - } { - if _, err := db.ExecContext(ctx, remove); err == nil || !strings.Contains(err.Error(), "staging ctx must be provided") { - t.Fatalf("REMOVE without staging context: %v", err) - } + if _, err := db.ExecContext(ctx, "REMOVE "+literal(remote)); err == nil || !strings.Contains(err.Error(), "staging ctx must be provided") { + t.Fatalf("REMOVE without staging context: %v", err) } get := "GET " + literal(remote) + " TO " + literal(destination) rows, err := db.QueryContext(stagingCtx, get)