diff --git a/.github/workflows/pgo-validate.yml b/.github/workflows/pgo-validate.yml index 7bb585524..8c375a066 100644 --- a/.github/workflows/pgo-validate.yml +++ b/.github/workflows/pgo-validate.yml @@ -27,7 +27,7 @@ jobs: enable-cache: false - name: Export hashed requirements from uv.lock - run: python tools/scripts/export_hashed_requirements.py --profile cli --profile runtime --profile build --profile pgo-validate --output-dir .build-requirements + run: python tools/scripts/export_hashed_requirements.py --profile cli --profile runtime --profile build --profile pgo --profile test --output-dir .build-requirements - name: Install cibuildwheel run: python -m pip install --require-hashes -r .build-requirements/cli.txt @@ -46,23 +46,23 @@ jobs: PGO_DIR=/tmp/sqlspec-pgo && BUILD_DIR=/tmp/sqlspec-mypyc-build && mkdir -p "$PGO_DIR" "$BUILD_DIR" && - CFLAGS="-fprofile-generate=$PGO_DIR" + CFLAGS="-DNDEBUG -fno-strict-overflow -fno-semantic-interposition -fprofile-generate=$PGO_DIR" + LDFLAGS="-Wl,-Bsymbolic-functions" HATCH_BUILD_HOOKS_ENABLE=1 HATCH_MYPYC_BUILD_DIR="$BUILD_DIR" - MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 pip install --no-build-isolation --no-deps "{package}" && - pip install --require-hashes -r "{project}/.build-requirements/pgo-validate.txt" && - python "{project}/tools/scripts/pgo_training.py" && + pip install --require-hashes -r "{project}/.build-requirements/pgo.txt" && + PYTHONSAFEPATH=1 python "{project}/tools/scripts/pgo_training.py" --strict && pip uninstall -y sqlspec && rm -rf "$BUILD_DIR/build" "$BUILD_DIR/tmp" CIBW_ENVIRONMENT_LINUX: >- HATCH_BUILD_HOOKS_ENABLE=1 HATCH_MYPYC_BUILD_DIR=/tmp/sqlspec-mypyc-build - MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 - CFLAGS="-fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch" + CFLAGS="-DNDEBUG -fno-strict-overflow -fno-semantic-interposition -flto=auto -fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch" + LDFLAGS="-Wl,-Bsymbolic-functions -flto=auto" - CIBW_BEFORE_TEST: 'pip install --require-hashes -r "{project}/.build-requirements/runtime.txt"' + CIBW_BEFORE_TEST: 'pip install --require-hashes -r "{project}/.build-requirements/runtime.txt" -r "{project}/.build-requirements/test.txt"' CIBW_TEST_COMMAND: 'python "{project}/tools/scripts/mypyc_smoke.py" --require-compiled' - name: List built wheels diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b27893a3c..62a393ec2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -143,32 +143,30 @@ jobs: PGO_DIR=/tmp/sqlspec-pgo && BUILD_DIR=/tmp/sqlspec-mypyc-build && mkdir -p "$PGO_DIR" "$BUILD_DIR" && - CFLAGS="-fprofile-generate=$PGO_DIR" + CFLAGS="-DNDEBUG -fno-strict-overflow -fno-semantic-interposition -fprofile-generate=$PGO_DIR" + LDFLAGS="-Wl,-Bsymbolic-functions" HATCH_BUILD_HOOKS_ENABLE=1 HATCH_MYPYC_BUILD_DIR="$BUILD_DIR" - MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 pip install --no-build-isolation --no-deps "{package}" && pip install --require-hashes -r "{project}/.build-requirements/pgo.txt" && - python "{project}/tools/scripts/pgo_training.py" && + PYTHONSAFEPATH=1 python "{project}/tools/scripts/pgo_training.py" --strict && pip uninstall -y sqlspec && rm -rf "$BUILD_DIR/build" "$BUILD_DIR/tmp" CIBW_ENVIRONMENT_LINUX: >- HATCH_BUILD_HOOKS_ENABLE=1 HATCH_MYPYC_BUILD_DIR=/tmp/sqlspec-mypyc-build - MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 - CFLAGS="-fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch" + CFLAGS="-DNDEBUG -fno-strict-overflow -fno-semantic-interposition -flto=auto -fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch" + LDFLAGS="-Wl,-Bsymbolic-functions -flto=auto" # ── macOS (Clang): plain mypyc, no PGO ── CIBW_ENVIRONMENT_MACOS: >- HATCH_BUILD_HOOKS_ENABLE=1 - MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 # ── Windows (MSVC): plain mypyc, no PGO ── CIBW_ENVIRONMENT_WINDOWS: >- HATCH_BUILD_HOOKS_ENABLE=1 HATCH_MYPYC_BUILD_DIR=C:/b - MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 - name: Upload wheel artifacts uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -184,7 +182,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] - python-version: ["3.11", "3.13", "3.14", "3.15"] + python-version: ["3.11", "3.12", "3.13", "3.14", "3.15"] runs-on: ${{ matrix.os }} steps: @@ -223,7 +221,7 @@ jobs: shell: bash env: PYTHON_VERSION: ${{ matrix.python-version }} - run: uv run --no-project --python "$PYTHON_VERSION" python tools/scripts/export_hashed_requirements.py --profile runtime --profile smoke --output-dir .build-requirements + run: uv run --no-project --python "$PYTHON_VERSION" python tools/scripts/export_hashed_requirements.py --profile runtime --profile smoke --profile drivers --profile drivers-db2 --profile drivers-sqlanywhere --profile drivers-sqlanywhere-build --output-dir .build-requirements - name: Download standard wheel artifacts uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 @@ -242,6 +240,10 @@ jobs: # missing host wheel fails loudly instead of silently grabbing a stale # PyPI release. Core and smoke dependencies use hash-pinned requirements. # The version assert catches drift between pyproject.toml and the wheel. + # Linux also installs the hash-pinned driver SDK profiles so the smoke run + # imports those compiled adapter drivers instead of skipping them. sqlanydb + # ships only as a source distribution, so it builds with a hash-pinned + # backend; ibm_db ships Linux wheels for x86_64 only. - name: Test standard wheel installation shell: bash env: @@ -268,7 +270,22 @@ jobs: else echo "::warning::obstore or pyarrow binary wheel unavailable for Python ${PYTHON_VERSION}; the obstore smoke check will be skipped" fi - uv run --no-project --python test-mypyc python -I tools/scripts/mypyc_smoke.py --require-compiled + smoke_args=(--require-compiled) + if [ "$RUNNER_OS" = "Linux" ]; then + uv pip install --no-config --python test-mypyc --require-hashes --only-binary :all: -r .build-requirements/drivers.txt + uv pip install --no-config --python test-mypyc --require-hashes --build-constraints .build-requirements/drivers-sqlanywhere-build.txt -r .build-requirements/drivers-sqlanywhere.txt + smoke_args+=(--require-sdk-profile) + if [ "$RUNNER_ARCH" = "X64" ]; then + grep -v '^[[:space:]]*--hash=' .build-requirements/drivers-db2.txt | sed -e 's/[[:space:]]*\\$//' > .build-requirements/drivers-db2-specs.txt + if uv pip install --no-config --python test-mypyc --dry-run --only-binary :all: -r .build-requirements/drivers-db2-specs.txt; then + uv pip install --no-config --python test-mypyc --require-hashes --only-binary :all: -r .build-requirements/drivers-db2.txt + smoke_args+=(--require-sdk ibm_db --require-sdk ibm_db_dbi) + else + echo "::warning::ibm_db binary wheel unavailable for Python ${PYTHON_VERSION}; the Db2 driver smoke check will be skipped" + fi + fi + fi + uv run --no-project --python test-mypyc python -I tools/scripts/mypyc_smoke.py "${smoke_args[@]}" publish-release: name: Publish to PyPI diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 8f9a77c34..acbda523e 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -313,32 +313,30 @@ jobs: PGO_DIR=/tmp/sqlspec-pgo && BUILD_DIR=/tmp/sqlspec-mypyc-build && mkdir -p "$PGO_DIR" "$BUILD_DIR" && - CFLAGS="-fprofile-generate=$PGO_DIR" + CFLAGS="-DNDEBUG -fno-strict-overflow -fno-semantic-interposition -fprofile-generate=$PGO_DIR" + LDFLAGS="-Wl,-Bsymbolic-functions" HATCH_BUILD_HOOKS_ENABLE=1 HATCH_MYPYC_BUILD_DIR="$BUILD_DIR" - MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 pip install --no-build-isolation --no-deps "{package}" && pip install --require-hashes -r "{project}/.build-requirements/pgo.txt" && - python "{project}/tools/scripts/pgo_training.py" && + PYTHONSAFEPATH=1 python "{project}/tools/scripts/pgo_training.py" --strict && pip uninstall -y sqlspec && rm -rf "$BUILD_DIR/build" "$BUILD_DIR/tmp" CIBW_ENVIRONMENT_LINUX: >- HATCH_BUILD_HOOKS_ENABLE=1 HATCH_MYPYC_BUILD_DIR=/tmp/sqlspec-mypyc-build - MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 - CFLAGS="-fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch" + CFLAGS="-DNDEBUG -fno-strict-overflow -fno-semantic-interposition -flto=auto -fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch" + LDFLAGS="-Wl,-Bsymbolic-functions -flto=auto" # macOS + Windows: plain mypyc, no PGO. macOS PGO needs filter # deps absent from BEFORE_BUILD; MSVC PGO isn't validated. CIBW_ENVIRONMENT_MACOS: >- HATCH_BUILD_HOOKS_ENABLE=1 - MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 CIBW_ENVIRONMENT_WINDOWS: >- HATCH_BUILD_HOOKS_ENABLE=1 HATCH_MYPYC_BUILD_DIR=C:/b - MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 - name: Upload wheel artifacts uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -389,7 +387,7 @@ jobs: shell: bash env: PYTHON_VERSION: ${{ matrix.python-version }} - run: uv run --no-project --python "$PYTHON_VERSION" python tools/scripts/export_hashed_requirements.py --profile runtime --profile smoke --output-dir .build-requirements + run: uv run --no-project --python "$PYTHON_VERSION" python tools/scripts/export_hashed_requirements.py --profile runtime --profile smoke --profile drivers --profile drivers-db2 --profile drivers-sqlanywhere --profile drivers-sqlanywhere-build --output-dir .build-requirements - name: Download standard wheel artifacts uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 @@ -430,7 +428,22 @@ jobs: else echo "::warning::obstore or pyarrow binary wheel unavailable for Python ${PYTHON_VERSION}; the obstore smoke check will be skipped" fi - uv run --no-project --python test-mypyc python -I tools/scripts/mypyc_smoke.py --require-compiled + smoke_args=(--require-compiled) + if [ "$RUNNER_OS" = "Linux" ]; then + uv pip install --no-config --python test-mypyc --require-hashes --only-binary :all: -r .build-requirements/drivers.txt + uv pip install --no-config --python test-mypyc --require-hashes --build-constraints .build-requirements/drivers-sqlanywhere-build.txt -r .build-requirements/drivers-sqlanywhere.txt + smoke_args+=(--require-sdk-profile) + if [ "$RUNNER_ARCH" = "X64" ]; then + grep -v '^[[:space:]]*--hash=' .build-requirements/drivers-db2.txt | sed -e 's/[[:space:]]*\\$//' > .build-requirements/drivers-db2-specs.txt + if uv pip install --no-config --python test-mypyc --dry-run --only-binary :all: -r .build-requirements/drivers-db2-specs.txt; then + uv pip install --no-config --python test-mypyc --require-hashes --only-binary :all: -r .build-requirements/drivers-db2.txt + smoke_args+=(--require-sdk ibm_db --require-sdk ibm_db_dbi) + else + echo "::warning::ibm_db binary wheel unavailable for Python ${PYTHON_VERSION}; the Db2 driver smoke check will be skipped" + fi + fi + fi + uv run --no-project --python test-mypyc python -I tools/scripts/mypyc_smoke.py "${smoke_args[@]}" verify-packages: name: Verify package integrity diff --git a/Makefile b/Makefile index dd8b2614d..60b962107 100644 --- a/Makefile +++ b/Makefile @@ -454,17 +454,21 @@ bench-analyze: ## Analyze saved profile dat PGO_DIR := /tmp/sqlspec-pgo PGO_BUILD_DIR := /tmp/sqlspec-mypyc-build +PGO_BASE_CFLAGS := -DNDEBUG -fno-strict-overflow -fno-semantic-interposition +PGO_BASE_LDFLAGS := -Wl,-Bsymbolic-functions .PHONY: pgo-verify pgo-verify: ## Verify CFLAGS reach the C compiler through hatch-mypyc @echo "${INFO} Testing CFLAGS passthrough to C compiler..." - @CFLAGS="-DPGO_CANARY=1" uv run python -c "\ + @CFLAGS="-DPGO_CANARY=1" LDFLAGS="-Wl,-Bsymbolic-functions" uv run --no-sync python -c "\ from distutils.sysconfig import customize_compiler; \ from distutils.ccompiler import new_compiler; \ cc = new_compiler(); customize_compiler(cc); \ cmd = ' '.join(cc.compiler_so); \ + ld_cmd = ' '.join(cc.linker_so); \ assert 'PGO_CANARY' in cmd, f'CFLAGS not found in: {cmd}'; \ - print('CFLAGS reach compiler: ' + cmd)" 2>/dev/null + assert '-Wl,-Bsymbolic-functions' in ld_cmd, f'LDFLAGS not found in: {ld_cmd}'; \ + print('CFLAGS reach compiler: ' + cmd + ' | LDFLAGS: ' + ld_cmd)" 2>/dev/null @echo "${OK} CFLAGS passthrough verified" .PHONY: pgo-local @@ -473,17 +477,19 @@ pgo-local: ## Run full three-stage PGO @rm -rf $(PGO_DIR) $(PGO_BUILD_DIR) dist/ @mkdir -p $(PGO_DIR) $(PGO_BUILD_DIR) @echo "${INFO} Stage 1: Building instrumented wheel..." - @CFLAGS="-fprofile-generate=$(PGO_DIR)" \ + @CFLAGS="$(PGO_BASE_CFLAGS) -fprofile-generate=$(PGO_DIR)" \ + LDFLAGS="$(PGO_BASE_LDFLAGS)" \ HATCH_MYPYC_BUILD_DIR=$(PGO_BUILD_DIR) \ HATCH_BUILD_HOOKS_ENABLE=1 uv build --wheel >/dev/null 2>&1 @echo "${OK} Instrumented wheel built" @echo "${INFO} Stage 2: Running training workload..." @uv pip install dist/*.whl --force-reinstall --no-deps >/dev/null 2>&1 - @.venv/bin/python tools/scripts/pgo_training.py + @PYTHONSAFEPATH=1 .venv/bin/python tools/scripts/pgo_training.py @echo "${OK} Training complete" @rm -rf dist/ $(PGO_BUILD_DIR)/build $(PGO_BUILD_DIR)/tmp @echo "${INFO} Stage 3: Building PGO-optimized wheel..." - @CFLAGS="-fprofile-use=$(PGO_DIR) -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch" \ + @CFLAGS="$(PGO_BASE_CFLAGS) -flto=auto -fprofile-use=$(PGO_DIR) -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch" \ + LDFLAGS="$(PGO_BASE_LDFLAGS) -flto=auto" \ HATCH_MYPYC_BUILD_DIR=$(PGO_BUILD_DIR) \ HATCH_BUILD_HOOKS_ENABLE=1 uv build --wheel >/dev/null 2>&1 @echo "${OK} PGO-optimized wheel built: $$(ls dist/*.whl)" diff --git a/docs/changelog.md b/docs/changelog.md index a29cb7e6f..7b188b385 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -63,6 +63,7 @@ important operational fixes. * Litestar injected connections, Litestar on-demand request sessions, and the Sanic plugin now forward handler exceptions (including mapped exceptions and cancellations) to the connection context and force rollback under `autocommit` when an exception handler returns a `2xx` response. A failed or cancelled request now rolls back on psycopg, cockroach_psycopg, sqlite, aiosqlite, duckdb, and spanner, where it committed; discards the outermost thread connection on pymssql, sybase, sqlanywhere, and db2; and rolls back explicitly on psqlpy, oracledb sync, and mssql_python. * Litestar `commit_mode`, `extra_commit_statuses`, and `extra_rollback_statuses` now apply to injected connections, which stay open until the response starts, while `ConnectionCleanupMiddleware` and Sanic `http.lifecycle.complete` release residual connections on pre-response cancellations, disconnects, and streaming on-demand sessions. +* Litestar `get_annotation`, `provide_db_session`, and `provide_db_connection` (with the `_sync` and `_async` forms of the last two) raise `KeyError` when given a config class shared by several registered configurations, as `get_config` does, where they used the first one registered. Pass the configuration instance or its `session_key`, `connection_key`, or `pool_key` instead. * aiosqlite provide_connection, provide_session, and pool.get_connection now commit open work on a clean exit, matching sqlite, where they rolled it back. Exiting with an exception, including cancellation, still rolls back. Call rollback() before a clean exit to discard work. * `where_any()`, `where_not_any()`, their `or_where_*` forms, and `sql.not_any_()` no longer parse strings as subqueries, and raise `SQLBuilderError` for a string. Any other iterable, including a generator, binds its elements. Before: `.where_any("a", "SELECT x FROM y")`. After: `.where_any("a", sql.select("x").from_("y"))`. * `sql.exists_()`, `sql.any_()`, `sql.all_()`, `sql.in_()`, `where_exists()`, and `where_not_exists()` accept only query builders, `SQL` objects, and SQLGlot nodes, plus `BuiltQuery` for the `sql.*_()` fragments, and raise `SQLBuilderError` for strings and other objects, where they parsed the value's text as SQL. Before: `.where_exists("SELECT 1 FROM y")`. After: `.where_exists(sql.select("1").from_("y"))`. @@ -70,12 +71,13 @@ important operational fixes. * Function calls and subqueries in table strings now raise `SQLBuilderError`; pass expression objects instead, such as `from_(sql.func("fn", 1), alias="f")` or `from_(sql.raw("fn(:n)", n=1), alias="f")`. A parenthesis inside a name quoted in the builder's dialect stays part of that name. * Query builders now validate table and alias strings. `from_`, `join`, `using`, `into`, `table`, and CTE names read the text with the builder's dialect and accept a dotted table name with an optional single-identifier alias (`t u` or `t AS u`), including quoted identifiers, and raise `SQLBuilderError` for any other text. Pass join conditions through `on`. DDL and statement target names accept a name only. T-SQL builders also accept a table variable such as `@tv`. Unquoted text holding whitespace other than ASCII space, tab, and line breaks, or format or control characters, raises. Table text must be trusted, because engines may resolve names to files or external sources; DuckDB users can set `enable_external_access` to `False` to block file access. * Framework default `commit_mode` changed from `"manual"` to `"autocommit"` across Litestar, FastAPI, Starlette, Flask, and Sanic (`"autocommit_include_redirect"` commits `2xx`–`3xx` responses for Inertia, HTMX, and HTML form redirects, and `"manual"` now explicitly rolls back uncommitted work on request release on every adapter). ([#916](https://github.com/litestar-org/sqlspec/pull/916)) +* BigQuery configurations use `commit_mode="manual"` in the Litestar, FastAPI, Starlette, Flask, and Sanic plugins, and an explicit `"autocommit"` or `"autocommit_include_redirect"` raises `ImproperConfigurationError` when the plugin is created. Remove the setting or set `commit_mode="manual"`, and call `commit()` in the handler to keep work done inside an explicit BigQuery transaction. * Updated Google ADK support to `google-adk >= 2.14.1` and moved ADK scoped state merging into store transactions (`create_session` and `append_event_and_update_state` replace `app_state` / `user_state` snapshot keyword arguments with `app_state_delta` and `user_state_delta`, merge atomically via row locks, `BEGIN IMMEDIATE`, Spanner read-write transactions, or compare-and-swap retries where row locks are unavailable, and return merged state in `StoredSession["state"]`); `get_events(limit=N)` across all 24 ADK stores now returns the `N` most recent events ordered chronologically ascending. ([#916](https://github.com/litestar-org/sqlspec/pull/916)) * FastAPI and Litestar cursor pagination requires a cursor secret (`pagination_cursor_secret`, `SQLSPEC_PAGINATION_CURSOR_SECRET`, or `SECRET_KEY`) and raises `ImproperConfigurationError` without one. Cursor tokens are bound to the request tenant of every tenant-enabled configuration, so tokens issued earlier restart from the first page. * FastAPI and Litestar search providers escape `%`, `_`, and `/` in search input by default; set `search_escape_wildcards=False` in the filter config to keep wildcard matching. * Explicit builder parameter names passed to `add_parameter(name=...)` or `load_parameters` must be ASCII identifiers or positional numbers. * `upgrade` and `downgrade` refuse to run when an applied migration file no longer matches its recorded checksum. Restore the file, or review the change and set `migration_config={"verify_checksums": False}`. -* On PostgreSQL, the migration tracking table is resolved in the configured `version_table_schema` or the session's `current_schema()`, never through `search_path`, and must be owned by the migrating role, a role it belongs to, or a superuser. +* On PostgreSQL, the migration tracking table is resolved in the configured `version_table_schema` or the session's `current_schema()`, never through `search_path`, and must be owned by the migrating role, a role it belongs to, or a superuser. The table is refused when it has rewrite rules, or when row-level security applies to the migrating role (forced, or enabled on a table the role does not own). * `delete_entries_by_session` on ADK memory stores now requires `app_name` and `user_id` and deletes only that owner's entries. * SQLite and aiosqlite treat a `file:` database path as a URI only when `uri=True` is set; without it the value is a plain filename. * `LocalStore`, `file`-protocol `FSSpecBackend`, and local `ObStoreBackend` delete directories only with `recursive=True` and refuse to delete their root. @@ -116,6 +118,7 @@ important operational fixes. * **Core & Query Builders**: Normalized generated parameter names and dialect-aware placeholder scanning (`sqlspec.core`), skipped statement caching for oversized scripts and scripts containing embedded values, preserved bare negative numeric literals during parameter normalization, and aligned mapping column ordering, `Merge` dict sources, scalar `parse_column_expression` parsing, and vector distance validation in `sqlspec.builder`. ([#916](https://github.com/litestar-org/sqlspec/pull/916)) * **Storage & Ingestion**: Confined local file URIs to resolved root paths after unquoting across `LocalStore`, `ObStoreBackend`, and `FSSpecBackend`, dialect-quoted staged merge and checkpoint DDL in `sqlspec.storage`, rejected empty conflict keys and empty parquet/ORC shards, and bounded gzip decompression fixture reads. ([#916](https://github.com/litestar-org/sqlspec/pull/916)) +* **Object storage keys**: `FSSpecBackend` copies, moves, and deletes keys containing `*`, `?`, or `[` literally instead of expanding them as glob patterns, and deletes on filesystems that implement only `rm()`. `s3a://`, `abfs://`, and `abfss://` URIs resolve object keys without the bucket or container prefix on the default backend. * **Adapters & Connection Pools**: Standardized ODBC and URI connection escaping and validation across database adapters, routed `AdbcConfig.provide_connection()` through the pool when configured, enforced `create_connection()` exception mapping (`db2`, `adbc`, `arrow_odbc`), aligned MySQL (`aiomysql`, `asyncmy`, `mysqlconnector`, `pymysql`) exception mapping parity and post-connect replica read-only initialization, enforced rollback-on-checkin and unhealthy connection eviction in sync and async pools, preserved caller-supplied connections and Oracle DRCP session resets on exit, handled MySQL `REPEATABLE READ` EOF snapshot rollback (`aiomysql`, `asyncmy`), and aligned PostgreSQL (`asyncpg`, `psycopg`, `psqlpy`) JSON string scalar decoding. ([#916](https://github.com/litestar-org/sqlspec/pull/916)) * **Web Frameworks, Events & Google ADK**: Aligned lazy request driver preparation, streaming session release via `ConnectionCleanupMiddleware`, and sub-second session store TTL expiration across `litestar`, `fastapi`, `starlette`, `flask`, and `sanic`, honored `extension_config["starlette"]` fallback/conflict checks in FastAPI, validated duplicate `session_key`/`connection_key`/`pool_key` configs in Litestar, enforced label allow-list validation in `PrometheusStatementObserver`, added channel-scoped event claims, lease-checked `ack`/`nack` updates, and `EventsConfig` (`max_attempts`, `retry_backoff_seconds`) exponential backoff in `sqlspec.extensions.events`, enforced `(app_name, user_id)` scoping across all 24 ADK session stores and 23 ADK memory stores, and resolved DuckDB 2.0.0 ADK foreign-key index/cascade delete constraints. ([#916](https://github.com/litestar-org/sqlspec/pull/916)) * **Migrations & CLI**: Propagated `MigrationError` for non-zero exit codes across migration CLI commands (including `sqlspec db fix` target collisions) and `adk memory verify`, escaped `.py` template docstrings and `.sql` metadata headers, fixed MySQL `--from-schema` / `dump-schema` `table_name` parameter binding (`MySQLSyncDataDictionary` / `MySQLAsyncDataDictionary`), switched Python migration inspection (`verify`, `--dry-run`, `stamp`, reconciliation) to AST parsing without module execution, rejected duplicate migration versions and non-empty `init` overwrites, skipped `-- name:` markers inside SQL literals/block comments in `SQLFileLoader`, restricted reconciler extension teardown strictly to explicit `downgrade("base")`, matched unquoted tracking table schemas case-insensitively while preserving quoted case, and enabled strict checksum verification across versioned extension migrations and `replaces` squashes. ([#916](https://github.com/litestar-org/sqlspec/pull/916)) diff --git a/docs/databases/mysql.md b/docs/databases/mysql.md index 8987b33f8..be990e7d9 100644 --- a/docs/databases/mysql.md +++ b/docs/databases/mysql.md @@ -425,6 +425,7 @@ Set `driver_features["reset_session_on_release"] = True` on `PyMysqlConfig`, `Ai - **Upserts with ON DUPLICATE KEY UPDATE**: Build conflict-tolerant writes with `sql.insert("users").values(...).on_duplicate_key_update(...)`, which compiles to MySQL and MariaDB `ON DUPLICATE KEY UPDATE` syntax. - **LAST_INSERT_ID() vs MariaDB RETURNING**: MariaDB 10.5+ supports `INSERT ... RETURNING` and `DELETE ... RETURNING` directly via `session.select(...)`. Standard MySQL does not support `RETURNING`, and SQLSpec captures `LAST_INSERT_ID()` into `result.last_inserted_id` and `result.rows_affected` on the returned `SQLResult`. - **Placeholder Compilation and utf8mb4**: Write `:named` or `?` parameters in SQLSpec; all MySQL adapters compile placeholders to `%s` automatically while preserving literal `%` signs in `LIKE '100%'` expressions. Always use `utf8mb4` so 4-byte UTF-8 characters never truncate. +- **Multi-Statement Hardening on `aiomysql` and `asyncmy`**: Upstream `aiomysql` and `asyncmy` unconditionally enable `CLIENT.MULTI_STATEMENTS` during the MySQL handshake without exposing a disable keyword argument. By default, SQLSpec sends `COM_SET_OPTION` (`MYSQL_OPTION_MULTI_STATEMENTS_OFF`) once per physical connection (and re-applies it after `COM_RESET_CONNECTION`) so single-statement `execute()` calls reject stacked SQL; use `execute_script()` for explicit multi-statement execution or pass `connection_config={"client_flag": CLIENT.MULTI_STATEMENTS}` to opt in. - **Implicit DDL Commits**: MySQL and MariaDB issue an implicit `COMMIT` before and after `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, and `TRUNCATE TABLE`. Keep schema migrations separate from transactional application tests that rely on rollback isolation. ## Read Replicas and GTID Waits diff --git a/docs/learn/migrations.md b/docs/learn/migrations.md index 81838bf4d..3b1769288 100644 --- a/docs/learn/migrations.md +++ b/docs/learn/migrations.md @@ -123,7 +123,7 @@ Choose the squash strategy that matches your migration history: - `ddl`: Parses chronological DDL with SQLGlot and folds `CREATE` / `ALTER` / `DROP` / `RENAME` operations into net `CREATE TABLE`, `CREATE SEQUENCE`, and `CREATE INDEX` statements. Fails closed with `SquashValidationError` whenever a statement (such as data DML, views/triggers, dynamic PL/SQL, or destructive column rewrites) cannot be folded faithfully. - `schema`: Emits topological DDL reflected directly from the live database catalog via `driver.data_dictionary.get_schema_ddl(...)`. -Schema extraction (`schema` squash, `create-migration --from-schema`, and `dump-schema`) emits tables, columns, indexes, and constraints. It refuses to run when an included table is protected by row-level security: PostgreSQL RLS or policies, Oracle VPD policies, or SQL Server security policies, because rebuilding the database from the dump would silently drop that protection. On PostgreSQL it also logs one `migration.schema_extraction.omitted` warning naming the grants, functions, triggers, and domains it does not emit; carry those over with `--strategy ddl` or hand-written migrations. The tracking table and its `_lock` table are ignored by these checks. +Schema extraction (`schema` squash, `create-migration --from-schema`, and `dump-schema`) emits tables, columns, indexes, and constraints. It refuses to run when an included table is protected by row-level security: PostgreSQL RLS or policies, Oracle VPD policies, or SQL Server security policies, because rebuilding the database from the dump would silently drop that protection. On PostgreSQL it also logs one `migration.schema_extraction.omitted` warning naming the grants, materialized views, functions, triggers, and domains it does not emit; carry those over with `--strategy ddl` or hand-written migrations. The tracking table and its `_lock` table are ignored by these checks. When `upgrade()` runs against downstream environments after a squash has been committed: @@ -187,7 +187,7 @@ Remaining built-in refusals and their remedies: ## Schema Snapshots, Verification, and Data Bootstrap - **Read-Only `verify`**: Checks applied tracker rows against migration files on disk without creating `version_table_name` on empty databases. Combine `--strict` with `--fail-on-pending` as a pre-deploy CI gate. -- **Extraction limits**: `dump-schema` and `--from-schema` refuse tables protected by PostgreSQL RLS, Oracle VPD, or SQL Server security policies, and warn about PostgreSQL grants, functions, triggers, and domains they do not emit (see Squashing Strategies above). +- **Extraction limits**: `dump-schema` and `--from-schema` refuse tables protected by PostgreSQL RLS, Oracle VPD, or SQL Server security policies, and warn about PostgreSQL grants, materialized views, functions, triggers, and domains they do not emit (see Squashing Strategies above). - **Deterministic `dump-schema`**: Exports live schema DDL excluding `version_table_name` by default (`--include-migrations-table` includes it; `--target-dialect` transpiles across SQLGlot dialects). Omitting `--output` writes pure SQL to stdout so `sqlspec dump-schema > schema.sql` produces a clean SQL script. ### How Data Bootstrap Classifies Sources diff --git a/pyproject.toml b/pyproject.toml index 3b70169a6..29cb245f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -233,7 +233,6 @@ exclude = [ "sqlspec/config.py", # Main config "sqlspec/**/__init__.py", # Init files (usually just imports) "sqlspec/protocols.py", # Protocol definitions - "sqlspec/driver/_routing.py", # interpreted: generic routing facades retain typed Python context and streaming boundaries "sqlspec/migrations/commands.py", # interpreted: inspect.signature/functools.wraps on coroutines "sqlspec/dialects/postgres/_pgvector.py", # interpreted: Dialect subclass needs sqlglot's metaclass "sqlspec/dialects/postgres/_paradedb.py", # interpreted: Dialect subclass needs sqlglot's metaclass @@ -288,6 +287,7 @@ include = [ "sqlspec/extensions/fastapi/providers.py", # FastAPI filter provider helpers "sqlspec/extensions/litestar/providers.py", # Litestar filter provider helpers "sqlspec/extensions/litestar/_tenant_resolution.py", # Litestar tenant resolution over verified auth values + "sqlspec/extensions/events/_buffer.py", # Event message buffer helpers "sqlspec/extensions/events/_hints.py", # Adapter event runtime defaults "sqlspec/extensions/events/_models.py", # Event message value object "sqlspec/extensions/events/_names.py", # Event channel/table name validators @@ -295,12 +295,24 @@ include = [ "sqlspec/extensions/events/_channel.py", # Event channel runtime and iterator shell "sqlspec/extensions/events/_queue.py", # Table-backed event queue "sqlspec/extensions/events/primitives.py", # Table-queue SQL and claim primitives + "sqlspec/extensions/adk/_table_utils.py", # ADK table schema validation helpers "sqlspec/extensions/adk/_types.py", # ADK session/event record types "sqlspec/extensions/adk/memory/_types.py", # ADK memory record types "sqlspec/extensions/adk/artifact/_types.py", # ADK artifact record types + "sqlspec/extensions/starlette/_state.py", # Starlette extension config state model + "sqlspec/extensions/starlette/_utils.py", # Starlette per-request session and tenant state helpers + "sqlspec/extensions/flask/_state.py", # Flask extension config state model + "sqlspec/extensions/flask/_utils.py", # Flask per-request session and tenant state helpers + "sqlspec/extensions/flask/_portal.py", # Flask blocking portal session view + "sqlspec/extensions/litestar/_utils.py", # Litestar per-request scope and session lease helpers + "sqlspec/extensions/sanic/_state.py", # Sanic extension config state model + "sqlspec/extensions/sanic/_utils.py", # Sanic per-request session and tenant state helpers "sqlspec/adapters/**/core.py", # Adapter compiled helpers + "sqlspec/adapters/**/driver.py", # Adapter compiled drivers "sqlspec/adapters/**/type_converter.py", # All adapters type converters "sqlspec/adapters/**/tenant.py", # Adapter tenant strategies and leases + "sqlspec/adapters/snowflake/migrations.py", # Snowflake compiled migration tracker + "sqlspec/adapters/teradata/migrations.py", # Teradata compiled migration tracker "sqlspec/adapters/asyncpg/events/_hub.py", # Native event listener runtimes "sqlspec/adapters/oracledb/events/_hub.py", "sqlspec/adapters/psqlpy/events/_hub.py", @@ -326,6 +338,7 @@ include = [ "sqlspec/utils/correlation.py", # Correlation context helpers "sqlspec/utils/tenant/**/*.py", # Tenant context, scopes, and shared session-setting helpers "sqlspec/utils/portal.py", # Thread portal utilities + "sqlspec/utils/routing.py", # Request routing directives and HMAC pin helpers "sqlspec/utils/schema.py", # Schema transformation (refactored for mypyc) "sqlspec/utils/uuids.py", # UUID utilities ] @@ -344,7 +357,7 @@ require-runtime-features = ["mypyc"] [tool.hatch.build.targets.wheel.hooks.mypyc.options] debug_level = "0" # No debug info in production (0-2) -multi_file = true # Enable cross-module optimization +multi_file = true # One C file per module to bound compile memory; Stage-3 LTO (-flto=auto) inlines across TUs opt_level = "3" # Maximum optimization (0-3) diff --git a/sqlspec/adapters/adbc/_typing.py b/sqlspec/adapters/adbc/_typing.py index d29ca015c..66ad2e466 100644 --- a/sqlspec/adapters/adbc/_typing.py +++ b/sqlspec/adapters/adbc/_typing.py @@ -68,6 +68,9 @@ def __exit__(self, *_: object) -> None: self.cursor.close() +_resolved_dialect_resolver: Any = None + + class AdbcSessionContext: """Sync context manager for ADBC sessions. @@ -109,12 +112,17 @@ def __init__( self._driver: AdbcDriver | None = None def __enter__(self) -> "AdbcDriver": + global _resolved_dialect_resolver driver_class = self._driver_class if driver_class is None: from sqlspec.adapters.adbc.driver import AdbcDriver driver_class = AdbcDriver - from sqlspec.adapters.adbc.core import resolve_dialect_name + resolve_dialect_name = _resolved_dialect_resolver + if resolve_dialect_name is None: + from sqlspec.adapters.adbc.core import resolve_dialect_name + + _resolved_dialect_resolver = resolve_dialect_name self._connection = self._acquire_connection() try: diff --git a/sqlspec/adapters/adbc/adk/store.py b/sqlspec/adapters/adbc/adk/store.py index 3fd58d1fa..62ff16a99 100644 --- a/sqlspec/adapters/adbc/adk/store.py +++ b/sqlspec/adapters/adbc/adk/store.py @@ -3,7 +3,7 @@ import contextlib import re import struct -from collections.abc import Iterator, Mapping +from collections.abc import Generator, Mapping from datetime import UTC, datetime, timedelta from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, Literal, NotRequired, cast @@ -3925,7 +3925,7 @@ def __getattr__(self, name: str) -> Any: @contextlib.contextmanager -def _adbc_atomic_connection(config: "AdbcConfig", dialect: str) -> Iterator[Any]: +def _adbc_atomic_connection(config: "AdbcConfig", dialect: str) -> Generator[Any, None, None]: """Require transaction support before compound Teradata and DuckDB mutations. Yields: @@ -4031,7 +4031,7 @@ def _is_error_matching(exc: Exception, patterns: "tuple[str, ...]") -> bool: @contextlib.contextmanager -def _provide_state_transaction(config: "AdbcConfig", dialect: str) -> Iterator[Any]: +def _provide_state_transaction(config: "AdbcConfig", dialect: str) -> Generator[Any, None, None]: """Open a transaction for scoped-state merges with SQLite immediate write locking. Yields: diff --git a/sqlspec/adapters/adbc/core.py b/sqlspec/adapters/adbc/core.py index 2d917aae6..3ecb508f4 100644 --- a/sqlspec/adapters/adbc/core.py +++ b/sqlspec/adapters/adbc/core.py @@ -22,16 +22,13 @@ from sqlspec.adapters.adbc._typing import AdbcNativeError, AdbcStatusCode from sqlspec.adapters.adbc.type_converter import ADBCOutputConverter, get_adbc_type_converter -from sqlspec.core import ( - DriverParameterProfile, - ParameterStyle, - StatementConfig, - build_null_pruning_transform, - build_statement_config_from_profile, -) -from sqlspec.core.parameters import type_coercion_dispatcher +from sqlspec.core.parameters._processor import type_coercion_dispatcher +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._transformers import build_null_pruning_transform +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult -from sqlspec.data_dictionary import normalize_dialect_name +from sqlspec.core.statement import StatementConfig +from sqlspec.data_dictionary._registry import normalize_dialect_name from sqlspec.exceptions import ( CheckViolationError, DatabaseConnectionError, @@ -54,11 +51,12 @@ UniqueViolationError, _classify_timeout_or_cancellation, map_sqlstate_to_exception, + with_cause, ) from sqlspec.typing import PGVECTOR_INSTALLED, Empty, import_optional_attr from sqlspec.utils.correlation import CorrelationContext from sqlspec.utils.module_loader import import_optional, import_string -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import normalize_identifier, split_qualified_identifier from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_rowcount, has_sqlstate, resolve_row_format @@ -67,8 +65,8 @@ if TYPE_CHECKING: from collections.abc import Callable - from sqlspec.core import SQL from sqlspec.core.cache import CachedQuery + from sqlspec.core.statement import SQL from sqlspec.utils.dispatch import TypeDispatcher __all__ = ( @@ -618,7 +616,7 @@ def resolve_driver_connect_func(driver_name: str | None, uri: str | None) -> "Ca connect_func = import_string(driver_path) except ImportError as exc: msg = f"Failed to import connect function from '{driver_path}'. Is the driver installed? Error: {exc}" - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc if not callable(connect_func): msg = f"The path '{driver_path}' did not resolve to a callable function." @@ -1899,7 +1897,7 @@ def _rewrite_postgres_uuid_placeholders( ) except ParseError as exc: msg = f"Failed to parse PostgreSQL ADBC SQL for UUID parameter binding: {exc}" - raise SQLSpecError(msg) from exc + raise with_cause(SQLSpecError(msg), exc) from exc if not expressions: msg = "Failed to parse PostgreSQL ADBC SQL for UUID parameter binding: SQLGlot returned no statements." raise SQLSpecError(msg) @@ -2006,7 +2004,7 @@ def _convert_uuid_value(value: Any, ordinal: int, row_number: int, element_index try: return str(uuid_from_string(value)) except (TypeError, ValueError) as exc: - raise SQLSpecError(_uuid_binding_error(ordinal, value, row_number, element_index)) from exc + raise with_cause(SQLSpecError(_uuid_binding_error(ordinal, value, row_number, element_index)), exc) from exc def _null_array_element_error(ordinal: int, row_number: int, element_index: int) -> str: @@ -2887,7 +2885,7 @@ def _normalize_db2_cli_dsn(config: "dict[str, Any]", uri: str) -> None: port_num = int(port_raw) except ValueError as exc: msg = f"Db2 port must be an integer between 1 and 65535, got {port_raw!r}." - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc if not 1 <= port_num <= _MAX_TCP_PORT: msg = f"Db2 port must be an integer between 1 and 65535, got {port_num}." raise ImproperConfigurationError(msg) diff --git a/sqlspec/adapters/adbc/driver.py b/sqlspec/adapters/adbc/driver.py index 0a10f9823..37b4525da 100644 --- a/sqlspec/adapters/adbc/driver.py +++ b/sqlspec/adapters/adbc/driver.py @@ -8,7 +8,9 @@ import sys from pathlib import Path from time import perf_counter -from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast, final +from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast + +from mypy_extensions import mypyc_attr from sqlspec.adapters.adbc._typing import AdbcCursor, AdbcNativeError, AdbcNotSupportedError, AdbcSessionContext from sqlspec.adapters.adbc.core import ( @@ -41,25 +43,23 @@ resolve_rowcount, ) from sqlspec.adapters.adbc.data_dictionary import AdbcDataDictionary -from sqlspec.core import ( - SQL, - StatementConfig, - build_arrow_result_from_reader, - build_arrow_result_from_table, - get_cache_config, - register_driver_profile, -) -from sqlspec.core.parameters import ParameterValidator -from sqlspec.core.result import DMLResult -from sqlspec.driver import BaseSyncExceptionHandler, SyncDriverAdapterBase, SyncRowStream, validate_savepoint_name -from sqlspec.driver._common import cache_hit_hooks_unchanged -from sqlspec.exceptions import DatabaseConnectionError, DataError, SQLSpecError, TransactionError -from sqlspec.observability import resolve_db_system +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.parameters._validator import ParameterValidator +from sqlspec.core.result._base import DMLResult, build_arrow_result_from_reader, build_arrow_result_from_table +from sqlspec.core.statement import SQL, StatementConfig +from sqlspec.driver._common import cache_hit_hooks_unchanged, validate_savepoint_name +from sqlspec.driver._exception_handler import BaseSyncExceptionHandler +from sqlspec.driver._stream import SyncRowStream +from sqlspec.driver._sync import SyncDriverAdapterBase +from sqlspec.exceptions import DatabaseConnectionError, DataError, SQLSpecError, TransactionError, with_cause +from sqlspec.observability._common import resolve_db_system +from sqlspec.storage._utils import import_pyarrow from sqlspec.utils.arrow_helpers import arrow_reader_with_deferred_close, arrow_table_to_pylist from sqlspec.utils.correlation import CorrelationContext from sqlspec.utils.logging import get_logger from sqlspec.utils.module_loader import ensure_pyarrow, import_optional, import_optional_attr -from sqlspec.utils.serializers import to_json +from sqlspec.utils.serializers._json import to_json from sqlspec.utils.text import ( normalize_identifier, quote_identifier, @@ -71,10 +71,13 @@ from collections.abc import Callable, Mapping, Sequence from sqlspec.adapters.adbc._typing import AdbcConnection, AdbcRawCursor - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, SQLResult, Statement, StatementFilter - from sqlspec.driver import CachedQuery, ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.cache import CachedQuery + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.core.statement import Statement + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry from sqlspec.typing import ArrowReturnFormat, StatementParameters __all__ = ("AdbcCursor", "AdbcDriver", "AdbcExceptionHandler", "AdbcSessionContext") @@ -90,7 +93,7 @@ _SCRIPT_PARAMETER_VALIDATOR = ParameterValidator() -@final +@mypyc_attr(allow_interpreted_subclasses=True) class AdbcExceptionHandler(BaseSyncExceptionHandler): """Context manager for handling ADBC database exceptions. @@ -241,7 +244,7 @@ def close(self, error: bool = False) -> None: self._owner.close() -@final +@mypyc_attr(allow_interpreted_subclasses=True) class AdbcDriver(SyncDriverAdapterBase): """ADBC driver for Arrow Database Connectivity. @@ -589,7 +592,7 @@ def begin(self) -> None: except (AdbcNotSupportedError, NotImplementedError): pass except AdbcNativeError as exc: - raise self._map_native_error(exc) from exc + raise with_cause(self._map_native_error(exc), exc) from exc if ( native is not None and callable(getattr(native, "set_autocommit", None)) @@ -607,12 +610,14 @@ def begin(self) -> None: except AdbcNotSupportedError as exc: self._previous_autocommit = None msg = f"ADBC {self._dialect_name} cannot disable autocommit; this driver supports autocommit-only operations." - raise TransactionError(msg) from exc + error = TransactionError(msg) + error.__cause__ = exc + raise error from exc except BaseException as exc: with contextlib.suppress(BaseException): native.set_autocommit(True) if isinstance(exc, AdbcNativeError): - raise self._map_native_error(exc) from exc + raise with_cause(self._map_native_error(exc), exc) from exc raise self._previous_autocommit = True self._transaction_active = True @@ -1059,8 +1064,7 @@ def _align_snowflake_load_columns( def resolve(name: str) -> str: return resolve_catalog_identifier(name, catalog_columns) or name - ensure_pyarrow() - import pyarrow as pa + pa = import_pyarrow() source_data: Any = source.get_data() if hasattr(source, "get_data") else source if hasattr(source_data, "read_next_batch"): @@ -1079,22 +1083,6 @@ def resolve(name: str) -> str: {resolve(name): value for name, value in type_casts.items()} if type_casts else type_casts, ) - def load_from_storage( - self, - table: str, - source: "StorageDestination", - *, - file_format: "StorageFormat", - partitioner: "dict[str, object] | None" = None, - overwrite: bool = False, - **kwargs: Any, - ) -> "StorageBridgeJob": - """Read an artifact from storage and ingest it via ADBC.""" - - return super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs - ) - def collect_rows(self, cursor: "AdbcRawCursor", fetched: "list[Any]") -> "tuple[list[Any], list[str], int]": """Collect ADBC rows for the direct execution path.""" column_names = self._resolve_column_names(cursor.description) @@ -1210,7 +1198,7 @@ def _finish_transaction(self, *, commit: bool) -> None: self._previous_autocommit = None self._transaction_active = False if isinstance(primary, AdbcNativeError): - raise self._map_native_error(primary) from primary + raise with_cause(self._map_native_error(primary), primary) from primary if primary is not None: raise primary diff --git a/sqlspec/adapters/adbc/pool.py b/sqlspec/adapters/adbc/pool.py index da59552e4..1005f6ebf 100644 --- a/sqlspec/adapters/adbc/pool.py +++ b/sqlspec/adapters/adbc/pool.py @@ -9,7 +9,7 @@ from sqlspec.adapters.adbc._typing import AdbcNativeError from sqlspec.adapters.adbc.core import create_mapped_exception -from sqlspec.exceptions import ConnectionTimeoutError, DatabaseConnectionError, ImproperConfigurationError +from sqlspec.exceptions import ConnectionTimeoutError, DatabaseConnectionError, ImproperConfigurationError, with_cause if TYPE_CHECKING: from collections.abc import Callable @@ -98,7 +98,7 @@ def acquire(self) -> "AdbcConnection": try: connection = self._seed.adbc_clone() except AdbcNativeError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc except BaseException: with self._condition: self._reservations -= 1 diff --git a/sqlspec/adapters/adbc/type_converter.py b/sqlspec/adapters/adbc/type_converter.py index 63d1b2ee1..00aac25d8 100644 --- a/sqlspec/adapters/adbc/type_converter.py +++ b/sqlspec/adapters/adbc/type_converter.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Final -from sqlspec.utils.serializers import to_json +from sqlspec.utils.serializers._json import to_json if TYPE_CHECKING: from collections.abc import Callable diff --git a/sqlspec/adapters/aiomysql/_typing.py b/sqlspec/adapters/aiomysql/_typing.py index 29bfd0f2d..065306f5d 100644 --- a/sqlspec/adapters/aiomysql/_typing.py +++ b/sqlspec/adapters/aiomysql/_typing.py @@ -19,16 +19,15 @@ from aiomysql.cursors import DictCursor as _AiomysqlDictCursor from pymysql.constants import FIELD_TYPE as _PYMYSQL_FIELD_TYPE +from sqlspec.driver._session import AsyncDriverSessionContext + if TYPE_CHECKING: - from collections.abc import Awaitable, Callable - from types import TracebackType from typing import Protocol, TypeAlias from pymysql.err import Error as _PymysqlError from pymysql.err import MySQLError as _PymysqlMySQLError from sqlspec.adapters.aiomysql.driver import AiomysqlDriver - from sqlspec.core import StatementConfig class AiomysqlConnectionProtocol(Protocol): async def cursor(self, cursor: "type[AiomysqlRawCursor] | None" = None) -> "AiomysqlRawCursor": ... @@ -87,8 +86,6 @@ class AiomysqlFieldTypeProtocol(Protocol): "AiomysqlSessionContext", ) -_resolved_driver_type: "type[AiomysqlDriver] | None" = None - class AiomysqlCursor: """Context manager for aiomysql cursor operations. @@ -122,85 +119,16 @@ async def __aexit__(self, *_: object) -> None: await self.cursor.close() -class AiomysqlSessionContext: - """Async context manager for aiomysql sessions. +class AiomysqlSessionContext(AsyncDriverSessionContext): + """Async context manager for aiomysql sessions.""" - This class is intentionally excluded from mypyc compilation to avoid ABI - boundary issues. It receives callables from uncompiled config classes and - instantiates compiled Driver objects, acting as a bridge between compiled - and uncompiled code. + __slots__ = () - Uses callable-based connection management to decouple from config implementation. - """ + if TYPE_CHECKING: - __slots__ = ( - "_acquire_connection", - "_connection", - "_driver", - "_driver_features", - "_prepare_driver", - "_release_connection", - "_statement_config", - ) - - def __init__( - self, - acquire_connection: "Callable[[], Awaitable[AiomysqlConnection]]", - release_connection: "Callable[..., Any]", - statement_config: "StatementConfig", - driver_features: "dict[str, Any]", - prepare_driver: "Callable[[AiomysqlDriver], AiomysqlDriver]", - ) -> None: - self._acquire_connection = acquire_connection - self._release_connection = release_connection - self._statement_config = statement_config - self._driver_features = driver_features - self._prepare_driver = prepare_driver - self._connection: AiomysqlConnection | None = None - self._driver: AiomysqlDriver | None = None - - async def __aenter__(self) -> "AiomysqlDriver": - driver_cls = _get_aiomysql_driver_type() - self._connection = await self._acquire_connection() - try: - self._driver = driver_cls( - connection=self._connection, - statement_config=self._statement_config, - driver_features=self._driver_features, - ) - return self._prepare_driver(self._driver) - except BaseException as exc: - with contextlib.suppress(Exception): - await self.__aexit__(type(exc), exc, exc.__traceback__) - raise - - async def __aexit__( - self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None" - ) -> "bool | None": - driver = self._driver - try: - if driver is not None: - await driver._close_open_streams(error=exc_type is not None) - except BaseException as close_exc: - if exc_type is None: - exc_type, exc_val, exc_tb = type(close_exc), close_exc, close_exc.__traceback__ - raise - finally: - try: - if self._connection is not None: - await self._release_connection(self._connection, exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) - self._connection = None - finally: - self._driver = None - if driver is not None: - driver._mark_closed() - return None - - -def _get_aiomysql_driver_type() -> "type[AiomysqlDriver]": - global _resolved_driver_type - if _resolved_driver_type is None: - from sqlspec.adapters.aiomysql.driver import AiomysqlDriver as _Driver - - _resolved_driver_type = _Driver - return _resolved_driver_type + async def __aenter__(self) -> "AiomysqlDriver": ... + + def _resolve_driver_class(self) -> "type[AiomysqlDriver]": + from sqlspec.adapters.aiomysql.driver import AiomysqlDriver + + return AiomysqlDriver diff --git a/sqlspec/adapters/aiomysql/adk/store.py b/sqlspec/adapters/aiomysql/adk/store.py index 43c3177d5..0f652728b 100644 --- a/sqlspec/adapters/aiomysql/adk/store.py +++ b/sqlspec/adapters/aiomysql/adk/store.py @@ -89,6 +89,7 @@ async def create_tables(self) -> None: await driver.execute_script(await self._app_states_table_ddl()) await driver.execute_script(await self._user_states_table_ddl()) await driver.execute_script(await self._metadata_table_ddl()) + await self.reconcile_schema(assume_existing=True) async def reconcile_companion_objects_async(self, driver: Any) -> None: """Reconcile legacy column collations and missing session/event secondary indexes.""" @@ -770,6 +771,7 @@ async def create_tables(self) -> None: async with self._config.provide_session() as driver: await driver.execute_script(await self._memory_table_ddl()) + await self.reconcile_schema(assume_existing=True) async def reconcile_companion_objects_async(self, driver: Any) -> None: """Reconcile legacy memory identity collations and missing ordinary/FULLTEXT indexes.""" diff --git a/sqlspec/adapters/aiomysql/config.py b/sqlspec/adapters/aiomysql/config.py index a4cfa17ff..f89417102 100644 --- a/sqlspec/adapters/aiomysql/config.py +++ b/sqlspec/adapters/aiomysql/config.py @@ -55,6 +55,9 @@ "build_connection_config", ) +_COM_SET_OPTION: Final[int] = 0x1B +_MYSQL_OPTION_MULTI_STATEMENTS_OFF: Final[bytes] = b"\x01\x00" +_CLIENT_MULTI_STATEMENTS: Final[int] = 1 << 16 _COM_RESET_CONNECTION: Final[int] = 0x1F _POOL_ONLY_CONFIG_KEYS = frozenset({"maxsize", "minsize", "pool_recycle"}) aiomysql: "AiomysqlModule" = cast("AiomysqlModule", AiomysqlModule) @@ -296,7 +299,7 @@ async def __aexit__( class AiomysqlConfig(AsyncDatabaseConfig[AiomysqlConnection, "AiomysqlPool", AiomysqlDriver]): """Configuration for aiomysql database connections.""" - __slots__ = ("_initialized_connections", "_read_only", "_user_connection_hook") + __slots__ = ("_allow_multi_statements", "_initialized_connections", "_read_only", "_user_connection_hook") driver_type: ClassVar[type[AiomysqlDriver]] = AiomysqlDriver connection_type: "ClassVar[type[Any]]" = cast("type[Any]", AiomysqlConnection) @@ -352,6 +355,10 @@ def __init__( self._read_only: bool = ( bool(kwargs.get("is_replica", False)) if replica_read_only is None else bool(replica_read_only) ) + raw_client_flag = connection_config.get("client_flag") + self._allow_multi_statements: bool = bool( + isinstance(raw_client_flag, int) and (raw_client_flag & _CLIENT_MULTI_STATEMENTS) + ) self._initialized_connections: WeakSet[Any] = WeakSet() features_dict.setdefault("enable_local_infile_bulk_load", connection_config["local_infile"]) @@ -483,13 +490,23 @@ def _pool_kwargs(self) -> "dict[str, Any]": return pool_kwargs async def _ensure_connection(self, connection: "AiomysqlConnection") -> None: - """Ensure read-only session state and connection callback run once per physical connection. + """Disable implicit multi-statements and apply read-only / create hooks once per connection. - Uses WeakSet tracking to ensure initialization runs once per physical connection. + Upstream ``aiomysql`` unconditionally ORs ``CLIENT.MULTI_STATEMENTS`` during the initial + handshake without an opt-out keyword parameter. Unless ``CLIENT.MULTI_STATEMENTS`` is + explicitly requested via ``connection_config["client_flag"]``, SQLSpec sends + ``COM_SET_OPTION(MYSQL_OPTION_MULTI_STATEMENTS_OFF)`` once per physical connection and + again after ``COM_RESET_CONNECTION`` so single-statement queries cannot smuggle stacked SQL. """ - if not self._read_only and self._user_connection_hook is None: + if self._allow_multi_statements and not self._read_only and self._user_connection_hook is None: return if connection not in self._initialized_connections: + if not self._allow_multi_statements: + try: + await _disable_multi_statements(connection) + except Exception as exc: + _raise_mapped_connection_error(exc) + raise if self._read_only: try: async with AiomysqlCursor(connection) as cursor: @@ -549,6 +566,20 @@ def _is_autocommit_enabled(connection: Any) -> bool: return bool(getattr(connection, "autocommit_mode", False)) +async def _disable_multi_statements(connection: Any) -> None: + """Send ``COM_SET_OPTION(MYSQL_OPTION_MULTI_STATEMENTS_OFF)`` when supported by the connection.""" + execute_command = getattr(connection, "_execute_command", None) + read_packet = getattr(connection, "_read_packet", None) + if not callable(execute_command) or not callable(read_packet): + return + result = execute_command(_COM_SET_OPTION, _MYSQL_OPTION_MULTI_STATEMENTS_OFF) + if inspect.isawaitable(result): + await result + packet = read_packet() + if inspect.isawaitable(packet): + await packet + + async def _reset_session(config: Any, connection: Any) -> None: """Reset server session state on a released connection, closing it when the reset cannot complete. diff --git a/sqlspec/adapters/aiomysql/core.py b/sqlspec/adapters/aiomysql/core.py index fc0aebba4..b6a13d2ee 100644 --- a/sqlspec/adapters/aiomysql/core.py +++ b/sqlspec/adapters/aiomysql/core.py @@ -7,9 +7,10 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast from sqlspec.adapters.aiomysql._typing import AiomysqlPymysqlError, AiomysqlSSCursor -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult -from sqlspec.driver import rows_to_dicts +from sqlspec.driver._stream import rows_to_dicts from sqlspec.exceptions import ( CheckViolationError, ConnectionTimeoutError, @@ -31,14 +32,15 @@ ) from sqlspec.utils.config_tools import parse_mysql_dsn from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import quote_backtick_identifier, split_qualified_identifier from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_lastrowid, has_rowcount, has_sqlstate, has_type_code, resolve_row_format if TYPE_CHECKING: from sqlspec.core.cache import CachedQuery - from sqlspec.core.parameters import ParameterValidator + from sqlspec.core.parameters._validator import ParameterValidator + from sqlspec.core.statement import StatementConfig __all__ = ( "AiomysqlStreamSource", @@ -627,7 +629,7 @@ def create_mapped_exception(error: Any, *, logger: Any | None = None) -> SQLSpec dispatch = _MYSQL_DATA_ERROR_DISPATCH.get(error_code) if error_code is not None else None if dispatch is not None: return _create_mysql_error(error, sqlstate, error_code, dispatch[0], dispatch[1]) - if sqlstate_prefix == "22" or type(error).__name__ == "DataError": + if sqlstate_prefix == "22" or type(error).__name__ == "DataError" or isinstance(error, TypeError): dispatch = _MYSQL_SQLSTATE_PREFIX_DISPATCH["22"] return _create_mysql_error(error, sqlstate, error_code, dispatch[0], dispatch[1]) diff --git a/sqlspec/adapters/aiomysql/driver.py b/sqlspec/adapters/aiomysql/driver.py index 80195d056..98633ae85 100644 --- a/sqlspec/adapters/aiomysql/driver.py +++ b/sqlspec/adapters/aiomysql/driver.py @@ -14,6 +14,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Final, Literal, cast +from mypy_extensions import mypyc_attr + from sqlspec.adapters.aiomysql._typing import ( AIOMYSQL_INSERT_VALUES_PATTERN, AiomysqlCursor, @@ -44,20 +46,29 @@ resolve_rowcount, ) from sqlspec.adapters.aiomysql.data_dictionary import AiomysqlDataDictionary -from sqlspec.core import ArrowResult, get_cache_config, register_driver_profile -from sqlspec.driver import AsyncDriverAdapterBase, AsyncRowStream, BaseAsyncExceptionHandler +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.driver._async import AsyncDriverAdapterBase from sqlspec.driver._common import cache_hit_hooks_unchanged +from sqlspec.driver._exception_handler import BaseAsyncExceptionHandler +from sqlspec.driver._stream import AsyncRowStream +from sqlspec.exceptions import with_cause from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json +from sqlspec.utils.serializers._json import from_json +from sqlspec.utils.sync_tools import exit_async_context from sqlspec.utils.type_guards import supports_json_type if TYPE_CHECKING: from collections.abc import Callable, Mapping, Sequence + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.aiomysql._typing import AiomysqlConnection - from sqlspec.core import SQL, SQLResult, StatementConfig - from sqlspec.driver import CachedQuery, ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.core.cache import CachedQuery + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.core.statement import SQL, StatementConfig + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry __all__ = ("AiomysqlCursor", "AiomysqlDriver", "AiomysqlExceptionHandler", "AiomysqlSessionContext") @@ -67,13 +78,16 @@ AiomysqlFieldType.JSON if AiomysqlFieldType is not None and supports_json_type(AiomysqlFieldType) else None ) AIOMYSQL_JSON_TYPE_CODES: Final[set[int]] = {json_type_value} if json_type_value is not None else set() +_MYSQL_CLIENT_PACKAGES: Final[frozenset[str]] = frozenset({"aiomysql", "pymysql"}) +@mypyc_attr(allow_interpreted_subclasses=True) class AiomysqlExceptionHandler(BaseAsyncExceptionHandler): """Async context manager for handling aiomysql (MySQL) database exceptions. Maps MySQL error codes and SQLSTATE values from ``pymysql.err.Error`` to - SQLSpec exceptions stored on ``pending_exception``. + SQLSpec exceptions stored on ``pending_exception``. A ``TypeError`` is mapped + only when the aiomysql or PyMySQL client raised it while binding parameters. """ __slots__ = () @@ -81,12 +95,15 @@ class AiomysqlExceptionHandler(BaseAsyncExceptionHandler): def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "BaseException") -> bool: if exc_type is None: return False - if issubclass(exc_type, AiomysqlPymysqlError): + if issubclass(exc_type, AiomysqlPymysqlError) or ( + issubclass(exc_type, TypeError) and _raised_by_mysql_client(exc_val) + ): self.pending_exception = create_mapped_exception(exc_val) return True return False +@mypyc_attr(allow_interpreted_subclasses=True) class AiomysqlDriver(AsyncDriverAdapterBase): """MySQL/MariaDB database driver using aiomysql client library. @@ -96,7 +113,7 @@ class AiomysqlDriver(AsyncDriverAdapterBase): """ __slots__ = ("_data_dictionary", "_json_deserializer") - dialect = "mysql" + dialect: "DialectType | None" = "mysql" def __init__( self, @@ -232,7 +249,7 @@ async def begin(self) -> None: try: await self.connection.begin() except AiomysqlPymysqlError as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e async def commit(self) -> None: """Commit the current transaction. @@ -243,7 +260,7 @@ async def commit(self) -> None: try: await self.connection.commit() except AiomysqlPymysqlError as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e async def rollback(self) -> None: """Rollback the current transaction. @@ -254,7 +271,7 @@ async def rollback(self) -> None: try: await self.connection.rollback() except AiomysqlPymysqlError as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e def with_cursor(self, connection: "AiomysqlConnection") -> "AiomysqlCursor": """Create cursor context manager for the connection. @@ -365,6 +382,7 @@ async def load_from_arrow( if use_infile: payload = encode_records_for_local_infile(records) tmp_name = await asyncio.to_thread(_write_local_infile_tempfile, payload) + load_error: BaseException | None = None try: load_sql = build_load_data_statement(table, columns, escape_percent=True) exc_handler = self.handle_database_exceptions() @@ -372,8 +390,11 @@ async def load_from_arrow( exc_handler, self._execute_cursor_statement, load_sql, (tmp_name,) ) self._check_pending_exception(exc_handler) - finally: - await asyncio.to_thread(Path(tmp_name).unlink, True) + except BaseException as exc: + load_error = exc + await asyncio.to_thread(Path(tmp_name).unlink, True) + if load_error is not None: + raise load_error else: insert_sql = build_insert_statement(table, columns) prepared_records = ( @@ -392,7 +413,7 @@ async def load_from_arrow( self._attach_partition_telemetry(telemetry_payload, partitioner) return self._storage_job(telemetry_payload, telemetry) - def collect_rows(self, cursor: Any, fetched: "list[Any]") -> "tuple[list[Any], list[str], int]": + def collect_rows(self, cursor: Any, fetched: "Sequence[Any]") -> "tuple[list[Any], list[str], int]": """Collect aiomysql rows for the direct execution path.""" description = cursor.description or None row_plan = resolve_row_plan(description, AIOMYSQL_JSON_TYPE_CODES) @@ -420,15 +441,27 @@ async def _execute_cache_hit( return await super()._execute_cache_hit(sql, params, cached) async def _execute_cursor_statement(self, statement: str, parameters: Any = None) -> None: - async with self.with_cursor(self.connection) as cursor: + cursor_context = self.with_cursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: if parameters is None: await cursor.execute(statement) else: await cursor.execute(statement, parameters) + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) async def _executemany_cursor_statement(self, statement: str, parameters: Any) -> None: - async with self.with_cursor(self.connection) as cursor: + cursor_context = self.with_cursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: await cursor.executemany(statement, parameters) + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) def _connection_in_transaction(self) -> bool: """Check if connection is in transaction. @@ -439,6 +472,17 @@ def _connection_in_transaction(self) -> bool: return bool(self.connection.get_transaction_status()) +def _raised_by_mysql_client(error: BaseException) -> bool: + """Return whether the innermost traceback frame of ``error`` belongs to aiomysql or PyMySQL.""" + tb = error.__traceback__ + if tb is None: + return False + while tb.tb_next is not None: + tb = tb.tb_next + module_name = tb.tb_frame.f_globals.get("__name__") + return isinstance(module_name, str) and module_name.partition(".")[0] in _MYSQL_CLIENT_PACKAGES + + def _write_local_infile_tempfile(payload: bytes) -> str: """Write LOCAL INFILE payload to a temporary file.""" fd, tmp_name = tempfile.mkstemp(suffix=".tsv") diff --git a/sqlspec/adapters/aiosqlite/_typing.py b/sqlspec/adapters/aiosqlite/_typing.py index c3250e18f..8e9affb79 100644 --- a/sqlspec/adapters/aiosqlite/_typing.py +++ b/sqlspec/adapters/aiosqlite/_typing.py @@ -6,19 +6,19 @@ import contextlib import sqlite3 as aiosqlite_sqlite_module -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING import aiosqlite as aiosqlite_module +from sqlspec.driver._session import AsyncDriverSessionContext + _AiosqliteConnection = aiosqlite_module.Connection if TYPE_CHECKING: - from collections.abc import Callable from types import TracebackType from typing import TypeAlias from sqlspec.adapters.aiosqlite.driver import AiosqliteDriver - from sqlspec.core import StatementConfig AiosqliteConnection: TypeAlias = _AiosqliteConnection AiosqliteConnectionFactory: TypeAlias = type[aiosqlite_sqlite_module.Connection] @@ -60,76 +60,16 @@ async def __aexit__( await self.cursor.close() -class AiosqliteSessionContext: - """Async context manager for AIOSQLite sessions. - - This class is intentionally excluded from mypyc compilation to avoid ABI - boundary issues. It receives callables from uncompiled config classes and - instantiates compiled Driver objects, acting as a bridge between compiled - and uncompiled code. - - Uses callable-based connection management to decouple from config implementation. - """ - - __slots__ = ( - "_acquire_connection", - "_connection", - "_driver", - "_driver_features", - "_prepare_driver", - "_release_connection", - "_statement_config", - ) - - def __init__( - self, - acquire_connection: "Callable[[], Any]", - release_connection: "Callable[..., Any]", - statement_config: "StatementConfig", - driver_features: "dict[str, Any]", - prepare_driver: "Callable[[AiosqliteDriver], AiosqliteDriver]", - ) -> None: - self._acquire_connection = acquire_connection - self._release_connection = release_connection - self._statement_config = statement_config - self._driver_features = driver_features - self._prepare_driver = prepare_driver - self._connection: Any = None - self._driver: AiosqliteDriver | None = None - - async def __aenter__(self) -> "AiosqliteDriver": - from sqlspec.adapters.aiosqlite.driver import AiosqliteDriver +class AiosqliteSessionContext(AsyncDriverSessionContext): + """Async context manager for AIOSQLite sessions.""" - self._connection = await self._acquire_connection() - try: - self._driver = AiosqliteDriver( - connection=self._connection, - statement_config=self._statement_config, - driver_features=self._driver_features, - ) - return self._prepare_driver(self._driver) - except BaseException as exc: - await self.__aexit__(type(exc), exc, exc.__traceback__) - raise + __slots__ = () - async def __aexit__( - self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None" - ) -> "bool | None": - driver = self._driver - try: - if driver is not None: - await driver._close_open_streams(error=exc_type is not None) - except BaseException as close_exc: - if exc_type is None: - exc_type, exc_val, exc_tb = type(close_exc), close_exc, close_exc.__traceback__ - raise - finally: - try: - if self._connection is not None: - await self._release_connection(self._connection, exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) - self._connection = None - finally: - self._driver = None - if driver is not None: - driver._mark_closed() - return None + if TYPE_CHECKING: + + async def __aenter__(self) -> "AiosqliteDriver": ... + + def _resolve_driver_class(self) -> "type[AiosqliteDriver]": + from sqlspec.adapters.aiosqlite.driver import AiosqliteDriver + + return AiosqliteDriver diff --git a/sqlspec/adapters/aiosqlite/core.py b/sqlspec/adapters/aiosqlite/core.py index ca0e097c1..4e349345e 100644 --- a/sqlspec/adapters/aiosqlite/core.py +++ b/sqlspec/adapters/aiosqlite/core.py @@ -14,9 +14,10 @@ from sqlspec.adapters.aiosqlite._typing import aiosqlite_module as aiosqlite from sqlspec.adapters.aiosqlite._typing import aiosqlite_sqlite_module as sqlite3 -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult -from sqlspec.driver import rows_to_dicts +from sqlspec.driver._stream import rows_to_dicts from sqlspec.exceptions import ( CheckViolationError, DatabaseConnectionError, @@ -33,8 +34,9 @@ SQLParsingError, SQLSpecError, UniqueViolationError, + with_cause, ) -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import quote_identifier, split_qualified_identifier from sqlspec.utils.type_converters import build_decimal_converter, build_uuid_coercions, time_iso_convert from sqlspec.utils.type_guards import has_lastrowid, has_rowcount, has_sqlite_error, resolve_row_format @@ -45,6 +47,7 @@ from sqlspec.adapters.aiosqlite._typing import AiosqliteConnection from sqlspec.core.cache import CachedQuery from sqlspec.core.compiler import OperationType + from sqlspec.core.statement import StatementConfig _T = TypeVar("_T") @@ -177,6 +180,14 @@ async def end_transaction( "commit" in conn_dict or "rollback" in conn_dict or "execute" in conn_dict ) if not has_method_override: + if commit: + worker_queue = getattr(connection, "_tx", None) + if worker_queue is not None and worker_queue.empty(): + try: + if not raw_conn.in_transaction: + return + except (ValueError, sqlite3.ProgrammingError): + return await run_on_worker_thread( connection, _end_transaction_on_worker, raw_conn, commit=commit, supports_autocommit=supports_autocommit ) @@ -234,7 +245,7 @@ async def execute_cache_hit( params: Any, cached: "CachedQuery", statement_config: "StatementConfig", - rowid_target_cache: "dict[tuple[str | None, str], bool]", + rowid_target_cache: "dict[Any, bool]", ) -> "SQLResult": """Execute a cached statement on the aiosqlite worker thread and build its result. @@ -266,6 +277,7 @@ async def execute_cache_hit( operation_type, cached.processed_state.parsed_expression, rowid_target_cache, + cached.compiled_sql, ) return DMLResult(operation_type, affected_rows, dml_last_inserted_id) fetched_data, description, _affected_rows, last_inserted_id = await run_on_worker_thread( @@ -277,6 +289,7 @@ async def execute_cache_hit( operation_type, cached.processed_state.parsed_expression, rowid_target_cache, + cached.compiled_sql, ) except (aiosqlite.Error, sqlite3.Error) as exc: mapped_error = create_mapped_exception(exc) @@ -320,7 +333,8 @@ def resolve_lastrowid( operation_type: "OperationType", rowcount: int, expression: Any, - eligibility_cache: "dict[tuple[str | None, str], bool]", + eligibility_cache: "dict[Any, bool]", + compiled_sql: "str | None" = None, ) -> "int | None": """Resolve lastrowid only when compiled target metadata proves rowid eligibility. @@ -334,23 +348,45 @@ def resolve_lastrowid( operation_type: Compiled statement operation type. rowcount: Normalized affected-row count. expression: Compiled SQLGlot expression. - eligibility_cache: Driver-local target eligibility cache. + eligibility_cache: Driver-local eligibility cache, keyed by ``(schema, table)`` target and + by compiled SQL text. + compiled_sql: Optional compiled SQL text used to key eligibility lookups without AST walks. + Text that ends with a comment is resolved by target and never stored, so per-request + comments do not add entries. Returns: Integer lastrowid when the current target is a rowid table, otherwise None. """ if operation_type != "INSERT" or rowcount <= 0: return None + if compiled_sql is not None: + supports_rowid = eligibility_cache.get(compiled_sql) + if supports_rowid is None: + supports_rowid = _target_rowid_eligibility(connection, expression, eligibility_cache) + if supports_rowid is not None and not compiled_sql.endswith("*/"): + if len(eligibility_cache) >= _ROWID_ELIGIBILITY_CACHE_SIZE: + eligibility_cache.clear() + eligibility_cache[compiled_sql] = supports_rowid + if not supports_rowid: + return None + return normalize_lastrowid(cursor, operation_type, rowcount) + if not _target_rowid_eligibility(connection, expression, eligibility_cache): + return None + return normalize_lastrowid(cursor, operation_type, rowcount) + + +def _target_rowid_eligibility(connection: Any, expression: Any, eligibility_cache: "dict[Any, bool]") -> "bool | None": + """Return whether the INSERT target of ``expression`` is a rowid table, or None without a target.""" target = _resolve_insert_target(expression) if target is None: return None supports_rowid = eligibility_cache.get(target) if supports_rowid is None: supports_rowid = _target_supports_rowid(connection, target) + if len(eligibility_cache) >= _ROWID_ELIGIBILITY_CACHE_SIZE: + eligibility_cache.clear() eligibility_cache[target] = supports_rowid - if not supports_rowid: - return None - return normalize_lastrowid(cursor, operation_type, rowcount) + return supports_rowid def format_identifier(identifier: str) -> str: @@ -486,6 +522,7 @@ def normalize_execute_many_parameters(parameters: Any) -> Any: _INSERT_TARGET_CACHE_SIZE: Final[int] = 512 +_ROWID_ELIGIBILITY_CACHE_SIZE: Final[int] = 512 _INSERT_TARGET_CACHE: "dict[int, tuple[Any, tuple[str | None, str] | None]]" = {} _EXCLUDED_CONNECTION_KEYS: Final[frozenset[str]] = frozenset({ "query_only", @@ -741,7 +778,7 @@ def extension_pragma_statements(config: Any, extension_name: str) -> "tuple[str, msg = str(exc).replace( "driver_features['pragmas']", f"extension_config['{extension_name}']['pragma_overrides']" ) - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc return tuple(statements) @@ -821,7 +858,8 @@ def _execute_fetchall_with_metadata( parameters: Any, operation_type: "OperationType", expression: Any, - eligibility_cache: "dict[tuple[str | None, str], bool]", + eligibility_cache: "dict[Any, bool]", + cache_key: "str | None" = None, ) -> "tuple[list[Any], Any, int, int | None]": """Execute a query and return rows plus execution metadata on the worker thread.""" raw_connection = connection._conn @@ -831,7 +869,7 @@ def _execute_fetchall_with_metadata( rowcount = cursor.rowcount if isinstance(cursor.rowcount, int) and cursor.rowcount > 0 else 0 effective_rowcount = rowcount or (len(fetched_data) if operation_type == "INSERT" else 0) last_inserted_id = resolve_lastrowid( - raw_connection, cursor, operation_type, effective_rowcount, expression, eligibility_cache + raw_connection, cursor, operation_type, effective_rowcount, expression, eligibility_cache, cache_key ) return fetched_data, cursor.description, rowcount, last_inserted_id finally: @@ -845,7 +883,8 @@ def _execute_and_resolve_metadata( parameters: Any, operation_type: "OperationType", expression: Any, - eligibility_cache: "dict[tuple[str | None, str], bool]", + eligibility_cache: "dict[Any, bool]", + cache_key: "str | None" = None, ) -> "tuple[int, int | None]": """Execute a statement and resolve rowcount and lastrowid on the worker thread.""" raw_connection = connection._conn @@ -855,7 +894,7 @@ def _execute_and_resolve_metadata( cursor.rowcount if has_rowcount(cursor) and isinstance(cursor.rowcount, int) and cursor.rowcount > 0 else 0 ) return rowcount, resolve_lastrowid( - raw_connection, cursor, operation_type, rowcount, expression, eligibility_cache + raw_connection, cursor, operation_type, rowcount, expression, eligibility_cache, cache_key ) finally: with contextlib.suppress(Exception): diff --git a/sqlspec/adapters/aiosqlite/driver.py b/sqlspec/adapters/aiosqlite/driver.py index 837ac75b0..7eb877b52 100644 --- a/sqlspec/adapters/aiosqlite/driver.py +++ b/sqlspec/adapters/aiosqlite/driver.py @@ -4,7 +4,7 @@ import contextlib import inspect import secrets -from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, cast from mypy_extensions import mypyc_attr @@ -30,21 +30,29 @@ run_on_worker_thread, ) from sqlspec.adapters.aiosqlite.data_dictionary import AiosqliteDataDictionary -from sqlspec.core import ArrowResult, get_cache_config, register_driver_profile -from sqlspec.core.result import DMLResult -from sqlspec.driver import AsyncDriverAdapterBase, AsyncRowStream, BaseAsyncExceptionHandler +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.result._base import ArrowResult, DMLResult +from sqlspec.driver._async import AsyncDriverAdapterBase from sqlspec.driver._common import cache_hit_hooks_unchanged -from sqlspec.exceptions import DeadlockError +from sqlspec.driver._exception_handler import BaseAsyncExceptionHandler +from sqlspec.driver._stream import AsyncRowStream +from sqlspec.exceptions import DeadlockError, with_cause +from sqlspec.utils.sync_tools import exit_async_context from sqlspec.utils.type_guards import resolve_row_format if TYPE_CHECKING: from collections.abc import Mapping, Sequence + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.aiosqlite._typing import AiosqliteConnection - from sqlspec.core import SQL, CachedQuery, SQLResult, StatementConfig + from sqlspec.core.cache import CachedQuery from sqlspec.core.compiler import OperationType - from sqlspec.driver import ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.core.result._base import SQLResult + from sqlspec.core.statement import SQL, StatementConfig + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry __all__ = ( "AiosqliteCursor", @@ -57,6 +65,16 @@ _execute_and_resolve_metadata = execute_and_resolve_metadata _execute_fetchall_with_metadata = execute_fetchall_with_metadata random = secrets.SystemRandom() +_NO_CURSOR: Any = None +_DISPATCH_HOOKS: Final[tuple[str, ...]] = ( + "with_cursor", + "dispatch_special_handling", + "dispatch_execute", + "dispatch_execute_many", + "dispatch_execute_script", + "create_execution_result", + "build_statement_result", +) @mypyc_attr(allow_interpreted_subclasses=True) @@ -81,8 +99,8 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba class AiosqliteDriver(AsyncDriverAdapterBase): """AIOSQLite driver for async SQLite database operations.""" - __slots__ = ("_data_dictionary", "_rowid_target_cache") - dialect = "sqlite" + __slots__ = ("_data_dictionary", "_dispatches_without_cursor", "_rowid_target_cache") + dialect: "DialectType | None" = "sqlite" export_integers_may_widen: ClassVar[bool] = True def __init__( @@ -98,9 +116,10 @@ def __init__( super().__init__(connection=connection, statement_config=statement_config, driver_features=driver_features) self._data_dictionary: AiosqliteDataDictionary | None = None - self._rowid_target_cache: dict[tuple[str | None, str], bool] = {} + self._rowid_target_cache: dict[Any, bool] = {} self._cache_hit_executor = execute_cache_hit if cache_hit_hooks_unchanged(type(self), AiosqliteDriver) else None self._cache_hit_state = self._rowid_target_cache + self._dispatches_without_cursor = _dispatch_hooks_unchanged(type(self)) @property def data_dictionary(self) -> "AiosqliteDataDictionary": @@ -129,6 +148,7 @@ async def dispatch_execute(self, cursor: "AiosqliteRawCursor", statement: "SQL") statement.operation_type, statement._get_shared_expression(), self._rowid_target_cache, + sql, ) data, column_names, row_count = collect_rows(fetched_data, description) row_format = resolve_row_format(data) @@ -152,6 +172,7 @@ async def dispatch_execute(self, cursor: "AiosqliteRawCursor", statement: "SQL") statement.operation_type, statement._get_shared_expression(), self._rowid_target_cache, + sql, ) return self.create_execution_result(cursor, rowcount_override=affected_rows, last_inserted_id=last_inserted_id) @@ -187,8 +208,12 @@ async def begin(self, mode: "Literal['DEFERRED', 'IMMEDIATE', 'EXCLUSIVE'] | Non if transaction_mode is not None and transaction_mode not in {"DEFERRED", "IMMEDIATE", "EXCLUSIVE"}: msg = "Transaction mode must be DEFERRED, IMMEDIATE, or EXCLUSIVE" raise ValueError(msg) + begin_error: aiosqlite.Error | sqlite3.Error | None = None try: - if not await _check_in_transaction(self.connection): + raw_conn = getattr(self.connection, "_conn", None) + if _executes_on_worker_thread(self.connection, raw_conn): + await run_on_worker_thread(self.connection, _begin_on_worker, raw_conn, stmt) + elif not await _check_in_transaction(self.connection): cursor = await self.connection.execute(stmt) if cursor is not None: with contextlib.suppress(Exception): @@ -196,14 +221,16 @@ async def begin(self, mode: "Literal['DEFERRED', 'IMMEDIATE', 'EXCLUSIVE'] | Non if inspect.isawaitable(close_result): await close_result except (aiosqlite.Error, sqlite3.Error) as e: - await _retry_begin_with_backoff(self.connection, e, statement=stmt) + begin_error = e + if begin_error is not None: + await _retry_begin_with_backoff(self.connection, begin_error, statement=stmt) async def commit(self) -> None: """Commit the current transaction.""" try: await end_transaction(self.connection, commit=True) except (aiosqlite.Error, sqlite3.Error) as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e async def rollback(self) -> None: """Rollback the current transaction.""" @@ -212,7 +239,7 @@ async def rollback(self) -> None: self._rowid_target_cache.clear() except (aiosqlite.Error, sqlite3.Error) as e: self._rowid_target_cache.clear() - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e def with_cursor(self, connection: "AiosqliteConnection") -> "AiosqliteCursor": """Create async context manager for AIOSQLite cursor.""" @@ -300,10 +327,14 @@ async def load_from_arrow( needs_prep = self._arrow_rows_need_preparation(arrow_table) owns_transaction = not await _check_in_transaction(self.connection) + load_error: BaseException | None = None try: if owns_transaction: await self.begin() - async with self.with_cursor(self.connection) as cursor: + cursor_context = self.with_cursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: if overwrite: statement = f"DELETE FROM {format_identifier(table)}" await cursor.execute(statement) @@ -317,14 +348,19 @@ async def load_from_arrow( else records ) await cursor.executemany(insert_sql, cast("Any", prepared_records)) + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) if owns_transaction: await self.commit() except BaseException as exc: + load_error = exc + if load_error is not None: if owns_transaction: await self.rollback() - if isinstance(exc, (aiosqlite.Error, sqlite3.Error)): - raise create_mapped_exception(exc) from exc - raise + if isinstance(load_error, (aiosqlite.Error, sqlite3.Error)): + raise with_cause(create_mapped_exception(load_error), load_error) from load_error + raise load_error telemetry_payload = self._ingest_telemetry(arrow_table) telemetry_payload["destination"] = table @@ -356,6 +392,7 @@ async def _execute_cache_hit_with_cursor( cached.operation_type, cached.processed_state.parsed_expression, self._rowid_target_cache, + cached.compiled_sql, ) data, column_names, row_count = collect_rows(fetched_data, description) return self.build_cached_select_result( @@ -378,9 +415,40 @@ async def _execute_cache_hit_with_cursor( cached.operation_type, cached.processed_state.parsed_expression, self._rowid_target_cache, + cached.compiled_sql, ) return DMLResult(cached.operation_type, affected_rows, last_inserted_id) + async def _dispatch_statement_with_cursor( + self, connection: Any, statement: "SQL", *, has_execution_parameters: bool + ) -> "SQLResult": + """Dispatch a statement, opening a cursor only when a dispatch hook is overridden. + + The driver's own dispatch methods run each statement on the worker thread with a + cursor they open and close there, so no cursor is created on the event-loop side. + """ + if not self._dispatches_without_cursor: + return await super()._dispatch_statement_with_cursor( + connection, statement, has_execution_parameters=has_execution_parameters + ) + if statement.is_script: + execution_result = await self.dispatch_execute_script(_NO_CURSOR, statement) + elif statement.is_many: + if has_execution_parameters: + execution_result = await self.dispatch_execute_many(_NO_CURSOR, statement) + else: + execution_result = self.create_execution_result(_NO_CURSOR, rowcount_override=0, is_many_result=True) + else: + execution_result = await self.dispatch_execute(_NO_CURSOR, statement) + return self.build_statement_result(statement, execution_result) + + async def _execute_cached_statement_with_cursor(self, statement: "SQL") -> "SQLResult": + """Execute a prepared statement, opening a cursor only when a dispatch hook is overridden.""" + if not self._dispatches_without_cursor: + return await super()._execute_cached_statement_with_cursor(statement) + execution_result = await self.dispatch_execute(_NO_CURSOR, statement) + return self.build_statement_result(statement, execution_result) + def _invalidate_rowid_target_cache(self, operation_type: "OperationType") -> None: if operation_type not in ROWID_SAFE_OPERATIONS: self._rowid_target_cache.clear() @@ -398,10 +466,37 @@ async def _connection_in_transaction_async(self) -> bool: return await _check_in_transaction(self.connection) +def _dispatch_hooks_unchanged(driver_type: type) -> bool: + """Return whether ``driver_type`` dispatches statements exactly as ``AiosqliteDriver`` does.""" + if driver_type is AiosqliteDriver: + return True + return all(getattr(driver_type, name, None) is getattr(AiosqliteDriver, name, None) for name in _DISPATCH_HOOKS) + + def _read_in_transaction_on_worker(raw_conn: Any) -> bool: return bool(getattr(raw_conn, "in_transaction", False)) +def _begin_on_worker(raw_conn: Any, statement: str) -> None: + """Start a transaction on the worker thread unless the connection already holds one.""" + if _read_in_transaction_on_worker(raw_conn): + return + cursor = raw_conn.execute(statement) + with contextlib.suppress(Exception): + cursor.close() + + +def _executes_on_worker_thread(connection: Any, raw_conn: Any) -> bool: + """Return whether statements can run directly on the connection's worker thread. + + A connection whose ``execute`` is replaced on the instance keeps running through that method. + """ + if not isinstance(raw_conn, sqlite3.Connection) or not callable(getattr(connection, "_execute", None)): + return False + conn_dict = getattr(connection, "__dict__", None) + return not (isinstance(conn_dict, dict) and "execute" in conn_dict) + + async def _check_in_transaction(connection: "AiosqliteConnection") -> bool: raw_conn = getattr(connection, "_conn", None) if isinstance(raw_conn, sqlite3.Connection) and callable(getattr(connection, "_execute", None)): @@ -436,7 +531,7 @@ async def _retry_begin_with_backoff( SQLSpecError: If a non-transient error occurs or every retry attempt fails. """ if not _is_busy_or_locked_error(initial_error): - raise create_mapped_exception(initial_error) from initial_error + raise with_cause(create_mapped_exception(initial_error), initial_error) from initial_error for attempt in range(max_retries): delay = 0.01 * (2**attempt) + random.uniform(0, 0.01) @@ -450,13 +545,15 @@ async def _retry_begin_with_backoff( await close_result except (aiosqlite.Error, sqlite3.Error) as exc: if not _is_busy_or_locked_error(exc): - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc if attempt == max_retries - 1: break else: return msg = f"Failed to begin transaction after retries: {initial_error}" - raise DeadlockError(msg) from initial_error + error = DeadlockError(msg) + error.__cause__ = initial_error + raise error from initial_error register_driver_profile("aiosqlite", driver_profile) diff --git a/sqlspec/adapters/aiosqlite/pool.py b/sqlspec/adapters/aiosqlite/pool.py index 1a97a0387..97ee92830 100644 --- a/sqlspec/adapters/aiosqlite/pool.py +++ b/sqlspec/adapters/aiosqlite/pool.py @@ -19,7 +19,7 @@ is_read_only_sqlite_config, run_on_worker_thread, ) -from sqlspec.exceptions import ConnectionTimeoutError, ImproperConfigurationError, SQLSpecError +from sqlspec.exceptions import ConnectionTimeoutError, ImproperConfigurationError, SQLSpecError, with_cause from sqlspec.utils.logging import POOL_LOGGER_NAME, get_logger, log_with_context from sqlspec.utils.sync_tools import exit_async_context from sqlspec.utils.uuids import uuid4 @@ -352,7 +352,7 @@ async def new_connection(self) -> "AiosqliteConnection": sqlite3.SQLITE_CANTOPEN, 26, ): - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc raise pragma_lines: list[str] = [] @@ -446,7 +446,9 @@ async def acquire(self) -> AiosqlitePoolConnection: raise except TimeoutError as e: msg = f"Connection acquisition timed out after {self._connect_timeout}s" - raise AiosqliteConnectTimeoutError(msg) from e + error = AiosqliteConnectTimeoutError(msg) + error.__cause__ = e + raise error from e return connection @@ -480,7 +482,8 @@ async def release(self, connection: AiosqlitePoolConnection) -> None: reset_error: BaseException | None = None try: await end_transaction(raw_conn, commit=False) - await self._reset_connection(raw_conn) + if self._reset_session_on_release: + await self._reset_connection(raw_conn) connection.idle_since = time.time() self._queue.put_nowait(connection) except BaseException as e: @@ -584,24 +587,25 @@ async def _release_checked(self, connection: AiosqlitePoolConnection) -> None: connection.mark_unhealthy() await self._retire_connection(connection, reason="unhealthy") return - reset_error: Exception | None = None - try: - await self._reset_connection(connection.connection) - except Exception as e: - reset_error = e - if reset_error is not None: - log_with_context( - logger, - logging.DEBUG, - "pool.connection.reset.error", - adapter=_ADAPTER_NAME, - pool_id=self._pool_id, - connection_id=connection.id, - error_type=type(reset_error).__name__, - ) - connection.mark_unhealthy() - await self._retire_connection(connection, reason="reset_failed") - return + if self._reset_session_on_release: + reset_error: Exception | None = None + try: + await self._reset_connection(connection.connection) + except Exception as e: + reset_error = e + if reset_error is not None: + log_with_context( + logger, + logging.DEBUG, + "pool.connection.reset.error", + adapter=_ADAPTER_NAME, + pool_id=self._pool_id, + connection_id=connection.id, + error_type=type(reset_error).__name__, + ) + connection.mark_unhealthy() + await self._retire_connection(connection, reason="reset_failed") + return connection.idle_since = time.time() self._queue.put_nowait(connection) diff --git a/sqlspec/adapters/arrow_odbc/core.py b/sqlspec/adapters/arrow_odbc/core.py index 893f52df3..1d68b83ad 100644 --- a/sqlspec/adapters/arrow_odbc/core.py +++ b/sqlspec/adapters/arrow_odbc/core.py @@ -7,7 +7,8 @@ from sqlglot import exp -from sqlspec.core import DriverParameterProfile, ParameterStyle, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.exceptions import ( CheckViolationError, DatabaseConnectionError, @@ -28,13 +29,13 @@ ) from sqlspec.typing import ArrowException from sqlspec.utils.config_tools import parse_odbc_connection_string -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.type_converters import build_uuid_coercions if TYPE_CHECKING: - from collections.abc import Callable, Iterator, Mapping + from collections.abc import Callable, Generator, Mapping - from sqlspec.core import StatementConfig + from sqlspec.core.statement import StatementConfig __all__ = ( "DB2_INDEX_EXISTS_SQL", @@ -492,7 +493,7 @@ def strip_bracket_quotes(part: str) -> str: @contextlib.contextmanager -def atomic_store_write(driver: Any) -> "Iterator[None]": +def atomic_store_write(driver: Any) -> "Generator[None, None, None]": """Wrap multi-statement store mutations in an explicit transaction when supported. Yields: diff --git a/sqlspec/adapters/arrow_odbc/driver.py b/sqlspec/adapters/arrow_odbc/driver.py index 3695be8b2..c49971f00 100644 --- a/sqlspec/adapters/arrow_odbc/driver.py +++ b/sqlspec/adapters/arrow_odbc/driver.py @@ -8,6 +8,7 @@ from itertools import chain from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, cast +from mypy_extensions import mypyc_attr from sqlglot import exp from sqlspec.adapters.arrow_odbc._typing import ArrowOdbcConnection, ArrowOdbcCursor, ArrowOdbcError, ArrowOdbcRawCursor @@ -19,31 +20,35 @@ resolve_dialect_from_dbms_name, ) from sqlspec.adapters.arrow_odbc.data_dictionary import ArrowOdbcDataDictionary -from sqlspec.core import ( - SQL, - build_arrow_result_from_reader, - build_arrow_result_from_table, - get_cache_config, - register_driver_profile, -) +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile from sqlspec.core.parameters._validator import ParameterValidator -from sqlspec.driver import BaseSyncExceptionHandler, SyncDriverAdapterBase, SyncRowStream, validate_savepoint_name +from sqlspec.core.result._base import build_arrow_result_from_reader, build_arrow_result_from_table +from sqlspec.driver._common import validate_savepoint_name +from sqlspec.driver._exception_handler import BaseSyncExceptionHandler +from sqlspec.driver._stream import SyncRowStream +from sqlspec.driver._sync import SyncDriverAdapterBase from sqlspec.exceptions import DataError, ImproperConfigurationError, SQLSpecError +from sqlspec.storage._utils import import_pyarrow from sqlspec.typing import ArrowException from sqlspec.utils.arrow_helpers import arrow_reader_with_deferred_close from sqlspec.utils.module_loader import ensure_pyarrow from sqlspec.utils.text import quote_identifier, split_qualified_identifier if TYPE_CHECKING: - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, ParameterProfile, Statement, StatementConfig, StatementFilter - from sqlspec.driver import ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.filters import StatementFilter + from sqlspec.core.parameters._types import ParameterProfile + from sqlspec.core.result._base import ArrowResult + from sqlspec.core.statement import SQL, Statement, StatementConfig + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry from sqlspec.typing import ArrowRecordBatch, ArrowRecordBatchReader, ArrowReturnFormat, StatementParameters __all__ = ("ArrowOdbcCursor", "ArrowOdbcDriver", "ArrowOdbcExceptionHandler", "resolve_dialect_from_dbms_name") +@mypyc_attr(allow_interpreted_subclasses=True) class ArrowOdbcExceptionHandler(BaseSyncExceptionHandler): """Sync context manager handling arrow-odbc exceptions.""" @@ -125,6 +130,7 @@ def close(self, error: bool = False) -> None: close() +@mypyc_attr(allow_interpreted_subclasses=True) class ArrowOdbcDriver(SyncDriverAdapterBase): """Sync driver for generic ODBC connections with Arrow-native transfer.""" @@ -315,7 +321,9 @@ def begin(self) -> None: except ArrowOdbcError as exc: mapped = create_mapped_exception(exc) msg = f"Failed to begin transaction: {mapped}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc self._transaction_active = True def commit(self) -> None: @@ -329,7 +337,9 @@ def commit(self) -> None: except ArrowOdbcError as exc: mapped = create_mapped_exception(exc) msg = f"Failed to commit transaction: {mapped}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc self._transaction_active = False def rollback(self) -> None: @@ -343,7 +353,9 @@ def rollback(self) -> None: except ArrowOdbcError as exc: mapped = create_mapped_exception(exc) msg = f"Failed to rollback transaction: {mapped}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc finally: self._transaction_active = False @@ -576,21 +588,6 @@ def load_from_arrow( self._attach_partition_telemetry(telemetry_payload, partitioner) return self._storage_job(telemetry_payload, telemetry) - def load_from_storage( - self, - table: str, - source: "StorageDestination", - *, - file_format: "StorageFormat", - partitioner: "dict[str, object] | None" = None, - overwrite: bool = False, - **kwargs: Any, - ) -> "StorageBridgeJob": - """Load staged artifacts from storage into a table via arrow-odbc bulk insert.""" - return super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs - ) - def _connection_in_transaction(self) -> bool: """Return whether SQLSpec holds an explicit transaction on the connection. @@ -681,8 +678,7 @@ def _normalize_reader(self, reader: "ArrowRecordBatchReader") -> "ArrowRecordBat renamed = schema for index, name in enumerate(names): renamed = renamed.set(index, renamed.field(index).with_name(name)) - ensure_pyarrow() - import pyarrow as pa + pa = import_pyarrow() return arrow_reader_with_deferred_close( pa.RecordBatchReader.from_batches( @@ -836,8 +832,7 @@ def _odbc_parameters(parameters: Any, *, naive_utc_datetimes: bool = False) -> " def _reader_to_table(reader: Any) -> Any: - ensure_pyarrow() - import pyarrow as pa + pa = import_pyarrow() if isinstance(reader, pa.Table): return reader @@ -855,31 +850,34 @@ def _reader_to_table(reader: Any) -> Any: return pa.Table.from_batches(batches) -def _to_pyarrow_reader(reader: object) -> "ArrowRecordBatchReader": - ensure_pyarrow() - import pyarrow as pa +def _to_pyarrow_reader(reader: Any) -> "ArrowRecordBatchReader": + pa = import_pyarrow() if isinstance(reader, pa.RecordBatchReader): - return reader + return cast("ArrowRecordBatchReader", reader) if isinstance(reader, pa.Table): - return pa.RecordBatchReader.from_batches(reader.schema, reader.to_batches()) + return cast("ArrowRecordBatchReader", pa.RecordBatchReader.from_batches(reader.schema, reader.to_batches())) into_reader = getattr(reader, "into_pyarrow_record_batch_reader", None) if callable(into_reader): return cast("ArrowRecordBatchReader", into_reader()) schema = getattr(reader, "schema", None) if isinstance(schema, pa.Schema): - return pa.RecordBatchReader.from_batches(schema, cast("Iterable[ArrowRecordBatch]", reader)) + return cast( + "ArrowRecordBatchReader", + pa.RecordBatchReader.from_batches(schema, cast("Iterable[ArrowRecordBatch]", reader)), + ) iterator = iter(cast("Iterable[ArrowRecordBatch]", reader)) try: first_batch = next(iterator) except StopIteration: - return pa.RecordBatchReader.from_batches(pa.schema([]), []) - return pa.RecordBatchReader.from_batches(first_batch.schema, chain((first_batch,), iterator)) + return cast("ArrowRecordBatchReader", pa.RecordBatchReader.from_batches(pa.schema([]), [])) + return cast( + "ArrowRecordBatchReader", pa.RecordBatchReader.from_batches(first_batch.schema, chain((first_batch,), iterator)) + ) def _table_to_reader(table: Any, chunk_size: int) -> Any: - ensure_pyarrow() - import pyarrow as pa + pa = import_pyarrow() return pa.RecordBatchReader.from_batches(table.schema, table.to_batches(max_chunksize=chunk_size)) diff --git a/sqlspec/adapters/arrow_odbc/litestar/store.py b/sqlspec/adapters/arrow_odbc/litestar/store.py index 4026d9143..aba06dfc6 100644 --- a/sqlspec/adapters/arrow_odbc/litestar/store.py +++ b/sqlspec/adapters/arrow_odbc/litestar/store.py @@ -123,7 +123,7 @@ def _get(self, key: str, renew_for: "int | timedelta | None" = None) -> "bytes | return None expires_at = _normalize_utc(_row_value(row, "expires_at", 1)) - if expires_at is not None and expires_at < datetime.now(UTC): + if expires_at is not None and expires_at <= datetime.now(UTC): driver.execute(f"DELETE FROM {self._table_name} WHERE session_id = ?", (key,)) if store_session.owned: driver.commit() diff --git a/sqlspec/adapters/asyncmy/_typing.py b/sqlspec/adapters/asyncmy/_typing.py index 1c4f81617..1e17e03ba 100644 --- a/sqlspec/adapters/asyncmy/_typing.py +++ b/sqlspec/adapters/asyncmy/_typing.py @@ -22,12 +22,14 @@ from asyncmy.pool import Pool as _AsyncmyPool from asyncmy.protocol import LoadLocalPacketWrapper as _LoadLocalPacketWrapper +from sqlspec.driver._session import AsyncDriverSessionContext from sqlspec.exceptions import SQLSpecError if TYPE_CHECKING: - from collections.abc import Iterator from typing import Protocol, TypeAlias + from sqlspec.adapters.asyncmy.driver import AsyncmyDriver + class AsyncmyMySQLError(Exception): errno: int @@ -64,6 +66,7 @@ class AsyncmyFieldTypeProtocol(Protocol): AsyncmyModule: TypeAlias = AsyncmyModuleProtocol AsyncmyPool: TypeAlias = _AsyncmyPool AsyncmyRawCursor: TypeAlias = _AsyncmyCursor + from collections.abc import Generator else: AsyncmyConnection = Connection AsyncmyDictCursor = _AsyncmyDictCursor @@ -88,6 +91,7 @@ class AsyncmyFieldTypeProtocol(Protocol): "AsyncmyProgrammingError", "AsyncmyRawCursor", "AsyncmySSCursor", + "AsyncmySessionContext", "asyncmy_local_infile", ) @@ -126,7 +130,7 @@ async def __aexit__(self, *_: object) -> None: @contextlib.contextmanager -def asyncmy_local_infile(connection: "AsyncmyConnection", filename: str) -> "Iterator[None]": +def asyncmy_local_infile(connection: "AsyncmyConnection", filename: str) -> "Generator[None, None, None]": """Scope the asyncmy 0.2.13/0.2.14 filename handoff fix to one native load. Args: @@ -194,3 +198,18 @@ async def _read_load_local_packet(self, first_packet: Any) -> None: raise SQLSpecError(msg) raw_self: Any = self raw_self._read_ok_packet(packet) + + +class AsyncmySessionContext(AsyncDriverSessionContext): + """Async context manager for AsyncMy sessions.""" + + __slots__ = () + + if TYPE_CHECKING: + + async def __aenter__(self) -> "AsyncmyDriver": ... + + def _resolve_driver_class(self) -> "type[AsyncmyDriver]": + from sqlspec.adapters.asyncmy.driver import AsyncmyDriver + + return AsyncmyDriver diff --git a/sqlspec/adapters/asyncmy/adk/store.py b/sqlspec/adapters/asyncmy/adk/store.py index 51cd4d53b..d08483621 100644 --- a/sqlspec/adapters/asyncmy/adk/store.py +++ b/sqlspec/adapters/asyncmy/adk/store.py @@ -89,6 +89,7 @@ async def create_tables(self) -> None: await driver.execute_script(await self._app_states_table_ddl()) await driver.execute_script(await self._user_states_table_ddl()) await driver.execute_script(await self._metadata_table_ddl()) + await self.reconcile_schema(assume_existing=True) async def reconcile_companion_objects_async(self, driver: Any) -> None: """Reconcile legacy column collations and missing session/event secondary indexes.""" @@ -769,6 +770,7 @@ async def create_tables(self) -> None: async with self._config.provide_session() as driver: await driver.execute_script(await self._memory_table_ddl()) + await self.reconcile_schema(assume_existing=True) async def reconcile_companion_objects_async(self, driver: Any) -> None: """Reconcile legacy memory identity collations and missing ordinary/FULLTEXT indexes.""" diff --git a/sqlspec/adapters/asyncmy/config.py b/sqlspec/adapters/asyncmy/config.py index ef8b85eff..9a10ee8f4 100644 --- a/sqlspec/adapters/asyncmy/config.py +++ b/sqlspec/adapters/asyncmy/config.py @@ -15,6 +15,7 @@ AsyncmyMySQLError, AsyncmyPool, AsyncmyRawCursor, + AsyncmySessionContext, ) from sqlspec.adapters.asyncmy.core import ( apply_driver_features, @@ -311,81 +312,11 @@ async def __aexit__( return exit_result -class AsyncmySessionContext: - """Async context manager for AsyncMy sessions. - - Acquires a connection from the config, builds the driver, and releases the - connection when the session ends. - """ - - __slots__ = ( - "_acquire_connection", - "_connection", - "_driver", - "_driver_features", - "_prepare_driver", - "_release_connection", - "_statement_config", - ) - - def __init__( - self, - acquire_connection: "Callable[[], Awaitable[AsyncmyConnection]]", - release_connection: "Callable[..., Any]", - statement_config: "StatementConfig", - driver_features: "dict[str, Any]", - prepare_driver: "Callable[[AsyncmyDriver], AsyncmyDriver]", - ) -> None: - self._acquire_connection = acquire_connection - self._release_connection = release_connection - self._statement_config = statement_config - self._driver_features = driver_features - self._prepare_driver = prepare_driver - self._connection: AsyncmyConnection | None = None - self._driver: AsyncmyDriver | None = None - - async def __aenter__(self) -> "AsyncmyDriver": - self._connection = await self._acquire_connection() - try: - self._driver = AsyncmyDriver( - connection=self._connection, - statement_config=self._statement_config, - driver_features=self._driver_features, - ) - return self._prepare_driver(self._driver) - except BaseException as exc: - with contextlib.suppress(Exception): - await self.__aexit__(type(exc), exc, exc.__traceback__) - raise - - async def __aexit__( - self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None" - ) -> "bool | None": - driver = self._driver - try: - if driver is not None: - await driver._close_open_streams(error=exc_type is not None) - except BaseException as close_exc: - if exc_type is None: - exc_type, exc_val, exc_tb = type(close_exc), close_exc, close_exc.__traceback__ - raise - finally: - try: - if self._connection is not None: - await self._release_connection(self._connection, exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) - self._connection = None - finally: - self._driver = None - if driver is not None: - driver._mark_closed() - return None - - @mypyc_attr(native_class=False) class AsyncmyConfig(AsyncDatabaseConfig[AsyncmyConnection, "AsyncmyPool", AsyncmyDriver]): """Configuration for Asyncmy database connections.""" - __slots__ = ("_initialized_connections", "_read_only", "_user_connection_hook") + __slots__ = ("_allow_multi_statements", "_initialized_connections", "_read_only", "_user_connection_hook") driver_type: ClassVar[type[AsyncmyDriver]] = AsyncmyDriver connection_type: "ClassVar[type[Any]]" = cast("type[Any]", AsyncmyConnection) @@ -440,6 +371,10 @@ def __init__( self._read_only: bool = ( bool(kwargs.get("is_replica", False)) if replica_read_only is None else bool(replica_read_only) ) + raw_client_flag = connection_config.get("client_flag") + self._allow_multi_statements: bool = bool( + isinstance(raw_client_flag, int) and (raw_client_flag & _CLIENT_MULTI_STATEMENTS) + ) self._initialized_connections: WeakSet[Any] = WeakSet() features_dict.setdefault("enable_local_infile_bulk_load", connection_config["local_infile"]) @@ -572,13 +507,23 @@ async def _create_pool(self) -> "AsyncmyPool": raise async def _ensure_connection(self, connection: "AsyncmyConnection") -> None: - """Ensure read-only session state and connection callback run once per physical connection. + """Disable implicit multi-statements and apply read-only / create hooks once per connection. - Uses WeakSet tracking to ensure initialization runs once per physical connection. + Upstream ``asyncmy`` unconditionally ORs ``CLIENT.MULTI_STATEMENTS`` during the initial + handshake without an opt-out keyword parameter. Unless ``CLIENT.MULTI_STATEMENTS`` is + explicitly requested via ``connection_config["client_flag"]``, SQLSpec sends + ``COM_SET_OPTION(MYSQL_OPTION_MULTI_STATEMENTS_OFF)`` once per physical connection and + again after ``COM_RESET_CONNECTION`` so single-statement queries cannot smuggle stacked SQL. """ - if not self._read_only and self._user_connection_hook is None: + if self._allow_multi_statements and not self._read_only and self._user_connection_hook is None: return if connection not in self._initialized_connections: + if not self._allow_multi_statements: + try: + await _disable_multi_statements(connection) + except Exception as exc: + _raise_mapped_connection_error(exc) + raise if self._read_only: try: async with AsyncmyCursor(connection) as cursor: @@ -598,6 +543,9 @@ async def _close_pool(self) -> None: self.connection_instance = None +_COM_SET_OPTION: Final[int] = 0x1B +_MYSQL_OPTION_MULTI_STATEMENTS_OFF: Final[bytes] = b"\x01\x00" +_CLIENT_MULTI_STATEMENTS: Final[int] = 1 << 16 _COM_RESET_CONNECTION: Final[int] = 0x1F _FALLBACK_ASYNCMY_CONNECT_PARAMS: Final[frozenset[str]] = frozenset({ "host", @@ -691,6 +639,20 @@ def _is_autocommit_enabled(connection: Any) -> bool: return bool(getattr(connection, "autocommit_mode", False)) +async def _disable_multi_statements(connection: Any) -> None: + """Send ``COM_SET_OPTION(MYSQL_OPTION_MULTI_STATEMENTS_OFF)`` when supported by the connection.""" + execute_command = getattr(connection, "_execute_command", None) + read_packet = getattr(connection, "read_packet", None) + if not callable(execute_command) or not callable(read_packet): + return + result = execute_command(_COM_SET_OPTION, _MYSQL_OPTION_MULTI_STATEMENTS_OFF) + if inspect.isawaitable(result): + await result + packet = read_packet() + if inspect.isawaitable(packet): + await packet + + async def _reset_session(config: Any, connection: Any) -> None: """Reset server session state on a released connection, closing it when the reset cannot complete. diff --git a/sqlspec/adapters/asyncmy/core.py b/sqlspec/adapters/asyncmy/core.py index a5eeb4d3e..d710429f6 100644 --- a/sqlspec/adapters/asyncmy/core.py +++ b/sqlspec/adapters/asyncmy/core.py @@ -7,9 +7,10 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast from sqlspec.adapters.asyncmy._typing import AsyncmyError, AsyncmySSCursor -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult -from sqlspec.driver import rows_to_dicts +from sqlspec.driver._stream import rows_to_dicts from sqlspec.exceptions import ( CheckViolationError, ConnectionTimeoutError, @@ -31,14 +32,15 @@ ) from sqlspec.utils.config_tools import parse_mysql_dsn from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import quote_backtick_identifier, split_qualified_identifier from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_lastrowid, has_rowcount, has_sqlstate, has_type_code, resolve_row_format if TYPE_CHECKING: from sqlspec.core.cache import CachedQuery - from sqlspec.core.parameters import ParameterValidator + from sqlspec.core.parameters._validator import ParameterValidator + from sqlspec.core.statement import StatementConfig __all__ = ( "AsyncmyStreamSource", diff --git a/sqlspec/adapters/asyncmy/driver.py b/sqlspec/adapters/asyncmy/driver.py index 5c26b8ee1..4bbcb695b 100644 --- a/sqlspec/adapters/asyncmy/driver.py +++ b/sqlspec/adapters/asyncmy/driver.py @@ -10,6 +10,8 @@ from collections.abc import Sized from typing import TYPE_CHECKING, Any, Final, Literal, cast +from mypy_extensions import mypyc_attr + from sqlspec.adapters.asyncmy._typing import ( ASYNCMY_INSERT_VALUES_PATTERN, AsyncmyCursor, @@ -41,20 +43,29 @@ resolve_rowcount, ) from sqlspec.adapters.asyncmy.data_dictionary import AsyncmyDataDictionary -from sqlspec.core import ArrowResult, get_cache_config, register_driver_profile -from sqlspec.driver import AsyncDriverAdapterBase, AsyncRowStream, BaseAsyncExceptionHandler +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.driver._async import AsyncDriverAdapterBase from sqlspec.driver._common import cache_hit_hooks_unchanged +from sqlspec.driver._exception_handler import BaseAsyncExceptionHandler +from sqlspec.driver._stream import AsyncRowStream +from sqlspec.exceptions import with_cause from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json +from sqlspec.utils.serializers._json import from_json +from sqlspec.utils.sync_tools import exit_async_context from sqlspec.utils.type_guards import supports_json_type if TYPE_CHECKING: from collections.abc import Callable, Mapping, Sequence + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.asyncmy._typing import AsyncmyConnection - from sqlspec.core import SQL, SQLResult, StatementConfig - from sqlspec.driver import CachedQuery, ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.core.cache import CachedQuery + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.core.statement import SQL, StatementConfig + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry __all__ = ("AsyncmyCursor", "AsyncmyDriver", "AsyncmyExceptionHandler") @@ -66,6 +77,7 @@ ASYNCMY_JSON_TYPE_CODES: Final[set[int]] = {json_type_value} if json_type_value is not None else set() +@mypyc_attr(allow_interpreted_subclasses=True) class AsyncmyExceptionHandler(BaseAsyncExceptionHandler): """Async context manager for handling asyncmy (MySQL) database exceptions. @@ -84,6 +96,7 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False +@mypyc_attr(allow_interpreted_subclasses=True) class AsyncmyDriver(AsyncDriverAdapterBase): """MySQL/MariaDB database driver using AsyncMy client library. @@ -93,7 +106,7 @@ class AsyncmyDriver(AsyncDriverAdapterBase): """ __slots__ = ("_data_dictionary", "_json_deserializer") - dialect = "mysql" + dialect: "DialectType | None" = "mysql" def __init__( self, @@ -237,7 +250,7 @@ async def begin(self) -> None: try: await self.connection.begin() except AsyncmyMySQLError as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e async def commit(self) -> None: """Commit the current transaction. @@ -248,7 +261,7 @@ async def commit(self) -> None: try: await self.connection.commit() except AsyncmyMySQLError as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e async def rollback(self) -> None: """Rollback the current transaction. @@ -259,7 +272,7 @@ async def rollback(self) -> None: try: await self.connection.rollback() except AsyncmyMySQLError as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e def with_cursor(self, connection: "AsyncmyConnection") -> "AsyncmyCursor": """Create cursor context manager for the connection. @@ -385,7 +398,7 @@ async def load_from_arrow( self._attach_partition_telemetry(telemetry_payload, partitioner) return self._storage_job(telemetry_payload, telemetry) - def collect_rows(self, cursor: Any, fetched: "list[Any]") -> "tuple[list[Any], list[str], int]": + def collect_rows(self, cursor: Any, fetched: "Sequence[Any]") -> "tuple[list[Any], list[str], int]": """Collect asyncmy rows for the direct execution path.""" description = cursor.description or None row_plan = resolve_row_plan(description, ASYNCMY_JSON_TYPE_CODES) @@ -413,20 +426,38 @@ async def _execute_cache_hit( return await super()._execute_cache_hit(sql, params, cached) async def _execute_cursor_statement(self, statement: str, parameters: Any = None) -> None: - async with self.with_cursor(self.connection) as cursor: + cursor_context = self.with_cursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: if parameters is None: await cursor.execute(statement) else: await cursor.execute(statement, parameters) + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) async def _executemany_cursor_statement(self, statement: str, parameters: Any) -> None: - async with self.with_cursor(self.connection) as cursor: + cursor_context = self.with_cursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: await cursor.executemany(statement, parameters) + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) async def _execute_local_infile_cursor(self, statement: str, filename: str) -> None: - async with self.with_cursor(self.connection) as cursor: + cursor_context = self.with_cursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: with asyncmy_local_infile(self.connection, filename): await cursor.execute(statement, {"sqlspec_infile_path": filename}) + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) async def _load_from_arrow_via_local_infile( self, table: str, columns: "list[str]", records: "list[tuple[Any, ...]]" @@ -446,23 +477,28 @@ async def _load_from_arrow_via_local_infile( if pending_cancellation is not None: raise pending_cancellation raise - try: - if pending_cancellation is not None: - raise pending_cancellation - statement = build_load_data_statement(table, columns) - exc_handler = self.handle_database_exceptions() - await self._run_with_exception_handler(exc_handler, self._execute_local_infile_cursor, statement, filename) - self._check_pending_exception(exc_handler) - finally: - cleanup = asyncio.create_task(asyncio.to_thread(shutil.rmtree, directory, ignore_errors=True)) - while not cleanup.done(): - try: - await asyncio.shield(cleanup) - except asyncio.CancelledError as exc: - pending_cancellation = exc - cleanup.result() - if pending_cancellation is not None: - raise pending_cancellation + load_error: BaseException | None = None + if pending_cancellation is None: + try: + statement = build_load_data_statement(table, columns) + exc_handler = self.handle_database_exceptions() + await self._run_with_exception_handler( + exc_handler, self._execute_local_infile_cursor, statement, filename + ) + self._check_pending_exception(exc_handler) + except BaseException as exc: + load_error = exc + cleanup = asyncio.create_task(asyncio.to_thread(shutil.rmtree, directory, ignore_errors=True)) + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError as exc: + pending_cancellation = exc + cleanup.result() + if pending_cancellation is not None: + raise pending_cancellation + if load_error is not None: + raise load_error def _connection_in_transaction(self) -> bool: """Check if connection is in transaction. diff --git a/sqlspec/adapters/asyncpg/_typing.py b/sqlspec/adapters/asyncpg/_typing.py index 026d6a778..8a1262e88 100644 --- a/sqlspec/adapters/asyncpg/_typing.py +++ b/sqlspec/adapters/asyncpg/_typing.py @@ -4,7 +4,7 @@ compilation to avoid ABI boundary issues. """ -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any import asyncpg as asyncpg_module from asyncpg import Connection as AsyncpgRawConnection @@ -21,11 +21,10 @@ from asyncpg.pool import PoolConnectionProxyMeta as AsyncpgPoolConnectionProxyMeta from asyncpg.prepared_stmt import PreparedStatement +from sqlspec.driver._session import AsyncDriverSessionContext from sqlspec.typing import import_optional_attr if TYPE_CHECKING: - from collections.abc import Callable - from types import TracebackType from typing import TypeAlias from asyncpg import Connection, Record @@ -33,7 +32,6 @@ from google.cloud.sql.connector import Connector as AsyncpgCloudSqlConnector from sqlspec.adapters.asyncpg.driver import AsyncpgDriver - from sqlspec.core import StatementConfig AsyncpgConnection: TypeAlias = Connection[Record] | PoolConnectionProxy[Record] AsyncpgPool: TypeAlias = Pool[Record] @@ -80,86 +78,19 @@ async def __aenter__(self) -> "AsyncpgConnection": async def __aexit__(self, *_: Any) -> None: ... -class AsyncpgSessionContext: - """Async context manager for AsyncPG sessions. - - This class is intentionally excluded from mypyc compilation to avoid ABI - boundary issues. It receives callables from uncompiled config classes and - instantiates compiled Driver objects, acting as a bridge between compiled - and uncompiled code. - - Uses callable-based connection management to decouple from config implementation. - """ - - __slots__ = ( - "_acquire_connection", - "_connection", - "_driver", - "_driver_features", - "_prepare_driver", - "_release_connection", - "_statement_config", - ) - - def __init__( - self, - acquire_connection: "Callable[[], Any]", - release_connection: "Callable[..., Any]", - statement_config: "StatementConfig | Callable[[], StatementConfig]", - driver_features: "dict[str, Any]", - prepare_driver: "Callable[[AsyncpgDriver], AsyncpgDriver]", - ) -> None: - self._acquire_connection = acquire_connection - self._release_connection = release_connection - self._statement_config = statement_config - self._driver_features = driver_features - self._prepare_driver = prepare_driver - self._connection: Any = None - self._driver: AsyncpgDriver | None = None - - async def __aenter__(self) -> "AsyncpgDriver": - driver_cls = _resolve_asyncpg_driver() - self._connection = await self._acquire_connection() - try: - statement_config = self._statement_config() if callable(self._statement_config) else self._statement_config - self._driver = driver_cls( - connection=self._connection, statement_config=statement_config, driver_features=self._driver_features - ) - return self._prepare_driver(self._driver) - except BaseException as exc: - driver, self._driver = self._driver, None - connection, self._connection = self._connection, None - try: - if connection is not None: - await self._release_connection( - connection, exc_type=type(exc), exc_val=exc, exc_tb=exc.__traceback__ - ) - finally: - if driver is not None: - driver._mark_closed() - raise - - async def __aexit__( - self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None" - ) -> "bool | None": - driver = self._driver - try: - if driver is not None: - await driver._close_open_streams(error=exc_type is not None) - except BaseException as close_exc: - if exc_type is None: - exc_type, exc_val, exc_tb = type(close_exc), close_exc, close_exc.__traceback__ - raise - finally: - try: - if self._connection is not None: - await self._release_connection(self._connection, exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) - self._connection = None - finally: - self._driver = None - if driver is not None: - driver._mark_closed() - return None +class AsyncpgSessionContext(AsyncDriverSessionContext): + """Async context manager for AsyncPG sessions.""" + + __slots__ = () + + if TYPE_CHECKING: + + async def __aenter__(self) -> "AsyncpgDriver": ... + + def _resolve_driver_class(self) -> "type[AsyncpgDriver]": + from sqlspec.adapters.asyncpg.driver import AsyncpgDriver + + return AsyncpgDriver _LAZY_DRIVER_EXPORTS: dict[str, tuple[str, str]] = { @@ -167,8 +98,6 @@ async def __aexit__( "AsyncpgCloudSqlConnector": ("google.cloud.sql.connector", "Connector"), } -_ASYNCPG_DRIVER_CLS: "type[AsyncpgDriver] | None" = None - def __getattr__(name: str) -> Any: """Resolve optional driver symbols only when a consumer requests them.""" @@ -182,16 +111,3 @@ def __getattr__(name: str) -> Any: msg = f"Cannot import {attribute!r} from {module_name!r}" raise ImportError(msg) return value - - -def _resolve_asyncpg_driver() -> "type[AsyncpgDriver]": - global _ASYNCPG_DRIVER_CLS - driver_cls = _ASYNCPG_DRIVER_CLS - if driver_cls is None: - resolved = import_optional_attr("sqlspec.adapters.asyncpg.driver", "AsyncpgDriver") - if resolved is None: - msg = "Cannot import 'AsyncpgDriver' from 'sqlspec.adapters.asyncpg.driver'" - raise ImportError(msg) - driver_cls = cast("type[AsyncpgDriver]", resolved) - _ASYNCPG_DRIVER_CLS = driver_cls - return driver_cls diff --git a/sqlspec/adapters/asyncpg/core.py b/sqlspec/adapters/asyncpg/core.py index a0eb5830b..a3052106e 100644 --- a/sqlspec/adapters/asyncpg/core.py +++ b/sqlspec/adapters/asyncpg/core.py @@ -12,8 +12,9 @@ from sqlspec.adapters.asyncpg._typing import AsyncpgPostgresError from sqlspec.adapters.asyncpg._typing import asyncpg_module as asyncpg -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile from sqlspec.core.config_runtime import resolve_runtime_statement_config +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult from sqlspec.exceptions import ( CheckViolationError, @@ -39,15 +40,17 @@ from sqlspec.utils.dispatch import TypeDispatcher from sqlspec.utils.logging import get_logger from sqlspec.utils.module_loader import import_optional -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_sqlstate if TYPE_CHECKING: from collections.abc import Callable, Mapping - from sqlspec.core import SQL, ParameterStyleConfig, StackOperation from sqlspec.core.cache import CachedQuery + from sqlspec.core.parameters._types import ParameterStyleConfig + from sqlspec.core.stack import StackOperation + from sqlspec.core.statement import SQL, StatementConfig __all__ = ( "AsyncpgStreamSource", @@ -84,7 +87,7 @@ class NormalizedStackOperation(NamedTuple): operation: "StackOperation" statement: "SQL" sql: str - parameters: "tuple[Any, ...] | dict[str, Any] | None" + parameters: "tuple[Any, ...] | list[Any] | dict[str, Any] | None" _EXCEPTION_MAPPING_DISPATCHER = TypeDispatcher["tuple[str, type[SQLSpecError], str]"]() @@ -401,11 +404,15 @@ def create_mapped_exception(error: Any, *, logger: Any | None = None) -> SQLSpec """ _ = logger if isinstance(error, (asyncio.TimeoutError, TimeoutError)): - return QueryTimeoutError(str(error)) + timeout_error = QueryTimeoutError(str(error)) + timeout_error.__cause__ = error + return timeout_error if isinstance(error, OSError) or ( asyncpg is not None and isinstance(error, getattr(asyncpg, "InterfaceError", ())) ): - return DatabaseConnectionError(str(error)) + connection_error = DatabaseConnectionError(str(error)) + connection_error.__cause__ = error if isinstance(error, BaseException) else None + return connection_error mapped_error = _EXCEPTION_MAPPING_DISPATCHER.get(error) if mapped_error is not None: diff --git a/sqlspec/adapters/asyncpg/driver.py b/sqlspec/adapters/asyncpg/driver.py index e565f41c4..8b038b3ae 100644 --- a/sqlspec/adapters/asyncpg/driver.py +++ b/sqlspec/adapters/asyncpg/driver.py @@ -2,11 +2,12 @@ import re from collections.abc import Mapping -from contextlib import asynccontextmanager, nullcontext, suppress +from contextlib import nullcontext, suppress from io import BytesIO from itertools import chain from typing import TYPE_CHECKING, Any, Literal, cast +from mypy_extensions import mypyc_attr from sqlglot import exp, parse_one from sqlglot.errors import ParseError @@ -24,38 +25,39 @@ resolve_many_rowcount, ) from sqlspec.adapters.asyncpg.data_dictionary import AsyncpgDataDictionary -from sqlspec.core import ( - SQL, - DMLResult, - StackResult, - StatementStack, - get_cache_config, - is_copy_from_operation, - is_copy_operation, - register_driver_profile, -) -from sqlspec.driver import ( - AsyncDriverAdapterBase, - AsyncRowStream, - BaseAsyncExceptionHandler, +from sqlspec.core.cache import get_cache_config +from sqlspec.core.compiler import is_copy_from_operation, is_copy_operation +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.result._base import DMLResult, StackResult +from sqlspec.core.stack import StatementStack +from sqlspec.driver._async import AsyncDriverAdapterBase +from sqlspec.driver._common import ( StackExecutionObserver, + cache_hit_hooks_unchanged, describe_stack_statement, validate_savepoint_name, ) -from sqlspec.driver._common import cache_hit_hooks_unchanged -from sqlspec.exceptions import ImproperConfigurationError, SQLSpecError, StackExecutionError +from sqlspec.driver._exception_handler import BaseAsyncExceptionHandler +from sqlspec.driver._stream import AsyncRowStream +from sqlspec.exceptions import ImproperConfigurationError, SQLSpecError, StackExecutionError, with_cause from sqlspec.utils.logging import get_logger +from sqlspec.utils.sync_tools import exit_async_context from sqlspec.utils.text import normalize_identifier, quote_identifier from sqlspec.utils.type_guards import has_sqlstate from sqlspec.utils.uuids import uuid4 if TYPE_CHECKING: - from collections.abc import AsyncIterator, Sequence + from collections.abc import Sequence + from types import TracebackType + + from sqlglot.dialects.dialect import DialectType from sqlspec.adapters.asyncpg._typing import AsyncpgConnection - from sqlspec.core import ArrowResult, SQLResult, StatementConfig - from sqlspec.driver import CachedQuery, ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.core.cache import CachedQuery + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.core.statement import SQL, StatementConfig + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry __all__ = ("AsyncpgCursor", "AsyncpgDriver", "AsyncpgExceptionHandler", "AsyncpgSessionContext") @@ -68,6 +70,34 @@ logger = get_logger("sqlspec.adapters.asyncpg") +@mypyc_attr(allow_interpreted_subclasses=False) +class _AsyncpgStackSavepoint: + """Async savepoint context manager compiled without async generators.""" + + __slots__ = ("_connection", "_name") + + def __init__(self, connection: Any) -> None: + self._connection = connection + self._name = validate_savepoint_name(f"sqlspec_stack_{uuid4().hex}") + + async def __aenter__(self) -> None: + await self._connection.execute(f"SAVEPOINT {self._name}") + + async def __aexit__( + self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None" + ) -> None: + rollback_error: BaseException | None = None + if exc_type is not None: + try: + await self._connection.execute(f"ROLLBACK TO SAVEPOINT {self._name}") + except BaseException as exc: + rollback_error = exc + await self._connection.execute(f"RELEASE SAVEPOINT {self._name}") + if rollback_error is not None: + raise rollback_error + + +@mypyc_attr(allow_interpreted_subclasses=True) class AsyncpgExceptionHandler(BaseAsyncExceptionHandler): """Async context manager for handling AsyncPG database exceptions. @@ -90,6 +120,7 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False +@mypyc_attr(allow_interpreted_subclasses=True) class AsyncpgDriver(AsyncDriverAdapterBase): """AsyncPG PostgreSQL driver for async database operations. @@ -99,7 +130,7 @@ class AsyncpgDriver(AsyncDriverAdapterBase): """ __slots__ = ("_data_dictionary", "_transaction") - dialect = "postgres" + dialect: "DialectType | None" = "postgres" def __init__( self, @@ -141,14 +172,20 @@ async def dispatch_execute(self, cursor: "AsyncpgConnection", statement: "SQL") ExecutionResult with statement execution details """ sql, prepared_parameters = self._compiled_sql(statement, self.statement_config) - params: tuple[Any, ...] = cast("tuple[Any, ...]", prepared_parameters) if prepared_parameters else () + args = ( + prepared_parameters + if isinstance(prepared_parameters, (tuple, list)) + else tuple(prepared_parameters.values()) + if isinstance(prepared_parameters, dict) + else () + ) timeout = self._timeout_value(statement.statement_config) if statement.returns_rows(): records = ( - await cursor.fetch(sql, *params, timeout=timeout) + await cursor.fetch(sql, *args, timeout=timeout) if timeout is not None - else await cursor.fetch(sql, *params) + else await cursor.fetch(sql, *args) ) data, column_names = collect_rows(records) @@ -162,9 +199,9 @@ async def dispatch_execute(self, cursor: "AsyncpgConnection", statement: "SQL") ) result = ( - await cursor.execute(sql, *params, timeout=timeout) + await cursor.execute(sql, *args, timeout=timeout) if timeout is not None - else await cursor.execute(sql, *params) + else await cursor.execute(sql, *args) ) affected_rows = parse_status(result) @@ -184,7 +221,11 @@ async def dispatch_execute_many(self, cursor: "AsyncpgConnection", statement: "S timeout = self._timeout_value(statement.statement_config) if prepared_parameters: - parameter_sets = cast("list[Sequence[object]]", prepared_parameters) + parameter_sets = ( + cast("list[Sequence[object]]", prepared_parameters) + if isinstance(prepared_parameters, list) + else list(cast("Sequence[Sequence[object]]", prepared_parameters)) + ) if timeout is not None: await cursor.executemany(sql, parameter_sets, timeout=timeout) else: @@ -278,14 +319,17 @@ async def begin(self) -> None: await transaction.start() except Exception as e: self._release_failed_transaction_claim(transaction) - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e + columnar_error: Exception | None = None try: if self._should_set_local_columnar(): await self.connection.execute("SET LOCAL google_columnar_engine.enable_columnar_scan = 'on'") except Exception as e: + columnar_error = e + if columnar_error is not None: with suppress(Exception): await transaction.rollback() - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(columnar_error), columnar_error) from columnar_error self._transaction = transaction async def commit(self) -> None: @@ -298,7 +342,7 @@ async def commit(self) -> None: elif self._connection_in_transaction(): await self.connection.execute("COMMIT") except Exception as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e async def rollback(self) -> None: """Rollback the current transaction.""" @@ -310,7 +354,7 @@ async def rollback(self) -> None: elif self._connection_in_transaction(): await self.connection.execute("ROLLBACK") except Exception as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e async def set_migration_session_schema(self, schema: str) -> None: """Set the PostgreSQL search path for migration SQL.""" @@ -346,13 +390,15 @@ def dispatch_select_stream(self, statement: "SQL", chunk_size: int) -> "AsyncRow if not statement.returns_rows(): return None sql, prepared_parameters = self._compiled_sql(statement, self.statement_config) - params: tuple[Any, ...] = () - if prepared_parameters: - params = ( - tuple(prepared_parameters) - if isinstance(prepared_parameters, list) - else cast("tuple[Any, ...]", prepared_parameters) - ) + params: tuple[Any, ...] = ( + prepared_parameters + if isinstance(prepared_parameters, tuple) + else tuple(prepared_parameters) + if isinstance(prepared_parameters, list) + else tuple(prepared_parameters.values()) + if isinstance(prepared_parameters, dict) + else () + ) timeout = self._timeout_value(statement.statement_config) return AsyncRowStream(AsyncpgStreamSource(self, sql, params, chunk_size, timeout)) @@ -467,16 +513,22 @@ async def load_from_arrow( ) table_name, schema_name, quoted_target = self._copy_target(table) if overwrite: + if self._auto_begin_on_write and not self._connection_in_transaction(): + await self.begin() try: await self.connection.execute(f"TRUNCATE TABLE {quoted_target}") except Exception as exc: mapped = create_mapped_exception(exc) msg = f"Failed to truncate table '{table}': {exc}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc columns, batch_iter = self._coerce_arrow_batches(source) first_batch = next((b for b in batch_iter if b.num_rows > 0), None) metrics: dict[str, int] = {"rows_processed": 0, "bytes_processed": 0} if first_batch is not None: + if self._auto_begin_on_write and not self._connection_in_transaction(): + await self.begin() all_batches = chain((first_batch,), (b for b in batch_iter if b.num_rows > 0)) use_binary = bool(self.driver_features.get("enable_pgcopy_binary", False)) try: @@ -495,7 +547,7 @@ async def load_from_arrow( table_name, records=records, columns=columns, schema_name=schema_name ) except Exception as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc telemetry_payload = self._ingest_telemetry_from_batches(metrics["rows_processed"], metrics["bytes_processed"]) telemetry_payload["destination"] = table self._attach_partition_telemetry(telemetry_payload, partitioner) @@ -538,12 +590,16 @@ async def load_from_records( if not materialized: if overwrite: table_name, schema_name, quoted_target = self._copy_target(table) + if self._auto_begin_on_write and not self._connection_in_transaction(): + await self.begin() try: await self.connection.execute(f"TRUNCATE TABLE {quoted_target}") except Exception as exc: mapped = create_mapped_exception(exc) msg = f"Failed to truncate table '{table}': {exc}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc empty_telemetry: StorageTelemetry = { "bytes_processed": 0, "destination": table, @@ -584,19 +640,23 @@ async def load_from_records( copy_rows.append(row) table_name, schema_name, quoted_target = self._copy_target(table) + if self._auto_begin_on_write and not self._connection_in_transaction(): + await self.begin() if overwrite: try: await self.connection.execute(f"TRUNCATE TABLE {quoted_target}") except Exception as exc: mapped = create_mapped_exception(exc) msg = f"Failed to truncate table '{table}': {exc}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc try: await self.connection.copy_records_to_table( table_name, records=copy_rows, columns=resolved_columns, schema_name=schema_name ) except Exception as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc telemetry_payload: StorageTelemetry = { "bytes_processed": 0, "destination": table, @@ -619,10 +679,17 @@ def _timeout_arguments(self, config: "StatementConfig") -> "dict[str, Any]": return {"timeout": timeout} if timeout is not None else {} def _release_failed_transaction_claim(self, transaction: Any) -> None: - """Clear the connection's transaction claim left by a failed start.""" + """Clear the connection's transaction claim left by a failed start. + + The claim is cleared on the connection that holds it, which is the + wrapped connection when the driver's connection is a pool proxy. + """ with suppress(Exception): - if getattr(self.connection, "_top_xact", None) is transaction: - self.connection._top_xact = None + connection = getattr(self.connection, "_con", None) + if connection is None: + connection = self.connection + if getattr(connection, "_top_xact", None) is transaction: + connection._top_xact = None @staticmethod def _copy_target(table: str) -> "tuple[str, str | None, str]": @@ -630,7 +697,9 @@ def _copy_target(table: str) -> "tuple[str, str | None, str]": parsed = parse_one(table, read="postgres", into=exp.Table) except ParseError as exc: msg = "AsyncPG COPY targets must be unqualified or schema-qualified table names." - raise ImproperConfigurationError(msg) from exc + error = ImproperConfigurationError(msg) + error.__cause__ = exc + raise error from exc schema_expression = parsed.args.get("db") schema_identifier = schema_expression if isinstance(schema_expression, exp.Identifier) else None if ( @@ -673,10 +742,15 @@ async def _execute_stack_native( with StackExecutionObserver(self, stack, continue_on_error, native_pipeline=False) as observer: if transaction_cm is not None: - async with transaction_cm: + await transaction_cm.__aenter__() + transaction_error: BaseException | None = None + try: await self._run_stack_operations( stack, continue_on_error, observer, results, caller_in_tx=caller_in_tx ) + except BaseException as exc: + transaction_error = exc + await exit_async_context(transaction_cm, transaction_error) else: await self._run_stack_operations(stack, continue_on_error, observer, results, caller_in_tx=caller_in_tx) @@ -696,9 +770,13 @@ async def _run_stack_operations( Extracted from _execute_stack_native to avoid closure compilation issues. """ for index, operation in enumerate(stack.operations): + operation_error: Exception | None = None + stack_result: StackResult | None = None try: scope = self._stack_savepoint() if continue_on_error and caller_in_tx else nullcontext() - async with scope: + await scope.__aenter__() + scope_error: BaseException | None = None + try: normalized: NormalizedStackOperation | None = None if operation.method == "execute": kwargs = dict(operation.keyword_arguments) if operation.keyword_arguments else {} @@ -710,7 +788,9 @@ async def _run_stack_operations( ) if not sql_statement.is_script and not sql_statement.is_many: sql_text, prepared_parameters = self._compiled_sql(sql_statement, config) - prepared_parameters = cast("tuple[Any, ...] | dict[str, Any] | None", prepared_parameters) + prepared_parameters = cast( + "tuple[Any, ...] | list[Any] | dict[str, Any] | None", prepared_parameters + ) normalized = NormalizedStackOperation( operation=operation, statement=sql_statement, @@ -723,11 +803,17 @@ async def _run_stack_operations( else: result = await self._execute_stack_operation(operation) stack_result = StackResult(result=result) + except BaseException as scope_exc: + scope_error = scope_exc + await exit_async_context(scope, scope_error) except Exception as exc: + operation_error = exc + + if operation_error is not None: stack_error = StackExecutionError( index, describe_stack_statement(operation.statement), - exc, + operation_error, adapter=type(self).__name__, mode="continue-on-error" if continue_on_error else "fail-fast", ) @@ -737,23 +823,16 @@ async def _run_stack_operations( observer.record_operation_error(stack_error) results.append(StackResult.from_error(stack_error)) continue - raise stack_error from exc + stack_error.__cause__ = operation_error + raise stack_error from operation_error + assert stack_result is not None results.append(stack_result) if continue_on_error and not caller_in_tx: await self._commit_stack_success() - @asynccontextmanager - async def _stack_savepoint(self) -> "AsyncIterator[None]": - name = validate_savepoint_name(f"sqlspec_stack_{uuid4().hex}") - await self.connection.execute(f"SAVEPOINT {name}") - try: - yield - except BaseException: - await self.connection.execute(f"ROLLBACK TO SAVEPOINT {name}") - raise - finally: - await self.connection.execute(f"RELEASE SAVEPOINT {name}") + def _stack_savepoint(self) -> "_AsyncpgStackSavepoint": + return _AsyncpgStackSavepoint(self.connection) async def _execute_stack_operation_prepared(self, normalized: "NormalizedStackOperation") -> StackResult: result = await self._execute_cached_statement(normalized.statement) diff --git a/sqlspec/adapters/asyncpg/events/_hub.py b/sqlspec/adapters/asyncpg/events/_hub.py index cc28a71cb..7d6433773 100644 --- a/sqlspec/adapters/asyncpg/events/_hub.py +++ b/sqlspec/adapters/asyncpg/events/_hub.py @@ -15,8 +15,8 @@ from weakref import WeakKeyDictionary from sqlspec.exceptions import EventChannelError, ImproperConfigurationError -from sqlspec.extensions.events import normalize_event_channel_name from sqlspec.extensions.events._buffer import enqueue_with_capacity, resolve_listener_queue_capacity +from sqlspec.extensions.events._store import normalize_event_channel_name from sqlspec.utils.logging import get_logger, log_with_context from sqlspec.utils.type_guards import has_add_listener diff --git a/sqlspec/adapters/asyncpg/tenant.py b/sqlspec/adapters/asyncpg/tenant.py index 4d657f9c7..222df660b 100644 --- a/sqlspec/adapters/asyncpg/tenant.py +++ b/sqlspec/adapters/asyncpg/tenant.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from sqlspec.adapters.asyncpg._typing import AsyncpgConnection from sqlspec.core.tenant.base import AsyncTenantStrategy - from sqlspec.utils.tenant import TenantContextPayload + from sqlspec.utils.tenant._context import TenantContextPayload __all__ = ("AsyncpgRLSStrategy", "AsyncpgTenantLease", "apply_session_baseline", "validate_asyncpg_tenant_config") diff --git a/sqlspec/adapters/bigquery/config.py b/sqlspec/adapters/bigquery/config.py index d20475a87..85a6b6b70 100644 --- a/sqlspec/adapters/bigquery/config.py +++ b/sqlspec/adapters/bigquery/config.py @@ -197,6 +197,7 @@ class BigQueryConfig(NoPoolSyncConfig[BigQueryConnection, BigQueryDriver]): driver_type: ClassVar[type[BigQueryDriver]] = BigQueryDriver connection_type: "ClassVar[type[BigQueryConnection]]" = BigQueryConnection + supports_transactions: ClassVar[bool] = False supports_transactional_ddl: ClassVar[bool] = False supports_native_parquet_import: ClassVar[bool] = True supports_native_arrow_export: ClassVar[bool] = True diff --git a/sqlspec/adapters/bigquery/core.py b/sqlspec/adapters/bigquery/core.py index f6f62ce86..b6b9d1c9a 100644 --- a/sqlspec/adapters/bigquery/core.py +++ b/sqlspec/adapters/bigquery/core.py @@ -21,15 +21,10 @@ from sqlspec.adapters.bigquery._typing import BigQueryRetry as Retry from sqlspec.adapters.bigquery._typing import bigquery_exceptions as api_exceptions from sqlspec.adapters.bigquery._typing import bigquery_module as bigquery -from sqlspec.core import ( - DriverParameterProfile, - ParameterProfile, - ParameterStyle, - StatementConfig, - TypedParameter, - build_null_pruning_transform, - build_statement_config_from_profile, -) +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._transformers import build_null_pruning_transform +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterProfile, ParameterStyle, TypedParameter +from sqlspec.core.statement import StatementConfig from sqlspec.exceptions import ( DataError, ImproperConfigurationError, @@ -48,7 +43,7 @@ from sqlspec.storage._utils import import_pyarrow, import_pyarrow_parquet from sqlspec.typing import PYARROW_INSTALLED from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_errors @@ -56,8 +51,8 @@ from collections.abc import Callable, Iterable, Iterator, Mapping from typing import Literal - from sqlspec.driver import SyncExceptionHandler - from sqlspec.storage import StorageFormat, StorageTelemetry + from sqlspec.driver._common import SyncExceptionHandler + from sqlspec.storage.pipeline import StorageFormat, StorageTelemetry from sqlspec.typing import StatementParameters BigQueryLoadFormat = Literal["jsonl", "json", "parquet", "arrow-ipc", "csv", "avro", "orc"] diff --git a/sqlspec/adapters/bigquery/driver.py b/sqlspec/adapters/bigquery/driver.py index 6a9a48f67..7f53835cc 100644 --- a/sqlspec/adapters/bigquery/driver.py +++ b/sqlspec/adapters/bigquery/driver.py @@ -12,6 +12,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast +from mypy_extensions import mypyc_attr from sqlglot import exp from sqlspec.adapters.bigquery._typing import BIGQUERY_POLLING_DEFAULT_VALUE as POLLING_DEFAULT_VALUE @@ -52,34 +53,37 @@ uses_local_bigquery_endpoint, ) from sqlspec.adapters.bigquery.data_dictionary import BigQueryDataDictionary -from sqlspec.core import ( - SQL, - StatementConfig, - build_arrow_result_from_reader, - build_arrow_result_from_table, - build_literal_inlining_transform, - get_cache_config, - register_driver_profile, -) -from sqlspec.driver import BaseSyncExceptionHandler, ExecutionResult, SyncDriverAdapterBase, SyncRowStream +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.parameters._transformers import build_literal_inlining_transform +from sqlspec.core.result._base import build_arrow_result_from_reader, build_arrow_result_from_table +from sqlspec.core.statement import SQL, StatementConfig +from sqlspec.driver._common import ExecutionResult, validate_load_from_storage_kwargs +from sqlspec.driver._exception_handler import BaseSyncExceptionHandler +from sqlspec.driver._stream import SyncRowStream +from sqlspec.driver._sync import SyncDriverAdapterBase from sqlspec.exceptions import ImproperConfigurationError, StorageOperationFailedError from sqlspec.storage._utils import import_pyarrow, import_pyarrow_parquet from sqlspec.utils.logging import get_logger from sqlspec.utils.module_loader import ensure_pyarrow -from sqlspec.utils.serializers import to_json +from sqlspec.utils.serializers._json import to_json from sqlspec.utils.text import split_qualified_identifier if TYPE_CHECKING: from collections.abc import Callable, Iterable, Sequence + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.bigquery._typing import BigQueryQueryJob as QueryJob from sqlspec.adapters.bigquery._typing import BigQueryQueryJobConfig as QueryJobConfig from sqlspec.adapters.bigquery._typing import BigQueryRetry as Retry from sqlspec.adapters.bigquery._typing import bigquery_storage_read_module as bigquery_storage from sqlspec.adapters.bigquery.core import BigQueryLoadFormat - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, Statement, StatementFilter - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult + from sqlspec.core.statement import Statement + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry from sqlspec.typing import ArrowRecordBatch, ArrowRecordBatchReader, ArrowReturnFormat, StatementParameters __all__ = ("BigQueryCursor", "BigQueryDriver", "BigQueryExceptionHandler", "BigQuerySessionContext") @@ -89,6 +93,7 @@ _PROJECT_DATASET_TABLE_PARTS = 3 +@mypyc_attr(allow_interpreted_subclasses=True) class BigQueryExceptionHandler(BaseSyncExceptionHandler): """Context manager for handling BigQuery API exceptions. @@ -107,6 +112,7 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False +@mypyc_attr(allow_interpreted_subclasses=True) class BigQueryDriver(SyncDriverAdapterBase): """BigQuery driver implementation. @@ -130,7 +136,7 @@ class BigQueryDriver(SyncDriverAdapterBase): "_request_timeout", "_session_id", ) - dialect = "bigquery" + dialect: "DialectType | None" = "bigquery" native_arrow_export: ClassVar[bool] = True def __init__( @@ -548,7 +554,10 @@ def select_to_arrow( "3. Grant permissions: roles/bigquery.dataViewer\n" "4. Use a real BigQuery endpoint instead of the local emulator" ) - raise ImproperConfigurationError(msg) from RuntimeError(msg) + cause = RuntimeError(msg) + error = ImproperConfigurationError(msg) + error.__cause__ = cause + raise error from cause return super().select_to_arrow( statement, @@ -609,7 +618,10 @@ def _streaming_op() -> Any: "3. Grant permissions: roles/bigquery.dataViewer\n" "4. Use a real BigQuery endpoint instead of the local emulator" ) - raise ImproperConfigurationError(msg) from RuntimeError(msg) + cause = RuntimeError(msg) + error = ImproperConfigurationError(msg) + error.__cause__ = cause + raise error from cause result: ArrowResult = super().select_to_arrow( statement, @@ -888,13 +900,27 @@ def load_from_storage( ) -> "StorageBridgeJob": """Load staged artifacts from storage into BigQuery.""" if kwargs or not isinstance(source, (str, Path)): + validate_load_from_storage_kwargs(kwargs) return super().load_from_storage( table, source, file_format=cast("StorageFormat", file_format), partitioner=partitioner, overwrite=overwrite, - **kwargs, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) job_config = build_load_job_config(file_format, overwrite) diff --git a/sqlspec/adapters/cockroach_asyncpg/_typing.py b/sqlspec/adapters/cockroach_asyncpg/_typing.py index 5265df903..e37e39640 100644 --- a/sqlspec/adapters/cockroach_asyncpg/_typing.py +++ b/sqlspec/adapters/cockroach_asyncpg/_typing.py @@ -1,6 +1,6 @@ """CockroachDB AsyncPG adapter type definitions.""" -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING import asyncpg as cockroach_asyncpg_module from asyncpg import InterfaceError as CockroachAsyncpgInterfaceError @@ -11,17 +11,14 @@ from asyncpg import create_pool as cockroach_asyncpg_create_pool from asyncpg.pool import PoolConnectionProxy -from sqlspec.typing import import_optional_attr +from sqlspec.driver._session import AsyncDriverSessionContext if TYPE_CHECKING: - from collections.abc import Callable - from types import TracebackType from typing import TypeAlias from asyncpg import Connection, Record from sqlspec.adapters.cockroach_asyncpg.driver import CockroachAsyncpgDriver - from sqlspec.core import StatementConfig CockroachAsyncpgConnection: TypeAlias = Connection[Record] | PoolConnectionProxy[Record] CockroachAsyncpgPool: TypeAlias = Pool[Record] @@ -41,82 +38,17 @@ "cockroach_asyncpg_module", ) -_COCKROACH_ASYNCPG_DRIVER_CLS: "type[CockroachAsyncpgDriver] | None" = None - -class CockroachAsyncpgSessionContext: +class CockroachAsyncpgSessionContext(AsyncDriverSessionContext): """Async context manager for CockroachDB AsyncPG sessions.""" - __slots__ = ( - "_acquire_connection", - "_connection", - "_driver", - "_driver_features", - "_prepare_driver", - "_release_connection", - "_statement_config", - ) - - def __init__( - self, - acquire_connection: "Callable[[], Any]", - release_connection: "Callable[..., Any]", - statement_config: "StatementConfig | Callable[[], StatementConfig]", - driver_features: "dict[str, Any]", - prepare_driver: "Callable[[CockroachAsyncpgDriver], CockroachAsyncpgDriver]", - ) -> None: - self._acquire_connection = acquire_connection - self._release_connection = release_connection - self._statement_config = statement_config - self._driver_features = driver_features - self._prepare_driver = prepare_driver - self._connection: Any = None - self._driver: CockroachAsyncpgDriver | None = None + __slots__ = () - async def __aenter__(self) -> "CockroachAsyncpgDriver": - driver_cls = _resolve_cockroach_asyncpg_driver() - self._connection = await self._acquire_connection() - try: - statement_config = self._statement_config() if callable(self._statement_config) else self._statement_config - self._driver = driver_cls( - connection=self._connection, statement_config=statement_config, driver_features=self._driver_features - ) - return self._prepare_driver(self._driver) - except BaseException as exc: - await self.__aexit__(type(exc), exc, exc.__traceback__) - raise + if TYPE_CHECKING: - async def __aexit__( - self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None" - ) -> "bool | None": - driver = self._driver - try: - if driver is not None: - await driver._close_open_streams(error=exc_type is not None) - except BaseException as close_exc: - if exc_type is None: - exc_type, exc_val, exc_tb = type(close_exc), close_exc, close_exc.__traceback__ - raise - finally: - try: - if self._connection is not None: - await self._release_connection(self._connection, exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) - self._connection = None - finally: - self._driver = None - if driver is not None: - driver._mark_closed() - return None + async def __aenter__(self) -> "CockroachAsyncpgDriver": ... + def _resolve_driver_class(self) -> "type[CockroachAsyncpgDriver]": + from sqlspec.adapters.cockroach_asyncpg.driver import CockroachAsyncpgDriver -def _resolve_cockroach_asyncpg_driver() -> "type[CockroachAsyncpgDriver]": - global _COCKROACH_ASYNCPG_DRIVER_CLS - driver_cls = _COCKROACH_ASYNCPG_DRIVER_CLS - if driver_cls is None: - resolved = import_optional_attr("sqlspec.adapters.cockroach_asyncpg.driver", "CockroachAsyncpgDriver") - if resolved is None: - msg = "Cannot import 'CockroachAsyncpgDriver' from 'sqlspec.adapters.cockroach_asyncpg.driver'" - raise ImportError(msg) - driver_cls = cast("type[CockroachAsyncpgDriver]", resolved) - _COCKROACH_ASYNCPG_DRIVER_CLS = driver_cls - return driver_cls + return CockroachAsyncpgDriver diff --git a/sqlspec/adapters/cockroach_asyncpg/config.py b/sqlspec/adapters/cockroach_asyncpg/config.py index 215f2d862..c5c05792b 100644 --- a/sqlspec/adapters/cockroach_asyncpg/config.py +++ b/sqlspec/adapters/cockroach_asyncpg/config.py @@ -45,7 +45,7 @@ if TYPE_CHECKING: from asyncio.events import AbstractEventLoop - from collections.abc import AsyncIterator, Awaitable, Callable + from collections.abc import AsyncGenerator, Awaitable, Callable from contextlib import AbstractAsyncContextManager from uuid import UUID @@ -445,12 +445,14 @@ def provide_session( return CockroachAsyncpgSessionContext( acquire_connection=factory.acquire_connection, release_connection=factory.release_connection, - statement_config=statement_config - or (lambda: resolve_runtime_statement_config(None, self.statement_config, default_statement_config)), + statement_config=statement_config or self._default_runtime_statement_config, driver_features=driver_features, prepare_driver=self._prepare_driver, ) + def _default_runtime_statement_config(self) -> "StatementConfig": + return resolve_runtime_statement_config(None, self.statement_config, default_statement_config) + def provide_replica_session( self, *args: Any, @@ -556,7 +558,7 @@ def _build_replica_spec_overrides(self, spec_input: str | bool | Mapping[str, An @asynccontextmanager async def _replica_connection( self, context: "AbstractAsyncContextManager[CockroachAsyncpgDriver]" - ) -> "AsyncIterator[Any]": + ) -> "AsyncGenerator[Any, None]": async with context as session: await session.begin() failed = False diff --git a/sqlspec/adapters/cockroach_asyncpg/core.py b/sqlspec/adapters/cockroach_asyncpg/core.py index 9b77384d0..a3214f5eb 100644 --- a/sqlspec/adapters/cockroach_asyncpg/core.py +++ b/sqlspec/adapters/cockroach_asyncpg/core.py @@ -14,7 +14,8 @@ from sqlspec.adapters.cockroach_asyncpg._typing import CockroachAsyncpgInterfaceError, CockroachAsyncpgPostgresError from sqlspec.adapters.cockroach_asyncpg._typing import cockroach_asyncpg_module as asyncpg -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult from sqlspec.exceptions import ( CheckViolationError, @@ -42,7 +43,7 @@ from sqlspec.utils.logging import get_logger from sqlspec.utils.module_loader import import_optional from sqlspec.utils.routing import RoutingContext -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import quote_identifier, split_qualified_identifier from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_sqlstate @@ -50,9 +51,10 @@ if TYPE_CHECKING: from collections.abc import Callable - from sqlspec.core import ParameterStyleConfig from sqlspec.core.cache import CachedQuery - from sqlspec.storage import StorageTelemetry + from sqlspec.core.parameters._types import ParameterStyleConfig + from sqlspec.core.statement import StatementConfig + from sqlspec.storage.pipeline import StorageTelemetry __all__ = ( "DEFAULT_FOLLOWER_READ_STALENESS", @@ -496,12 +498,16 @@ def create_mapped_exception(error: Any, *, logger: Any | None = None) -> SQLSpec return mapped if isinstance(error, (asyncio.TimeoutError, TimeoutError)): - return QueryTimeoutError(str(error)) + timeout_error = QueryTimeoutError(str(error)) + timeout_error.__cause__ = error + return timeout_error if isinstance(error, OSError) or ( asyncpg is not None and isinstance(error, getattr(asyncpg, "InterfaceError", ())) ): - return DatabaseConnectionError(str(error)) + connection_error = DatabaseConnectionError(str(error)) + connection_error.__cause__ = error if isinstance(error, BaseException) else None + return connection_error mapped_error = _EXCEPTION_MAPPING_DISPATCHER.get(error) if mapped_error is not None: diff --git a/sqlspec/adapters/cockroach_asyncpg/driver.py b/sqlspec/adapters/cockroach_asyncpg/driver.py index 4f1a65b73..5180e34c9 100644 --- a/sqlspec/adapters/cockroach_asyncpg/driver.py +++ b/sqlspec/adapters/cockroach_asyncpg/driver.py @@ -5,6 +5,8 @@ from contextlib import suppress from typing import TYPE_CHECKING, Any, TypeVar, cast +from mypy_extensions import mypyc_attr + from sqlspec.adapters.asyncpg.driver import AsyncpgDriver, AsyncpgExceptionHandler from sqlspec.adapters.cockroach_asyncpg._typing import ( CockroachAsyncpgInterfaceError, @@ -27,21 +29,26 @@ validate_follower_read_staleness, ) from sqlspec.adapters.cockroach_asyncpg.data_dictionary import CockroachAsyncpgDataDictionary -from sqlspec.core import SQL, register_driver_profile -from sqlspec.driver import AsyncRowStream -from sqlspec.driver._common import cache_hit_hooks_unchanged -from sqlspec.exceptions import ReadOnlyTransactionError +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.statement import SQL +from sqlspec.driver._common import cache_hit_hooks_unchanged, validate_load_from_storage_kwargs +from sqlspec.driver._stream import AsyncRowStream +from sqlspec.exceptions import ReadOnlyTransactionError, with_cause from sqlspec.utils.logging import get_logger from sqlspec.utils.type_guards import has_sqlstate if TYPE_CHECKING: from collections.abc import Awaitable, Callable + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.cockroach_asyncpg._typing import CockroachAsyncpgConnection - from sqlspec.core import SQLResult, StatementConfig - from sqlspec.driver import CachedQuery, ExecutionResult + from sqlspec.core.cache import CachedQuery + from sqlspec.core.result._base import SQLResult + from sqlspec.core.statement import StatementConfig + from sqlspec.driver._common import ExecutionResult from sqlspec.driver._stream import AsyncRowSource - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry __all__ = ( "CockroachAsyncpgDriver", @@ -70,35 +77,39 @@ async def start(self) -> None: self._connection = self._driver.connection if self._driver._wants_follower_read(self._statement): self._auto_started = await self._driver._begin_follower_read_transaction() + start_error: BaseException | None = None try: await self._underlying.start() - except BaseException: + except BaseException as exc: + start_error = exc + if start_error is not None: if self._auto_started and self._driver._connection_in_transaction(): with suppress(Exception): await self._driver.rollback() - raise + raise start_error async def fetch_chunk(self) -> "list[dict[str, Any]]": return await self._underlying.fetch_chunk() async def close(self, error: bool = False) -> None: + close_error: BaseException | None = None try: await self._underlying.close(error=error) - finally: - connection = self._connection - self._connection = None - if ( - self._auto_started - and self._driver.connection is connection - and self._driver._connection_in_transaction() - ): - if error: - with suppress(Exception): - await self._driver.rollback() - else: - await self._driver.commit() + except BaseException as exc: + close_error = exc + connection = self._connection + self._connection = None + if self._auto_started and self._driver.connection is connection and self._driver._connection_in_transaction(): + if error: + with suppress(Exception): + await self._driver.rollback() + else: + await self._driver.commit() + if close_error is not None: + raise close_error +@mypyc_attr(allow_interpreted_subclasses=True) class CockroachAsyncpgExceptionHandler(AsyncpgExceptionHandler): """Async context manager for CockroachDB AsyncPG exceptions.""" @@ -114,11 +125,12 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False +@mypyc_attr(allow_interpreted_subclasses=True) class CockroachAsyncpgDriver(AsyncpgDriver): """CockroachDB AsyncPG driver with retry support.""" __slots__ = ("_enable_retry", "_follower_staleness", "_retry_config") - dialect = "postgres" + dialect: "DialectType | None" = "postgres" def __init__( self, @@ -134,7 +146,6 @@ def __init__( self.driver_features.get("default_staleness") or (DEFAULT_FOLLOWER_READ_STALENESS if self.driver_features.get("enable_follower_reads") else None), ) - self._data_dictionary = None self._cache_hit_executor = ( execute_cache_hit if cache_hit_hooks_unchanged(type(self), CockroachAsyncpgDriver) else None ) @@ -143,7 +154,7 @@ def __init__( @property def data_dictionary(self) -> "CockroachAsyncpgDataDictionary": if self._data_dictionary is None: - object.__setattr__(self, "_data_dictionary", CockroachAsyncpgDataDictionary()) + self._data_dictionary = CockroachAsyncpgDataDictionary() return cast("CockroachAsyncpgDataDictionary", self._data_dictionary) async def select_to_storage( @@ -236,8 +247,27 @@ async def load_from_storage( produced = native_import_telemetry(rows, table, resolved.protocol, file_format) self._attach_partition_telemetry(produced, partitioner) return self._storage_job(produced) + validate_load_from_storage_kwargs(kwargs) return await super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs + table, + source, + file_format=file_format, + partitioner=partitioner, + overwrite=overwrite, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) async def dispatch_special_handling(self, cursor: Any, statement: SQL) -> "SQLResult | None": @@ -256,12 +286,15 @@ async def begin(self) -> None: await super().begin() return await super().begin() + begin_error: Exception | None = None try: await self._apply_follower_reads() except Exception as exc: + begin_error = exc + if begin_error is not None: with suppress(Exception): await self.rollback() - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(begin_error), begin_error) from begin_error async def run_transaction_with_retry(self, operation: "Callable[[], Awaitable[_T]]") -> _T: """Execute a full CockroachDB transaction callback with serialization retries. @@ -276,19 +309,21 @@ async def run_transaction_with_retry(self, operation: "Callable[[], Awaitable[_T attempt = 0 while True: + tx_error: Exception | None = None try: await self.begin() result = await operation() await self.commit() except Exception as exc: - try: - await self.rollback() - except Exception: - raise exc - if not is_retryable_error(exc) or attempt >= self._retry_config.max_retries: - raise + tx_error = exc else: return result + try: + await self.rollback() + except Exception: + raise tx_error + if not is_retryable_error(tx_error) or attempt >= self._retry_config.max_retries: + raise tx_error delay = calculate_backoff_seconds(attempt, self._retry_config) if self._retry_config.enable_logging: logger.debug("CockroachDB retry %s/%s after %.3fs", attempt + 1, self._retry_config.max_retries, delay) @@ -297,15 +332,20 @@ async def run_transaction_with_retry(self, operation: "Callable[[], Awaitable[_T async def dispatch_execute(self, cursor: Any, statement: SQL) -> "ExecutionResult": auto_started = await self._begin_follower_read_transaction() if self._wants_follower_read(statement) else False + exec_error: BaseException | None = None + result: ExecutionResult | None = None try: result = await super().dispatch_execute(cursor, statement) - except BaseException: + except BaseException as exc: + exec_error = exc + if exec_error is not None: if auto_started and self._connection_in_transaction(): with suppress(Exception): await self.rollback() - raise + raise exec_error if auto_started and self._connection_in_transaction(): await self.commit() + assert result is not None return result def dispatch_select_stream(self, statement: "SQL", chunk_size: int) -> "AsyncRowStream[dict[str, Any]] | None": @@ -336,15 +376,20 @@ async def _execute_cache_hit( auto_started = ( await self._begin_follower_read_transaction() if (cached.operation_profile.is_read_only) else False ) + exec_error: BaseException | None = None + result: SQLResult | None = None try: result = await super()._execute_cache_hit(sql, params, cached) - except BaseException: + except BaseException as exc: + exec_error = exc + if exec_error is not None: if auto_started and self._connection_in_transaction(): with suppress(Exception): await self.rollback() - raise + raise exec_error if auto_started and self._connection_in_transaction(): await self.commit() + assert result is not None return result def _native_storage_ready(self) -> bool: diff --git a/sqlspec/adapters/cockroach_asyncpg/tenant.py b/sqlspec/adapters/cockroach_asyncpg/tenant.py index 349f74621..472d4c747 100644 --- a/sqlspec/adapters/cockroach_asyncpg/tenant.py +++ b/sqlspec/adapters/cockroach_asyncpg/tenant.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from sqlspec.adapters.cockroach_asyncpg._typing import CockroachAsyncpgConnection from sqlspec.core.tenant.base import AsyncTenantStrategy - from sqlspec.utils.tenant import TenantContextPayload + from sqlspec.utils.tenant._context import TenantContextPayload __all__ = ( "CockroachAsyncpgRLSStrategy", diff --git a/sqlspec/adapters/cockroach_psycopg/config.py b/sqlspec/adapters/cockroach_psycopg/config.py index a66521c83..ae8bb660c 100644 --- a/sqlspec/adapters/cockroach_psycopg/config.py +++ b/sqlspec/adapters/cockroach_psycopg/config.py @@ -64,7 +64,7 @@ from sqlspec.utils.tenant import TenantContextPayload if TYPE_CHECKING: - from collections.abc import AsyncIterator, Awaitable, Callable, Iterator + from collections.abc import AsyncGenerator, Awaitable, Callable, Generator from contextlib import AbstractAsyncContextManager, AbstractContextManager from types import TracebackType from uuid import UUID @@ -525,12 +525,14 @@ def provide_session( return CockroachPsycopgSyncSessionContext( acquire_connection=handler.acquire_connection, release_connection=handler.release_connection, - statement_config=statement_config - or (lambda: resolve_runtime_statement_config(None, self.statement_config, default_statement_config)), + statement_config=statement_config or self._default_runtime_statement_config, driver_features=driver_features, prepare_driver=self._prepare_driver, ) + def _default_runtime_statement_config(self) -> "StatementConfig": + return resolve_runtime_statement_config(None, self.statement_config, default_statement_config) + def provide_replica_session( self, *args: Any, @@ -610,7 +612,9 @@ def _build_replica_spec_overrides(self, spec_input: str | bool | Mapping[str, An return super()._build_replica_spec_overrides(spec_input) @contextmanager - def _replica_connection(self, context: "AbstractContextManager[CockroachPsycopgSyncDriver]") -> "Iterator[Any]": + def _replica_connection( + self, context: "AbstractContextManager[CockroachPsycopgSyncDriver]" + ) -> "Generator[Any, None, None]": with context as session: session.begin() failed = False @@ -945,12 +949,14 @@ def provide_session( return CockroachPsycopgAsyncSessionContext( acquire_connection=handler.acquire_connection, release_connection=handler.release_connection, - statement_config=statement_config - or (lambda: resolve_runtime_statement_config(None, self.statement_config, default_statement_config)), + statement_config=statement_config or self._default_runtime_statement_config, driver_features=driver_features, prepare_driver=self._prepare_driver, ) + def _default_runtime_statement_config(self) -> "StatementConfig": + return resolve_runtime_statement_config(None, self.statement_config, default_statement_config) + def provide_replica_session( self, *args: Any, @@ -1032,7 +1038,7 @@ def _build_replica_spec_overrides(self, spec_input: str | bool | Mapping[str, An @asynccontextmanager async def _replica_connection( self, context: "AbstractAsyncContextManager[CockroachPsycopgAsyncDriver]" - ) -> "AsyncIterator[Any]": + ) -> "AsyncGenerator[Any, None]": async with context as session: await session.begin() failed = False diff --git a/sqlspec/adapters/cockroach_psycopg/core.py b/sqlspec/adapters/cockroach_psycopg/core.py index 4a6bbf1b9..355baa41d 100644 --- a/sqlspec/adapters/cockroach_psycopg/core.py +++ b/sqlspec/adapters/cockroach_psycopg/core.py @@ -17,7 +17,8 @@ ) from sqlspec.adapters.cockroach_psycopg._typing import cockroach_psycopg_errors as pg_errors from sqlspec.adapters.cockroach_psycopg._typing import cockroach_psycopg_module as psycopg -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.exceptions import ( CheckViolationError, ConnectionTimeoutError, @@ -44,7 +45,7 @@ from sqlspec.typing import import_optional_attr from sqlspec.utils.config_tools import normalize_connection_config from sqlspec.utils.routing import RoutingContext -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import quote_identifier, split_qualified_identifier from sqlspec.utils.type_converters import build_json_list_converter, build_json_tuple_converter, build_uuid_coercions from sqlspec.utils.type_guards import has_sqlstate @@ -53,10 +54,11 @@ from collections.abc import Callable, Mapping from sqlspec.adapters.cockroach_psycopg.config import CockroachPsycopgPoolConfig - from sqlspec.core import ParameterStyleConfig from sqlspec.core.cache import CachedQuery + from sqlspec.core.parameters._types import ParameterStyleConfig from sqlspec.core.result._base import SQLResult - from sqlspec.storage import StorageTelemetry + from sqlspec.core.statement import StatementConfig + from sqlspec.storage.pipeline import StorageTelemetry __all__ = ( "DEFAULT_FOLLOWER_READ_STALENESS", diff --git a/sqlspec/adapters/cockroach_psycopg/driver.py b/sqlspec/adapters/cockroach_psycopg/driver.py index fb0880a8e..b7c0c94ac 100644 --- a/sqlspec/adapters/cockroach_psycopg/driver.py +++ b/sqlspec/adapters/cockroach_psycopg/driver.py @@ -6,6 +6,8 @@ from contextlib import suppress from typing import TYPE_CHECKING, Any, TypeVar, cast +from mypy_extensions import mypyc_attr + from sqlspec.adapters.cockroach_psycopg._typing import ( CockroachAsyncConnection, CockroachPsycopgAsyncSessionContext, @@ -40,20 +42,25 @@ PsycopgSyncDriver, PsycopgSyncExceptionHandler, ) -from sqlspec.core import SQL, StatementConfig, get_cache_config, register_driver_profile -from sqlspec.driver import AsyncRowStream, SyncRowStream -from sqlspec.driver._common import cache_hit_hooks_unchanged -from sqlspec.exceptions import ReadOnlyTransactionError +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.statement import SQL, StatementConfig +from sqlspec.driver._common import cache_hit_hooks_unchanged, validate_load_from_storage_kwargs +from sqlspec.driver._stream import AsyncRowStream, SyncRowStream +from sqlspec.exceptions import ReadOnlyTransactionError, with_cause from sqlspec.utils.logging import get_logger if TYPE_CHECKING: from collections.abc import Awaitable, Callable + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.cockroach_psycopg._typing import CockroachAsyncCursor, CockroachSyncCursor from sqlspec.adapters.psycopg.core import PsycopgAsyncStreamSource, PsycopgSyncStreamSource - from sqlspec.core import SQLResult - from sqlspec.driver import CachedQuery, ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.core.cache import CachedQuery + from sqlspec.core.result._base import SQLResult + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry __all__ = ( "CockroachPsycopgAsyncDriver", @@ -139,52 +146,58 @@ async def start(self) -> None: self._connection = self._driver.connection if self._driver._wants_follower_read(self._statement): self._auto_started = await self._driver._begin_follower_read_transaction() + start_error: BaseException | None = None try: await self._underlying.start() - except BaseException: + except BaseException as exc: + start_error = exc + if start_error is not None: if self._auto_started and self._driver._connection_in_transaction(): with suppress(Exception): await self._driver.rollback() - raise + raise start_error async def fetch_chunk(self) -> "list[dict[str, Any]]": return await self._underlying.fetch_chunk() async def close(self, error: bool = False) -> None: + close_error: BaseException | None = None try: await self._underlying.close(error=error) - finally: - connection = self._connection - self._connection = None - if ( - self._auto_started - and self._driver.connection is connection - and self._driver._connection_in_transaction() - ): - if error: - with suppress(Exception): - await self._driver.rollback() - else: - await self._driver.commit() + except BaseException as exc: + close_error = exc + connection = self._connection + self._connection = None + if self._auto_started and self._driver.connection is connection and self._driver._connection_in_transaction(): + if error: + with suppress(Exception): + await self._driver.rollback() + else: + await self._driver.commit() + if close_error is not None: + raise close_error +@mypyc_attr(allow_interpreted_subclasses=True) class CockroachPsycopgSyncExceptionHandler(PsycopgSyncExceptionHandler): """Context manager for handling CockroachDB psycopg exceptions.""" __slots__ = () +@mypyc_attr(allow_interpreted_subclasses=True) class CockroachPsycopgAsyncExceptionHandler(PsycopgAsyncExceptionHandler): """Async context manager for handling CockroachDB psycopg exceptions.""" __slots__ = () +@mypyc_attr(allow_interpreted_subclasses=True) class CockroachPsycopgSyncDriver(PsycopgSyncDriver): """CockroachDB sync driver using psycopg.crdb.""" __slots__ = ("_enable_retry", "_follower_reads_enabled", "_follower_staleness", "_retry_config") - dialect = "postgres" + dialect: "DialectType | None" = "postgres" def __init__( self, @@ -314,8 +327,27 @@ def load_from_storage( produced = native_import_telemetry(rows, table, resolved.protocol, file_format) self._attach_partition_telemetry(produced, partitioner) return self._storage_job(produced) + validate_load_from_storage_kwargs(kwargs) return super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs + table, + source, + file_format=file_format, + partitioner=partitioner, + overwrite=overwrite, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) def dispatch_special_handling(self, cursor: Any, statement: SQL) -> "SQLResult | None": @@ -340,7 +372,7 @@ def begin(self) -> None: with suppress(Exception): self.rollback() self._transaction_active = False - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def run_transaction_with_retry(self, operation: "Callable[[], _T]") -> _T: """Execute a full CockroachDB transaction callback with serialization retries. @@ -457,7 +489,7 @@ def _fetch() -> list[dict[str, Any]]: with self.with_cursor(self.connection) as cursor: cursor.row_factory = dict_row cursor.execute(command.encode("utf-8"), parameters) - return cursor.fetchall() + return cursor.fetchall() or [] if sys.exc_info()[1] is None: with handler: @@ -492,11 +524,12 @@ def _begin_follower_read_transaction(self) -> bool: return True +@mypyc_attr(allow_interpreted_subclasses=True) class CockroachPsycopgAsyncDriver(PsycopgAsyncDriver): """CockroachDB async driver using psycopg.crdb.""" __slots__ = ("_enable_retry", "_follower_reads_enabled", "_follower_staleness", "_retry_config") - dialect = "postgres" + dialect: "DialectType | None" = "postgres" def __init__( self, @@ -626,8 +659,27 @@ async def load_from_storage( produced = native_import_telemetry(rows, table, resolved.protocol, file_format) self._attach_partition_telemetry(produced, partitioner) return self._storage_job(produced) + validate_load_from_storage_kwargs(kwargs) return await super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs + table, + source, + file_format=file_format, + partitioner=partitioner, + overwrite=overwrite, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) async def dispatch_special_handling(self, cursor: Any, statement: SQL) -> "SQLResult | None": @@ -646,13 +698,16 @@ async def begin(self) -> None: await super().begin() return await super().begin() + begin_error: Exception | None = None try: await self._apply_follower_reads() except Exception as exc: + begin_error = exc + if begin_error is not None: with suppress(Exception): await self.rollback() self._transaction_active = False - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(begin_error), begin_error) from begin_error async def run_transaction_with_retry(self, operation: "Callable[[], Awaitable[_T]]") -> _T: """Execute a full CockroachDB transaction callback with serialization retries. @@ -667,19 +722,21 @@ async def run_transaction_with_retry(self, operation: "Callable[[], Awaitable[_T attempt = 0 while True: + tx_error: Exception | None = None try: await self.begin() result = await operation() await self.commit() except Exception as exc: - try: - await self.rollback() - except Exception: - raise exc - if not is_retryable_error(exc) or attempt >= self._retry_config.max_retries: - raise + tx_error = exc else: return result + try: + await self.rollback() + except Exception: + raise tx_error + if not is_retryable_error(tx_error) or attempt >= self._retry_config.max_retries: + raise tx_error delay = calculate_backoff_seconds(attempt, self._retry_config) if self._retry_config.enable_logging: logger.debug("CockroachDB retry %s/%s after %.3fs", attempt + 1, self._retry_config.max_retries, delay) @@ -688,15 +745,20 @@ async def run_transaction_with_retry(self, operation: "Callable[[], Awaitable[_T async def dispatch_execute(self, cursor: "CockroachAsyncCursor", statement: SQL) -> "ExecutionResult": auto_started = await self._begin_follower_read_transaction() if self._wants_follower_read(statement) else False + exec_error: BaseException | None = None + result: ExecutionResult | None = None try: result = await super().dispatch_execute(cursor, statement) - except BaseException: + except BaseException as exc: + exec_error = exc + if exec_error is not None: if auto_started and self._connection_in_transaction(): with suppress(Exception): await self.rollback() - raise + raise exec_error if auto_started and self._connection_in_transaction(): await self.commit() + assert result is not None return result def dispatch_select_stream(self, statement: "SQL", chunk_size: int) -> "AsyncRowStream[dict[str, Any]] | None": @@ -732,15 +794,20 @@ async def _execute_cache_hit( ) else False ) + exec_error: BaseException | None = None + result: SQLResult | None = None try: result = await super()._execute_cache_hit(sql, params, cached) - except BaseException: + except BaseException as exc: + exec_error = exc + if exec_error is not None: if auto_started and self._connection_in_transaction(): with suppress(Exception): await self.rollback() - raise + raise exec_error if auto_started and self._connection_in_transaction(): await self.commit() + assert result is not None return result def _native_storage_ready(self) -> bool: @@ -758,7 +825,7 @@ async def _fetch() -> list[dict[str, Any]]: async with self.with_cursor(self.connection) as cursor: cursor.row_factory = dict_row await cursor.execute(command.encode("utf-8"), parameters) - return await cursor.fetchall() + return (await cursor.fetchall()) or [] if sys.exc_info()[1] is None: async with handler: diff --git a/sqlspec/adapters/cockroach_psycopg/tenant.py b/sqlspec/adapters/cockroach_psycopg/tenant.py index cdada7e78..9c373dc88 100644 --- a/sqlspec/adapters/cockroach_psycopg/tenant.py +++ b/sqlspec/adapters/cockroach_psycopg/tenant.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: from sqlspec.adapters.cockroach_psycopg._typing import CockroachAsyncConnection, CockroachSyncConnection from sqlspec.core.tenant.base import AsyncTenantStrategy, SyncTenantStrategy - from sqlspec.utils.tenant import TenantContextPayload + from sqlspec.utils.tenant._context import TenantContextPayload __all__ = ( "CockroachPsycopgAsyncRLSStrategy", diff --git a/sqlspec/adapters/db2/core.py b/sqlspec/adapters/db2/core.py index 04162c9eb..3e59a0018 100644 --- a/sqlspec/adapters/db2/core.py +++ b/sqlspec/adapters/db2/core.py @@ -8,8 +8,10 @@ from urllib.parse import parse_qsl, unquote, urlsplit from sqlspec.adapters.db2._typing import Db2Error -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult +from sqlspec.core.statement import StatementConfig from sqlspec.exceptions import ( CheckViolationError, DatabaseConnectionError, @@ -27,7 +29,7 @@ ) from sqlspec.utils.config_tools import parse_odbc_connection_string from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import normalize_identifier, quote_identifier from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_rowcount, resolve_row_format @@ -243,7 +245,9 @@ def create_mapped_exception(error: BaseException, *, logger: "Logger | None" = N if logger is not None and (sqlstate or sqlcode): logger.debug("Unmapped Db2 SQLSTATE: %s, SQLCODE: %s", sqlstate, sqlcode) - return SQLSpecError(f"Db2 database error. Original error: {error}") + mapped_error = SQLSpecError(f"Db2 database error. Original error: {error}") + mapped_error.__cause__ = error + return mapped_error def utc_now() -> datetime: @@ -383,6 +387,7 @@ def execute_cache_hit( cached: "CachedQuery", statement_config: "StatementConfig", state: "tuple[dict[int, tuple[Any, list[str]]], bool]", + wire_sql: "str | None" = None, ) -> "SQLResult": """Execute a cached statement on an ``ibm_db_dbi`` connection and build its result. @@ -393,6 +398,7 @@ def execute_cache_hit( cached: Cached compilation of ``sql``. statement_config: Statement configuration the result uses to rebuild its statement. state: Session-local column name cache and whether uppercase column names are lowercased. + wire_sql: Optional wire SQL override with sqlcommenter formatting applied. Returns: The statement result. @@ -401,7 +407,7 @@ def execute_cache_hit( SQLSpecError: Mapped from any ``ibm_db_dbi`` error the statement raises. """ parameters = normalize_execute_parameters(params) or () - compiled_sql = cached.compiled_sql + compiled_sql = cached.compiled_sql if wire_sql is None else wire_sql try: cursor = connection.cursor() try: @@ -439,6 +445,7 @@ async def execute_cache_hit_async( cached: "CachedQuery", statement_config: "StatementConfig", state: "tuple[dict[int, tuple[Any, list[str]]], bool]", + wire_sql: "str | None" = None, ) -> "SQLResult": """Execute a cached statement on an ``ibm_db_dbi`` async connection and build its result. @@ -449,6 +456,7 @@ async def execute_cache_hit_async( cached: Cached compilation of ``sql``. statement_config: Statement configuration the result uses to rebuild its statement. state: Session-local column name cache and whether uppercase column names are lowercased. + wire_sql: Optional wire SQL override with sqlcommenter formatting applied. Returns: The statement result. @@ -457,7 +465,7 @@ async def execute_cache_hit_async( SQLSpecError: Mapped from any ``ibm_db_dbi`` error the statement raises. """ parameters = normalize_execute_parameters(params) or () - compiled_sql = cached.compiled_sql + compiled_sql = cached.compiled_sql if wire_sql is None else wire_sql try: cursor = await connection.cursor() error: BaseException | None = None @@ -751,8 +759,11 @@ def _build_mapped_exception( SQLSpecError: The mapped exception. """ if sqlstate is not None: - return error_cls(f"Db2 SQLSTATE {sqlstate}: {description}. Original error: {error}") - return error_cls(f"Db2 SQLCODE {sqlcode}: {description}. Original error: {error}") + mapped = error_cls(f"Db2 SQLSTATE {sqlstate}: {description}. Original error: {error}") + else: + mapped = error_cls(f"Db2 SQLCODE {sqlcode}: {description}. Original error: {error}") + mapped.__cause__ = error + return mapped def _normalize_column_names(column_names: "list[str]", lowercase: bool) -> "list[str]": diff --git a/sqlspec/adapters/db2/driver.py b/sqlspec/adapters/db2/driver.py index 378cc753a..c9f7ffc48 100644 --- a/sqlspec/adapters/db2/driver.py +++ b/sqlspec/adapters/db2/driver.py @@ -30,30 +30,29 @@ resolve_rowcount, ) from sqlspec.adapters.db2.data_dictionary import Db2AsyncDataDictionary, Db2SyncDataDictionary -from sqlspec.core import SQL, StatementConfig, get_cache_config, register_driver_profile -from sqlspec.driver import ( - AsyncDriverAdapterBase, - AsyncRowStream, - BaseAsyncExceptionHandler, - BaseSyncExceptionHandler, - ExecutionResult, - SyncDriverAdapterBase, - SyncRowStream, - rows_to_dicts, - validate_savepoint_name, -) -from sqlspec.driver._common import cache_hit_hooks_unchanged +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.driver._async import AsyncDriverAdapterBase +from sqlspec.driver._common import ExecutionResult, cache_hit_hooks_unchanged, validate_savepoint_name +from sqlspec.driver._exception_handler import BaseAsyncExceptionHandler, BaseSyncExceptionHandler +from sqlspec.driver._stream import AsyncRowStream, SyncRowStream, rows_to_dicts +from sqlspec.driver._sync import SyncDriverAdapterBase from sqlspec.exceptions import MissingDependencyError, SQLSpecError from sqlspec.utils.logging import get_logger, log_with_context +from sqlspec.utils.sync_tools import exit_async_context from sqlspec.utils.text import normalize_identifier, quote_identifier if TYPE_CHECKING: from collections.abc import Sequence - from sqlspec.builder import QueryBuilder - from sqlspec.core import SQLResult, Statement, StatementFilter + from sqlglot.dialects.dialect import DialectType + + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import SQLResult + from sqlspec.core.statement import SQL, Statement, StatementConfig from sqlspec.driver._common import CachedQuery - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry from sqlspec.typing import StatementParameters __all__ = ( @@ -70,7 +69,7 @@ logger = get_logger("sqlspec.adapters.db2") -@mypyc_attr(allow_interpreted_subclasses=False) +@mypyc_attr(allow_interpreted_subclasses=True) class Db2SyncExceptionHandler(BaseSyncExceptionHandler): """Context manager that maps ``ibm_db_dbi`` errors to SQLSpec exceptions. @@ -152,7 +151,7 @@ def close(self, error: bool = False) -> None: cursor_manager.__exit__(None, None, None) -@mypyc_attr(allow_interpreted_subclasses=False) +@mypyc_attr(allow_interpreted_subclasses=True) class Db2SyncDriver(SyncDriverAdapterBase): """IBM Db2 database driver.""" @@ -164,7 +163,7 @@ class Db2SyncDriver(SyncDriverAdapterBase): "_restore_autocommit", "_transaction_active", ) - dialect = "db2" + dialect: "DialectType | None" = "db2" def __init__( self, @@ -275,7 +274,9 @@ def begin(self) -> None: self.connection.set_autocommit(False) except Db2Error as exc: msg = f"Failed to begin Db2 transaction: {exc}" - raise SQLSpecError(msg) from exc + error = SQLSpecError(msg) + error.__cause__ = exc + raise error from exc self._restore_autocommit = enabled self._transaction_active = True @@ -289,7 +290,9 @@ def commit(self) -> None: self.connection.commit() except Db2Error as exc: msg = f"Failed to commit Db2 transaction: {exc}" - raise SQLSpecError(msg) from exc + error = SQLSpecError(msg) + error.__cause__ = exc + raise error from exc self._transaction_active = False self._restore_connection_autocommit() @@ -303,7 +306,9 @@ def rollback(self) -> None: self.connection.rollback() except Db2Error as exc: msg = f"Failed to rollback Db2 transaction: {exc}" - raise SQLSpecError(msg) from exc + error = SQLSpecError(msg) + error.__cause__ = exc + raise error from exc finally: self._transaction_active = False self._restore_connection_autocommit() @@ -408,7 +413,10 @@ def _execute_cache_hit( self, sql: str, params: "tuple[Any, ...] | list[Any] | dict[str, Any]", cached: "CachedQuery" ) -> "SQLResult": """Execute a cached statement through the compiled Db2 cache-hit executor.""" - return execute_cache_hit(self.connection, sql, params, cached, self.statement_config, self._cache_hit_state) + wire_sql = self._format_wire_sql(cached.compiled_sql) if self.statement_config.enable_sqlcommenter else None + return execute_cache_hit( + self.connection, sql, params, cached, self.statement_config, self._cache_hit_state, wire_sql + ) def _release_open_work(self, *, autocommit_baseline: bool) -> None: """Roll back work left open before the connection is returned to its pool. @@ -442,7 +450,9 @@ def _restore_connection_autocommit(self) -> None: self.connection.set_autocommit(True) except Db2Error as exc: msg = f"Failed to restore Db2 autocommit: {exc}" - raise SQLSpecError(msg) from exc + error = SQLSpecError(msg) + error.__cause__ = exc + raise error from exc self._restore_autocommit = False def _connection_in_transaction(self) -> bool: @@ -457,7 +467,7 @@ def _connection_in_transaction(self) -> bool: return False -@mypyc_attr(allow_interpreted_subclasses=False) +@mypyc_attr(allow_interpreted_subclasses=True) class Db2AsyncExceptionHandler(BaseAsyncExceptionHandler): """Async context manager that maps ``ibm_db_dbi`` errors to SQLSpec exceptions. @@ -491,6 +501,7 @@ def __init__(self, driver: "Db2AsyncDriver", sql: str, parameters: Any, chunk_si async def start(self) -> None: cursor_manager = self._driver.with_cursor(self._driver.connection) + start_error: BaseException | None = None try: cursor = await cursor_manager.__aenter__() handler = self._driver.handle_database_exceptions() @@ -498,10 +509,12 @@ async def start(self) -> None: handler, cursor.execute, self._sql, normalize_execute_parameters(self._parameters) ) self._driver._check_pending_exception(handler) - except BaseException: + except BaseException as exc: + start_error = exc + if start_error is not None: with contextlib.suppress(Exception): await cursor_manager.__aexit__(None, None, None) - raise + raise start_error self._cursor_manager = cursor_manager async def fetch_chunk(self) -> "list[dict[str, Any]]": @@ -530,7 +543,7 @@ async def close(self, error: bool = False) -> None: await cursor_manager.__aexit__(None, None, None) -@mypyc_attr(allow_interpreted_subclasses=False) +@mypyc_attr(allow_interpreted_subclasses=True) class Db2AsyncDriver(AsyncDriverAdapterBase): """IBM Db2 async database driver over ``ibm_db_dbi.AsyncConnection``.""" @@ -542,7 +555,7 @@ class Db2AsyncDriver(AsyncDriverAdapterBase): "_restore_autocommit", "_transaction_active", ) - dialect = "db2" + dialect: "DialectType | None" = "db2" def __init__( self, @@ -655,7 +668,9 @@ async def begin(self) -> None: await self.connection.set_autocommit(False) except Db2Error as exc: msg = f"Failed to begin Db2 transaction: {exc}" - raise SQLSpecError(msg) from exc + error = SQLSpecError(msg) + error.__cause__ = exc + raise error from exc self._restore_autocommit = enabled self._transaction_active = True @@ -669,7 +684,9 @@ async def commit(self) -> None: await self.connection.commit() except Db2Error as exc: msg = f"Failed to commit Db2 transaction: {exc}" - raise SQLSpecError(msg) from exc + error = SQLSpecError(msg) + error.__cause__ = exc + raise error from exc self._transaction_active = False await self._restore_connection_autocommit() @@ -679,14 +696,18 @@ async def rollback(self) -> None: Raises: SQLSpecError: When the driver reports an error. """ + rollback_error: Db2Error | None = None try: await self.connection.rollback() except Db2Error as exc: - msg = f"Failed to rollback Db2 transaction: {exc}" - raise SQLSpecError(msg) from exc - finally: - self._transaction_active = False - await self._restore_connection_autocommit() + rollback_error = exc + self._transaction_active = False + await self._restore_connection_autocommit() + if rollback_error is not None: + msg = f"Failed to rollback Db2 transaction: {rollback_error}" + error = SQLSpecError(msg) + error.__cause__ = rollback_error + raise error from rollback_error def with_cursor(self, connection: Any) -> "Db2AsyncCursor": return Db2AsyncCursor(connection) @@ -735,7 +756,10 @@ async def set_migration_session_schema(self, schema: str) -> None: Args: schema: Schema to make current. Unquoted all-lowercase names fold to uppercase. """ - async with self.with_cursor(self.connection) as cursor: + cursor_context = self.with_cursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: handler = self.handle_database_exceptions() if self._migration_schema_restore is None: await self._run_with_exception_handler(handler, cursor.execute, "VALUES CURRENT SCHEMA") @@ -744,6 +768,9 @@ async def set_migration_session_schema(self, schema: str) -> None: self._migration_schema_restore = str(row[0]) await self._run_with_exception_handler(handler, cursor.execute, build_set_schema_sql(schema)) self._check_pending_exception(handler) + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) async def reset_migration_session_schema(self) -> None: """Restore the current schema captured by ``set_migration_session_schema``. @@ -756,12 +783,18 @@ async def reset_migration_session_schema(self) -> None: if previous_schema is None: return self._migration_schema_restore = None - async with self.with_cursor(self.connection) as cursor: + cursor_context = self.with_cursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: handler = self.handle_database_exceptions() await self._run_with_exception_handler( handler, cursor.execute, build_set_schema_sql(quote_identifier(previous_schema)) ) self._check_pending_exception(handler) + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) async def has_schema(self, schema: str) -> bool: """Return whether the schema exists in the catalog. @@ -772,7 +805,11 @@ async def has_schema(self, schema: str) -> bool: Returns: True when ``SYSCAT.SCHEMATA`` lists the schema. """ - async with self.with_cursor(self.connection) as cursor: + cursor_context = self.with_cursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + schema_exists = False + try: handler = self.handle_database_exceptions() await self._run_with_exception_handler( handler, @@ -782,7 +819,11 @@ async def has_schema(self, schema: str) -> bool: ) self._check_pending_exception(handler) row = await cursor.fetchone() - return row is not None + schema_exists = row is not None + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) + return schema_exists def collect_rows(self, cursor: Any, fetched: "list[Any]") -> "tuple[list[Any], list[str], int]": column_names = resolve_column_names( @@ -797,8 +838,9 @@ async def _execute_cache_hit( self, sql: str, params: "tuple[Any, ...] | list[Any] | dict[str, Any]", cached: "CachedQuery" ) -> "SQLResult": """Execute a cached statement through the compiled Db2 async cache-hit executor.""" + wire_sql = self._format_wire_sql(cached.compiled_sql) if self.statement_config.enable_sqlcommenter else None return await execute_cache_hit_async( - self.connection, sql, params, cached, self.statement_config, self._cache_hit_state + self.connection, sql, params, cached, self.statement_config, self._cache_hit_state, wire_sql ) async def _release_open_work(self, *, autocommit_baseline: bool) -> None: @@ -814,10 +856,13 @@ async def _release_open_work(self, *, autocommit_baseline: bool) -> None: """ if not self._transaction_active and autocommit_baseline: return + release_error: SQLSpecError | None = None try: await self.rollback() except SQLSpecError as exc: - log_with_context(logger, logging.DEBUG, "db2.session.rollback_failed", error=str(exc)) + release_error = exc + if release_error is not None: + log_with_context(logger, logging.DEBUG, "db2.session.rollback_failed", error=str(release_error)) with contextlib.suppress(Exception): await self.connection.close() @@ -833,7 +878,9 @@ async def _restore_connection_autocommit(self) -> None: await self.connection.set_autocommit(True) except Db2Error as exc: msg = f"Failed to restore Db2 autocommit: {exc}" - raise SQLSpecError(msg) from exc + error = SQLSpecError(msg) + error.__cause__ = exc + raise error from exc self._restore_autocommit = False def _connection_in_transaction(self) -> bool: diff --git a/sqlspec/adapters/db2/pool.py b/sqlspec/adapters/db2/pool.py index 42cf4576b..26b013d41 100644 --- a/sqlspec/adapters/db2/pool.py +++ b/sqlspec/adapters/db2/pool.py @@ -15,7 +15,7 @@ from sqlspec.adapters.db2 import _typing as typing_module from sqlspec.adapters.db2._typing import Db2Error, ibm_db_dbi from sqlspec.adapters.db2.core import build_dsn_string, create_mapped_exception -from sqlspec.exceptions import ConnectionTimeoutError, DatabaseConnectionError +from sqlspec.exceptions import ConnectionTimeoutError, DatabaseConnectionError, with_cause from sqlspec.utils.logging import POOL_LOGGER_NAME, get_logger, log_with_context from sqlspec.utils.uuids import uuid4 @@ -176,7 +176,7 @@ def new_connection(self) -> Any: try: connection = typing_module.ibm_db_dbi.connect(self._dsn, "", "", "", "", options) except Db2Error as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc try: schema = self._connection_parameters.get("current_schema") @@ -188,7 +188,7 @@ def new_connection(self) -> Any: with contextlib.suppress(BaseException): connection.close() if isinstance(exc, Db2Error): - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc raise return connection @@ -237,7 +237,7 @@ def acquire(self) -> Any: try: connection = self._get_thread_connection() except Db2Error as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc self._current_holder().depth += 1 return connection @@ -642,7 +642,7 @@ async def new_connection(self) -> Any: try: connection = await typing_module.ibm_db_dbi.AsyncConnection.connect(self._dsn, "", "", "", "", options) except Db2Error as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc setup_error: BaseException | None = None try: schema = self._connection_parameters.get("current_schema") @@ -658,7 +658,7 @@ async def new_connection(self) -> Any: with contextlib.suppress(BaseException): await connection.close() if isinstance(setup_error, Db2Error): - raise create_mapped_exception(setup_error) from setup_error + raise with_cause(create_mapped_exception(setup_error), setup_error) from setup_error raise setup_error return connection @@ -680,7 +680,9 @@ async def acquire(self) -> Any: await asyncio.wait_for(semaphore.acquire(), self._acquire_timeout) except TimeoutError as exc: msg = f"Timed out after {self._acquire_timeout}s waiting for a Db2 connection" - raise ConnectionTimeoutError(msg) from exc + error = ConnectionTimeoutError(msg) + error.__cause__ = exc + raise error from exc if self._closed: semaphore.release() msg = "Db2 async connection pool is closed" @@ -690,7 +692,7 @@ async def acquire(self) -> Any: except BaseException as exc: semaphore.release() if isinstance(exc, Db2Error): - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc raise if self._closed: try: diff --git a/sqlspec/adapters/db2/tenant.py b/sqlspec/adapters/db2/tenant.py index 1da65378f..854f55dad 100644 --- a/sqlspec/adapters/db2/tenant.py +++ b/sqlspec/adapters/db2/tenant.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: from sqlspec.core.tenant.base import AsyncTenantStrategy, SyncTenantStrategy - from sqlspec.utils.tenant import TenantContextPayload + from sqlspec.utils.tenant._context import TenantContextPayload __all__ = ( "DEFAULT_DB2_TENANT_VARIABLE", diff --git a/sqlspec/adapters/duckdb/core.py b/sqlspec/adapters/duckdb/core.py index 23213a49a..78c4dd0e6 100644 --- a/sqlspec/adapters/duckdb/core.py +++ b/sqlspec/adapters/duckdb/core.py @@ -38,8 +38,10 @@ DuckDBTypeMismatchException, duckdb_module, ) -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult +from sqlspec.core.statement import StatementConfig from sqlspec.exceptions import ( CheckViolationError, DatabaseConnectionError, @@ -57,13 +59,14 @@ SQLParsingError, SQLSpecError, UniqueViolationError, + with_cause, ) -from sqlspec.storage import ResolvedStorageTarget from sqlspec.storage.backends.fsspec import FSSpecBackend from sqlspec.storage.backends.obstore import ObStoreBackend +from sqlspec.storage.pipeline import ResolvedStorageTarget from sqlspec.typing import PYARROW_INSTALLED from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import quote_identifier, split_qualified_identifier from sqlspec.utils.type_converters import build_decimal_converter, build_uuid_coercions, time_iso_convert from sqlspec.utils.type_guards import has_rowcount @@ -74,7 +77,7 @@ from sqlspec.core.cache import CachedQuery, StatementCacheHolder from sqlspec.core.result._base import RowFormat - from sqlspec.storage import StorageDestination, SyncStoragePipeline + from sqlspec.storage.pipeline import StorageDestination, SyncStoragePipeline __all__ = ( @@ -866,7 +869,7 @@ def _build_storage_read_sql(table: str, uri: str, file_format: str, *, allow_glo targets = Dialect.get_or_raise("duckdb").parse_into(exp.Table, table) except ParseError as exc: msg = "Native storage import requires one qualified table identifier." - raise ValueError(msg) from exc + raise with_cause(ValueError(msg), exc) from exc if ( len(targets) != 1 or not isinstance(targets[0], exp.Table) diff --git a/sqlspec/adapters/duckdb/driver.py b/sqlspec/adapters/duckdb/driver.py index 7bf804731..877b583b9 100644 --- a/sqlspec/adapters/duckdb/driver.py +++ b/sqlspec/adapters/duckdb/driver.py @@ -32,19 +32,17 @@ resolve_rowcount, ) from sqlspec.adapters.duckdb.data_dictionary import DuckDBDataDictionary -from sqlspec.core import ( - SQL, - ParameterStyle, - StatementConfig, - build_arrow_result_from_reader, - build_arrow_result_from_table, - get_cache_config, - register_driver_profile, -) -from sqlspec.core.result import DMLResult -from sqlspec.driver import BaseSyncExceptionHandler, SyncDriverAdapterBase, SyncRowStream -from sqlspec.driver._common import cache_hit_hooks_unchanged -from sqlspec.exceptions import SQLSpecError +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.parameters._types import ParameterStyle +from sqlspec.core.result._base import DMLResult, build_arrow_result_from_reader, build_arrow_result_from_table +from sqlspec.core.statement import SQL, StatementConfig +from sqlspec.driver._common import cache_hit_hooks_unchanged, validate_load_from_storage_kwargs +from sqlspec.driver._exception_handler import BaseSyncExceptionHandler +from sqlspec.driver._stream import SyncRowStream +from sqlspec.driver._sync import SyncDriverAdapterBase +from sqlspec.exceptions import SQLSpecError, with_cause +from sqlspec.storage._utils import import_pyarrow from sqlspec.typing import PYARROW_INSTALLED, ArrowReturnFormat, StatementParameters from sqlspec.utils.arrow_helpers import arrow_reader_with_deferred_close from sqlspec.utils.logging import get_logger @@ -56,19 +54,23 @@ from collections.abc import Callable, Mapping, Sequence from pathlib import Path + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.duckdb._typing import DuckDBConnection - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, CachedQuery, SQLResult, Statement, StatementFilter - from sqlspec.core.result._base import RowFormat - from sqlspec.driver import ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.cache import CachedQuery + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult, RowFormat, SQLResult + from sqlspec.core.statement import Statement + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry __all__ = ("DuckDBCursor", "DuckDBDriver", "DuckDBExceptionHandler", "DuckDBSessionContext") logger = get_logger("sqlspec.adapters.duckdb") -@mypyc_attr(allow_interpreted_subclasses=False) +@mypyc_attr(allow_interpreted_subclasses=True) class DuckDBExceptionHandler(BaseSyncExceptionHandler): """Context manager for handling DuckDB database exceptions. @@ -87,6 +89,7 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False +@mypyc_attr(allow_interpreted_subclasses=True) class DuckDBDriver(SyncDriverAdapterBase): """Synchronous DuckDB database driver. @@ -99,7 +102,7 @@ class DuckDBDriver(SyncDriverAdapterBase): """ __slots__ = ("_created_profile_path", "_data_dictionary", "_migration_schema_restore", "_transaction_active") - dialect = "duckdb" + dialect: "DialectType | None" = "duckdb" native_arrow_export: ClassVar[bool] = True def __init__( @@ -230,7 +233,11 @@ def dispatch_execute_many(self, cursor: "DuckDBConnection", statement: SQL) -> " special_data={"dml_result": bulk_result}, is_many_result=True, ) - parameter_sets = cast("list[Any]", prepared_parameters) + parameter_sets = ( + prepared_parameters + if isinstance(prepared_parameters, list) + else list(cast("Sequence[Any]", prepared_parameters)) + ) cursor.executemany(sql, parameter_sets) row_count = len(parameter_sets) if statement.is_modifying_operation() else resolve_rowcount(cursor) @@ -654,13 +661,33 @@ def load_from_arrow( is_reader = hasattr(source_data, "read_next_batch") if is_reader: arrow_source = source_data + source_schema = getattr(source_data, "schema", None) + source_columns: list[str] | None = list(source_schema.names) if source_schema is not None else None else: arrow_table = self._coerce_arrow_table(source_data) arrow_source = arrow_table + source_columns = list(arrow_table.column_names) table_name = format_identifier(table) temp_view = f"_sqlspec_arrow_{uuid4().hex}" inserted_rows = 0 manage_transaction = overwrite and not self._connection_in_transaction() + columns_sql = ", ".join(format_identifier(col) for col in source_columns) if source_columns else None + if columns_sql and source_columns: + try: + table_info_rows = self.connection.execute(f"PRAGMA table_info({table_name})").fetchall() + except Exception: + table_info_rows = [] + if table_info_rows: + target_columns = {str(row[1]).lower() for row in table_info_rows if len(row) > 1} + if len(table_info_rows) == len(source_columns) and any( + col.lower() not in target_columns for col in source_columns + ): + columns_sql = None + insert_sql = ( + f"INSERT INTO {table_name} ({columns_sql}) SELECT {columns_sql} FROM {temp_view}" + if columns_sql + else f"INSERT INTO {table_name} SELECT * FROM {temp_view}" + ) exc_handler = self.handle_database_exceptions() with exc_handler: if manage_transaction: @@ -670,7 +697,7 @@ def load_from_arrow( self.connection.execute(f"TRUNCATE TABLE {table_name}") self.connection.register(temp_view, arrow_source) try: - insert_result = self.connection.execute(f"INSERT INTO {table_name} SELECT * FROM {temp_view}") + insert_result = self.connection.execute(insert_sql) inserted_rows = _resolve_duckdb_inserted_rows(insert_result) finally: with contextlib.suppress(Exception): @@ -726,8 +753,27 @@ def load_from_storage( operation="read", partitioner=partitioner, ) + validate_load_from_storage_kwargs(kwargs) return super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs + table, + source, + file_format=file_format, + partitioner=partitioner, + overwrite=overwrite, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) def collect_rows(self, cursor: "DuckDBConnection", fetched: "list[Any]") -> "tuple[list[Any], list[str], int]": @@ -977,7 +1023,7 @@ def _build_arrow_table(rows: "list[Any]", column_names: "list[str]") -> Any | No if any(not isinstance(row, (list, tuple)) or len(row) != len(column_names) for row in rows): return None try: - import pyarrow as pa + pa = import_pyarrow() return pa.Table.from_arrays([pa.array(col) for col in zip(*rows, strict=True)], names=column_names) except Exception: @@ -999,9 +1045,9 @@ def _end_transaction_if_active(self, *, commit: bool) -> None: except duckdb.TransactionException as exc: if not str(exc) or "no transaction is active" in str(exc).lower(): return - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc except duckdb.Error as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def _connection_in_transaction(self) -> bool: """Check if connection is in transaction. diff --git a/sqlspec/adapters/duckdb/pool.py b/sqlspec/adapters/duckdb/pool.py index 8fbd41969..f0c2ce268 100644 --- a/sqlspec/adapters/duckdb/pool.py +++ b/sqlspec/adapters/duckdb/pool.py @@ -22,7 +22,7 @@ create_duckdb_profile_path, create_mapped_exception, ) -from sqlspec.exceptions import ImproperConfigurationError +from sqlspec.exceptions import ImproperConfigurationError, with_cause from sqlspec.utils.logging import POOL_LOGGER_NAME, get_logger, log_with_context from sqlspec.utils.uuids import uuid4 @@ -816,9 +816,9 @@ def _open_duckdb_connection( try: connection = duckdb.connect(**connect_parameters) except duckdb.Error as retry_exc: - raise create_mapped_exception(retry_exc) from retry_exc + raise with_cause(create_mapped_exception(retry_exc), retry_exc) from retry_exc elif isinstance(exc, duckdb.Error): - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc else: raise return connection, deferred_parquet_metadata_cache diff --git a/sqlspec/adapters/mssql_python/core.py b/sqlspec/adapters/mssql_python/core.py index d1a6723d5..1d84dc0de 100644 --- a/sqlspec/adapters/mssql_python/core.py +++ b/sqlspec/adapters/mssql_python/core.py @@ -6,9 +6,8 @@ from typing import TYPE_CHECKING, Any, Final, Literal from sqlspec.adapters.mssql_python._typing import MssqlPythonError -from sqlspec.core.parameters import ParameterStyle from sqlspec.core.parameters._registry import build_statement_config_from_profile -from sqlspec.core.parameters._types import DriverParameterProfile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult from sqlspec.exceptions import ( CheckViolationError, @@ -32,7 +31,7 @@ ) from sqlspec.utils.config_tools import parse_odbc_connection_string from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import split_qualified_identifier from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_rowcount, resolve_row_format @@ -41,8 +40,8 @@ from collections.abc import Callable, Sequence from logging import Logger - from sqlspec.core import StatementConfig from sqlspec.core.cache import CachedQuery + from sqlspec.core.statement import StatementConfig __all__ = ( "MSSQL_PYTHON_VERSION", @@ -440,6 +439,7 @@ def execute_cache_hit( cached: "CachedQuery", statement_config: "StatementConfig", column_name_cache: "dict[int, tuple[Any, list[str]]]", + wire_sql: "str | None" = None, ) -> "SQLResult": """Execute a cached statement on an mssql-python connection and build its result. @@ -450,6 +450,7 @@ def execute_cache_hit( cached: Cached compilation of ``sql``. statement_config: Statement configuration the result uses to rebuild its statement. column_name_cache: Session-local cache of resolved column names. + wire_sql: Optional wire SQL override with sqlcommenter formatting applied. Returns: The statement result. @@ -458,7 +459,7 @@ def execute_cache_hit( SQLSpecError: Mapped from any mssql-python error the statement raises. """ parameters = normalize_execute_parameters(params) or () - compiled_sql = cached.compiled_sql + compiled_sql = cached.compiled_sql if wire_sql is None else wire_sql try: cursor = connection.cursor() try: diff --git a/sqlspec/adapters/mssql_python/driver.py b/sqlspec/adapters/mssql_python/driver.py index 167c95d3e..6ba735420 100644 --- a/sqlspec/adapters/mssql_python/driver.py +++ b/sqlspec/adapters/mssql_python/driver.py @@ -5,6 +5,8 @@ from collections.abc import Sized from typing import TYPE_CHECKING, Any, ClassVar, Literal, NotRequired, TypedDict, cast +from mypy_extensions import mypyc_attr + from sqlspec.adapters.mssql_python._typing import ( MssqlPythonConnection, MssqlPythonCursor, @@ -26,20 +28,13 @@ validate_bulk_identifier, ) from sqlspec.adapters.mssql_python.data_dictionary import MssqlPythonSyncDataDictionary -from sqlspec.core import ( - build_arrow_result_from_reader, - build_arrow_result_from_table, - get_cache_config, - register_driver_profile, -) -from sqlspec.driver import ( - BaseSyncExceptionHandler, - SyncDriverAdapterBase, - SyncRowStream, - rows_to_dicts, - validate_savepoint_name, -) -from sqlspec.driver._common import cache_hit_hooks_unchanged +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.result._base import build_arrow_result_from_reader, build_arrow_result_from_table +from sqlspec.driver._common import cache_hit_hooks_unchanged, validate_savepoint_name +from sqlspec.driver._exception_handler import BaseSyncExceptionHandler +from sqlspec.driver._stream import SyncRowStream, rows_to_dicts +from sqlspec.driver._sync import SyncDriverAdapterBase from sqlspec.exceptions import SQLSpecError from sqlspec.utils.arrow_helpers import arrow_reader_with_deferred_close from sqlspec.utils.logging import get_logger @@ -48,10 +43,15 @@ if TYPE_CHECKING: from collections.abc import Iterable, Mapping, Sequence - from sqlspec.builder import QueryBuilder - from sqlspec.core import SQL, ArrowResult, SQLResult, Statement, StatementConfig, StatementFilter - from sqlspec.driver import CachedQuery, ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlglot.dialects.dialect import DialectType + + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.cache import CachedQuery + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.core.statement import SQL, Statement, StatementConfig + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry from sqlspec.typing import ArrowRecordBatchReader, ArrowReturnFormat, StatementParameters @@ -72,6 +72,7 @@ class MssqlPythonBulkCopyResult(TypedDict): """Wall-clock bulk-copy execution time in seconds.""" +@mypyc_attr(allow_interpreted_subclasses=True) class MssqlPythonExceptionHandler(BaseSyncExceptionHandler): """Sync context manager handling mssql-python exceptions.""" @@ -145,6 +146,7 @@ def close(self, error: bool = False) -> None: cursor_manager.__exit__(None, None, None) +@mypyc_attr(allow_interpreted_subclasses=True) class MssqlPythonDriver(SyncDriverAdapterBase): """mssql-python sync driver.""" @@ -156,7 +158,7 @@ class MssqlPythonDriver(SyncDriverAdapterBase): "_restore_autocommit", "_transaction_active", ) - dialect = "tsql" + dialect: "DialectType | None" = "tsql" native_arrow_export: ClassVar[bool] = True def __init__( @@ -238,7 +240,9 @@ def begin(self) -> None: except MssqlPythonError as exc: mapped = create_mapped_exception(exc) msg = f"Failed to begin transaction: {mapped}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc self._restore_autocommit = restore_autocommit self._transaction_active = True @@ -248,7 +252,9 @@ def commit(self) -> None: except MssqlPythonError as exc: mapped = create_mapped_exception(exc) msg = f"Failed to commit transaction: {mapped}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc self._transaction_active = False self._implicit_transaction = False self._restore_connection_autocommit() @@ -259,7 +265,9 @@ def rollback(self) -> None: except MssqlPythonError as exc: mapped = create_mapped_exception(exc) msg = f"Failed to rollback transaction: {mapped}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc finally: self._transaction_active = False self._implicit_transaction = False @@ -689,26 +697,14 @@ def _do_bulkcopy_arrow() -> Any: self._attach_partition_telemetry(telemetry_payload, partitioner) return self._storage_job(telemetry_payload, telemetry) - def load_from_storage( - self, - table: str, - source: "StorageDestination", - *, - file_format: "StorageFormat", - partitioner: "dict[str, object] | None" = None, - overwrite: bool = False, - **kwargs: Any, - ) -> "StorageBridgeJob": - """Load staged artifacts from storage into SQL Server via BulkCopy.""" - return super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs - ) - def _execute_cache_hit( self, sql: str, params: "tuple[Any, ...] | list[Any] | dict[str, Any]", cached: "CachedQuery" ) -> "SQLResult": """Execute a cached statement through the compiled mssql-python cache-hit executor.""" - return execute_cache_hit(self.connection, sql, params, cached, self.statement_config, self._column_name_cache) + wire_sql = self._format_wire_sql(cached.compiled_sql) if self.statement_config.enable_sqlcommenter else None + return execute_cache_hit( + self.connection, sql, params, cached, self.statement_config, self._column_name_cache, wire_sql + ) @staticmethod def _validate_bulk_column_mappings(column_mappings: "Sequence[str] | Sequence[tuple[int, str]] | None") -> None: @@ -742,7 +738,9 @@ def _restore_connection_autocommit(self) -> None: except MssqlPythonError as exc: mapped = create_mapped_exception(exc) msg = f"Failed to restore autocommit: {mapped}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc def _execute_cursor(cursor: "MssqlPythonRawCursor", sql: str, parameters: Any) -> None: diff --git a/sqlspec/adapters/mssql_python/tenant.py b/sqlspec/adapters/mssql_python/tenant.py index acc47b1d3..f54253dc9 100644 --- a/sqlspec/adapters/mssql_python/tenant.py +++ b/sqlspec/adapters/mssql_python/tenant.py @@ -12,7 +12,7 @@ if TYPE_CHECKING: from sqlspec.core.tenant.base import SyncTenantStrategy - from sqlspec.utils.tenant import TenantContextPayload + from sqlspec.utils.tenant._context import TenantContextPayload __all__ = ("MssqlPythonSessionContextStrategy", "MssqlPythonTenantLease") diff --git a/sqlspec/adapters/mssql_python/type_converter.py b/sqlspec/adapters/mssql_python/type_converter.py index 858707086..5e53e50f9 100644 --- a/sqlspec/adapters/mssql_python/type_converter.py +++ b/sqlspec/adapters/mssql_python/type_converter.py @@ -9,7 +9,7 @@ from uuid import UUID from sqlspec.utils.module_loader import ensure_pyarrow, import_optional -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json if TYPE_CHECKING: from collections.abc import Callable diff --git a/sqlspec/adapters/mysqlconnector/_typing.py b/sqlspec/adapters/mysqlconnector/_typing.py index 0acd23ddc..a72e89f10 100644 --- a/sqlspec/adapters/mysqlconnector/_typing.py +++ b/sqlspec/adapters/mysqlconnector/_typing.py @@ -5,7 +5,6 @@ """ import contextlib -import importlib from typing import TYPE_CHECKING, Any import mysql @@ -18,15 +17,15 @@ from mysql.connector.aio import MySQLConnection as _MysqlConnectorAsyncConnection from mysql.connector.aio import cursor as _mysql_connector_aio_cursor +from sqlspec.driver._session import AsyncDriverSessionContext, SyncDriverSessionContext + if TYPE_CHECKING: - from collections.abc import Awaitable, Callable - from types import TracebackType + from collections.abc import Awaitable from typing import ClassVar, Protocol, TypeAlias from mysql.connector.aio.pooling import MySQLConnectionPool as MysqlConnectorAsyncPool from sqlspec.adapters.mysqlconnector.driver import MysqlConnectorAsyncDriver, MysqlConnectorSyncDriver - from sqlspec.core import StatementConfig class MysqlConnectorAsyncRawCursorProtocol(Protocol): description: Any @@ -117,9 +116,6 @@ class MysqlConnectorMysqlModuleProtocol(Protocol): "MysqlConnectorSyncSessionContext", ) -_resolved_sync_driver_type: "type[MysqlConnectorSyncDriver] | None" = None -_resolved_async_driver_type: "type[MysqlConnectorAsyncDriver] | None" = None - class MysqlConnectorSyncCursor: """Context manager for mysql-connector sync cursor operations.""" @@ -179,157 +175,31 @@ async def __aexit__(self, *_: object) -> None: await self.cursor.close() -class MysqlConnectorSyncSessionContext: +class MysqlConnectorSyncSessionContext(SyncDriverSessionContext): """Sync context manager for mysql-connector sessions.""" - __slots__ = ( - "_acquire_connection", - "_connection", - "_driver", - "_driver_features", - "_prepare_driver", - "_release_connection", - "_statement_config", - ) + __slots__ = () - def __init__( - self, - acquire_connection: "Callable[[], MysqlConnectorSyncConnection]", - release_connection: "Callable[..., Any]", - statement_config: "StatementConfig", - driver_features: "dict[str, Any]", - prepare_driver: "Callable[[MysqlConnectorSyncDriver], MysqlConnectorSyncDriver]", - ) -> None: - self._acquire_connection = acquire_connection - self._release_connection = release_connection - self._statement_config = statement_config - self._driver_features = driver_features - self._prepare_driver = prepare_driver - self._connection: MysqlConnectorSyncConnection | None = None - self._driver: MysqlConnectorSyncDriver | None = None - - def __enter__(self) -> "MysqlConnectorSyncDriver": - driver_cls = _get_mysqlconnector_sync_driver_type() - self._connection = self._acquire_connection() - try: - self._driver = driver_cls( - connection=self._connection, - statement_config=self._statement_config, - driver_features=self._driver_features, - ) - return self._prepare_driver(self._driver) - except BaseException as exc: - with contextlib.suppress(Exception): - self.__exit__(type(exc), exc, exc.__traceback__) - raise + if TYPE_CHECKING: - def __exit__( - self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None" - ) -> "bool | None": - driver = self._driver - try: - if driver is not None: - driver._close_open_streams(error=exc_type is not None) - except BaseException as close_exc: - if exc_type is None: - exc_type, exc_val, exc_tb = type(close_exc), close_exc, close_exc.__traceback__ - raise - finally: - try: - if self._connection is not None: - self._release_connection(self._connection, exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) - self._connection = None - finally: - self._driver = None - if driver is not None: - driver._mark_closed() - return None - - -class MysqlConnectorAsyncSessionContext: + def __enter__(self) -> "MysqlConnectorSyncDriver": ... + + def _resolve_driver_class(self) -> "type[MysqlConnectorSyncDriver]": + from sqlspec.adapters.mysqlconnector.driver import MysqlConnectorSyncDriver + + return MysqlConnectorSyncDriver + + +class MysqlConnectorAsyncSessionContext(AsyncDriverSessionContext): """Async context manager for mysql-connector sessions.""" - __slots__ = ( - "_acquire_connection", - "_connection", - "_driver", - "_driver_features", - "_prepare_driver", - "_release_connection", - "_statement_config", - ) + __slots__ = () - def __init__( - self, - acquire_connection: "Callable[[], Awaitable[MysqlConnectorAsyncConnection]]", - release_connection: "Callable[..., Any]", - statement_config: "StatementConfig", - driver_features: "dict[str, Any]", - prepare_driver: "Callable[[MysqlConnectorAsyncDriver], MysqlConnectorAsyncDriver]", - ) -> None: - self._acquire_connection = acquire_connection - self._release_connection = release_connection - self._statement_config = statement_config - self._driver_features = driver_features - self._prepare_driver = prepare_driver - self._connection: MysqlConnectorAsyncConnection | None = None - self._driver: MysqlConnectorAsyncDriver | None = None - - async def __aenter__(self) -> "MysqlConnectorAsyncDriver": - driver_cls = _get_mysqlconnector_async_driver_type() - self._connection = await self._acquire_connection() - try: - self._driver = driver_cls( - connection=self._connection, - statement_config=self._statement_config, - driver_features=self._driver_features, - ) - return self._prepare_driver(self._driver) - except BaseException as exc: - with contextlib.suppress(Exception): - await self.__aexit__(type(exc), exc, exc.__traceback__) - raise + if TYPE_CHECKING: - async def __aexit__( - self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None" - ) -> "bool | None": - driver = self._driver - try: - if driver is not None: - await driver._close_open_streams(error=exc_type is not None) - except BaseException as close_exc: - if exc_type is None: - exc_type, exc_val, exc_tb = type(close_exc), close_exc, close_exc.__traceback__ - raise - finally: - try: - if self._connection is not None: - await self._release_connection(self._connection, exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) - self._connection = None - finally: - self._driver = None - if driver is not None: - driver._mark_closed() - return None - - -def _get_mysqlconnector_sync_driver_type() -> "type[MysqlConnectorSyncDriver]": - """Resolve and cache MysqlConnectorSyncDriver lazily to avoid circular imports.""" - global _resolved_sync_driver_type - driver_cls = _resolved_sync_driver_type - if driver_cls is None: - module = importlib.import_module("sqlspec.adapters.mysqlconnector.driver") - driver_cls = module.MysqlConnectorSyncDriver - _resolved_sync_driver_type = driver_cls - return driver_cls - - -def _get_mysqlconnector_async_driver_type() -> "type[MysqlConnectorAsyncDriver]": - """Resolve and cache MysqlConnectorAsyncDriver lazily to avoid circular imports.""" - global _resolved_async_driver_type - driver_cls = _resolved_async_driver_type - if driver_cls is None: - module = importlib.import_module("sqlspec.adapters.mysqlconnector.driver") - driver_cls = module.MysqlConnectorAsyncDriver - _resolved_async_driver_type = driver_cls - return driver_cls + async def __aenter__(self) -> "MysqlConnectorAsyncDriver": ... + + def _resolve_driver_class(self) -> "type[MysqlConnectorAsyncDriver]": + from sqlspec.adapters.mysqlconnector.driver import MysqlConnectorAsyncDriver + + return MysqlConnectorAsyncDriver diff --git a/sqlspec/adapters/mysqlconnector/core.py b/sqlspec/adapters/mysqlconnector/core.py index c9739384b..0832d5e4d 100644 --- a/sqlspec/adapters/mysqlconnector/core.py +++ b/sqlspec/adapters/mysqlconnector/core.py @@ -7,9 +7,10 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast from sqlspec.adapters.mysqlconnector._typing import MysqlConnectorError -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult -from sqlspec.driver import rows_to_dicts +from sqlspec.driver._stream import rows_to_dicts from sqlspec.exceptions import ( CheckViolationError, ConnectionTimeoutError, @@ -31,13 +32,14 @@ ) from sqlspec.utils.config_tools import parse_mysql_dsn from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import quote_backtick_identifier, split_qualified_identifier from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_lastrowid, has_rowcount, has_sqlstate, has_type_code, resolve_row_format if TYPE_CHECKING: from sqlspec.core.cache import CachedQuery + from sqlspec.core.statement import StatementConfig __all__ = ( "MysqlConnectorAsyncStreamSource", diff --git a/sqlspec/adapters/mysqlconnector/driver.py b/sqlspec/adapters/mysqlconnector/driver.py index 4392e9e29..622bc84d6 100644 --- a/sqlspec/adapters/mysqlconnector/driver.py +++ b/sqlspec/adapters/mysqlconnector/driver.py @@ -11,6 +11,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Final, Literal, cast +from mypy_extensions import mypyc_attr + from sqlspec.adapters.mysqlconnector._typing import ( MysqlConnectorAsyncCursor, MysqlConnectorAsyncSessionContext, @@ -46,25 +48,27 @@ MysqlConnectorAsyncDataDictionary, MysqlConnectorSyncDataDictionary, ) -from sqlspec.core import ArrowResult, get_cache_config, register_driver_profile -from sqlspec.driver import ( - AsyncDriverAdapterBase, - AsyncRowStream, - BaseAsyncExceptionHandler, - BaseSyncExceptionHandler, - SyncDriverAdapterBase, - SyncRowStream, -) +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.driver._async import AsyncDriverAdapterBase from sqlspec.driver._common import cache_hit_hooks_unchanged +from sqlspec.driver._exception_handler import BaseAsyncExceptionHandler, BaseSyncExceptionHandler +from sqlspec.driver._stream import AsyncRowStream, SyncRowStream +from sqlspec.driver._sync import SyncDriverAdapterBase +from sqlspec.exceptions import with_cause from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json +from sqlspec.utils.serializers._json import from_json +from sqlspec.utils.sync_tools import exit_async_context from sqlspec.utils.type_guards import supports_json_type if TYPE_CHECKING: + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.mysqlconnector._typing import MysqlConnectorAsyncConnection, MysqlConnectorSyncConnection - from sqlspec.core import SQL, StatementConfig - from sqlspec.driver import ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.core.result._base import ArrowResult + from sqlspec.core.statement import SQL, StatementConfig + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry __all__ = ( "MysqlConnectorAsyncCursor", @@ -83,6 +87,7 @@ MYSQLCONNECTOR_JSON_TYPE_CODES: Final[set[int]] = {json_type_value} if json_type_value is not None else set() +@mypyc_attr(allow_interpreted_subclasses=True) class MysqlConnectorSyncExceptionHandler(BaseSyncExceptionHandler): """Context manager for handling mysql-connector sync exceptions.""" @@ -97,11 +102,12 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False +@mypyc_attr(allow_interpreted_subclasses=True) class MysqlConnectorSyncDriver(SyncDriverAdapterBase): """MySQL/MariaDB database driver using mysql-connector sync library.""" __slots__ = ("_cursor_options", "_data_dictionary", "_json_deserializer") - dialect = "mysql" + dialect: "DialectType | None" = "mysql" def __init__( self, @@ -200,19 +206,19 @@ def begin(self) -> None: with MysqlConnectorSyncCursor(self.connection) as cursor: cursor.execute("BEGIN") except MysqlConnectorError as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e def commit(self) -> None: try: self.connection.commit() except MysqlConnectorError as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e def rollback(self) -> None: try: self.connection.rollback() except MysqlConnectorError as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e def with_cursor(self, connection: "MysqlConnectorSyncConnection") -> "MysqlConnectorSyncCursor": return MysqlConnectorSyncCursor(connection, self._cursor_options) @@ -372,6 +378,7 @@ def _connection_in_transaction(self) -> bool: return bool(self.connection.in_transaction) +@mypyc_attr(allow_interpreted_subclasses=True) class MysqlConnectorAsyncExceptionHandler(BaseAsyncExceptionHandler): """Async context manager for handling mysql-connector exceptions.""" @@ -386,11 +393,12 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False +@mypyc_attr(allow_interpreted_subclasses=True) class MysqlConnectorAsyncDriver(AsyncDriverAdapterBase): """MySQL/MariaDB database driver using mysql-connector async library.""" __slots__ = ("_cursor_options", "_data_dictionary", "_json_deserializer") - dialect = "mysql" + dialect: "DialectType | None" = "mysql" def __init__( self, @@ -486,22 +494,28 @@ async def dispatch_execute_script(self, cursor: Any, statement: "SQL") -> "Execu async def begin(self) -> None: try: - async with MysqlConnectorAsyncCursor(self.connection) as cursor: + cursor_context = MysqlConnectorAsyncCursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: await cursor.execute("BEGIN") + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) except MysqlConnectorError as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e async def commit(self) -> None: try: await self.connection.commit() except MysqlConnectorError as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e async def rollback(self) -> None: try: await self.connection.rollback() except MysqlConnectorError as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e def with_cursor(self, connection: "MysqlConnectorAsyncConnection") -> "MysqlConnectorAsyncCursor": return MysqlConnectorAsyncCursor(connection, self._cursor_options) @@ -596,6 +610,7 @@ async def load_from_arrow( if use_infile: payload = encode_records_for_local_infile(records) fd, tmp_name = tempfile.mkstemp(suffix=".tsv") + load_error: BaseException | None = None try: with os.fdopen(fd, "wb") as tmp: tmp.write(payload) @@ -605,8 +620,11 @@ async def load_from_arrow( exc_handler, self._execute_cursor_statement_unprepared, load_sql, (tmp_name,) ) self._check_pending_exception(exc_handler) - finally: - await asyncio.to_thread(Path(tmp_name).unlink, True) + except BaseException as exc: + load_error = exc + await asyncio.to_thread(Path(tmp_name).unlink, True) + if load_error is not None: + raise load_error else: insert_sql = build_insert_statement(table, columns) prepared_records = ( @@ -638,25 +656,43 @@ def resolve_rowcount(self, cursor: Any) -> int: return resolve_rowcount(cursor) async def _execute_cursor_statement(self, statement: str, parameters: Any = None) -> None: - async with self.with_cursor(self.connection) as cursor: + cursor_context = self.with_cursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: if parameters is None: await cursor.execute(statement) else: await cursor.execute(statement, parameters) + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) async def _execute_cursor_statement_unprepared(self, statement: str, parameters: Any = None) -> None: options = self._cursor_options if options and options.get("prepared"): options = {k: v for k, v in options.items() if k != "prepared"} - async with MysqlConnectorAsyncCursor(self.connection, options or None) as cursor: + cursor_context = MysqlConnectorAsyncCursor(self.connection, options or None) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: if parameters is None: await cursor.execute(statement) else: await cursor.execute(statement, parameters) + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) async def _executemany_cursor_statement(self, statement: str, parameters: Any) -> None: - async with self.with_cursor(self.connection) as cursor: + cursor_context = self.with_cursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: await cursor.executemany(statement, parameters) + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) def _connection_in_transaction(self) -> bool: """Report whether the connector has an open transaction.""" diff --git a/sqlspec/adapters/oracledb/_json_handlers.py b/sqlspec/adapters/oracledb/_json_handlers.py index 0e9795232..f1678d91c 100644 --- a/sqlspec/adapters/oracledb/_json_handlers.py +++ b/sqlspec/adapters/oracledb/_json_handlers.py @@ -56,12 +56,12 @@ resolve_oracle_connection_compatible_major, resolve_oracle_connection_major, ) -from sqlspec.data_dictionary.dialects.oracle import ( +from sqlspec.data_dictionary.dialects.oracle.config import ( ORACLE_JSON_STORAGE_BLOB_PLAIN, ORACLE_JSON_STORAGE_NATIVE, resolve_oracle_json_storage, ) -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json if TYPE_CHECKING: from sqlspec.adapters.oracledb._typing import OracleAsyncConnection as AsyncConnection diff --git a/sqlspec/adapters/oracledb/_storage.py b/sqlspec/adapters/oracledb/_storage.py index 975f83c99..5e7d95d5c 100644 --- a/sqlspec/adapters/oracledb/_storage.py +++ b/sqlspec/adapters/oracledb/_storage.py @@ -12,6 +12,8 @@ from collections.abc import Mapping from typing import Any, TypedDict +from sqlspec.exceptions import with_cause + __all__ = ("oracle_table_feature_report", "validate_oracle_identifier") _UNQUOTED_ORACLE_IDENTIFIER_PATTERN: "re.Pattern[str]" = re.compile(r"\A[A-Za-z][A-Za-z0-9_$#]*\Z") @@ -66,7 +68,7 @@ def oracle_table_feature_report( except KeyError as exc: supported = ", ".join(sorted(ORACLE_COMPRESSION_CLAUSES)) msg = f"Unsupported Oracle compression algorithm {algorithm!r}. Supported values: {supported}" - raise ValueError(msg) from exc + raise with_cause(ValueError(msg), exc) from exc clauses.append(compression_clause) applied.append("basic_compression" if algorithm == "basic" else "advanced_compression") diff --git a/sqlspec/adapters/oracledb/core.py b/sqlspec/adapters/oracledb/core.py index d4a6748c7..dd0391251 100644 --- a/sqlspec/adapters/oracledb/core.py +++ b/sqlspec/adapters/oracledb/core.py @@ -23,22 +23,17 @@ from sqlspec.adapters.oracledb._param_types import OracleBlob, OracleClob, OracleJson, is_json_payload from sqlspec.adapters.oracledb._typing import DB_TYPE_BLOB, DB_TYPE_CLOB, oracledb_module -from sqlspec.core import ( - DriverParameterProfile, - ParameterStyle, - StackResult, - StatementConfig, - build_statement_config_from_profile, - create_sql_result, -) -from sqlspec.core.result._base import DMLResult, SQLResult +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle +from sqlspec.core.result._base import DMLResult, SQLResult, StackResult, create_sql_result from sqlspec.core.sqlcommenter import SQLCommenterContext, format_wire_sqlcomment, strip_sqlcomment -from sqlspec.data_dictionary.dialects.oracle import ( +from sqlspec.core.statement import StatementConfig +from sqlspec.data_dictionary.dialects.oracle.config import ( ORACLE_JSON_STORAGE_NATIVE, parse_oracle_version_components, resolve_oracle_json_storage, ) -from sqlspec.driver import rows_to_dicts +from sqlspec.driver._stream import rows_to_dicts from sqlspec.exceptions import ( CheckViolationError, ConnectionTimeoutError, @@ -60,12 +55,13 @@ SQLSpecError, TransactionError, UniqueViolationError, + with_cause, ) -from sqlspec.observability import get_trace_context, get_w3c_trace_context +from sqlspec.observability._common import get_trace_context, get_w3c_trace_context from sqlspec.typing import NUMPY_INSTALLED from sqlspec.utils.correlation import CorrelationContext from sqlspec.utils.module_loader import import_optional -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import normalize_identifier, quote_identifier from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_rowcount, is_async_readable, is_readable @@ -79,9 +75,9 @@ OracleSyncConnection, OracleSyncRawCursor, ) - from sqlspec.core import SQL from sqlspec.core.cache import CachedQuery - from sqlspec.driver import AsyncExceptionHandler, SyncExceptionHandler + from sqlspec.core.statement import SQL + from sqlspec.driver._common import AsyncExceptionHandler, SyncExceptionHandler __all__ = ( @@ -888,7 +884,7 @@ def apply_end_user_security_checkout(connection: Any, config: OracleEndUserSecur ) connection.set_end_user_security_context(context) except oracledb_module.DatabaseError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def clear_end_user_security_release(connection: Any) -> None: @@ -896,7 +892,7 @@ def clear_end_user_security_release(connection: Any) -> None: try: connection.clear_end_user_security_context() except oracledb_module.DatabaseError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc _INITIAL_SCHEMA_BY_CONN_ID: dict[int, str] = {} @@ -3042,7 +3038,9 @@ def build_tns_description(topology: "OracleTopologyConfig | Mapping[str, Any]") if resolved.retry_delay is not None: desc_parts.append(f"(RETRY_DELAY={resolved.retry_delay})") if resolved.transport_connect_timeout is not None: - desc_parts.append(f"(TRANSPORT_CONNECT_TIMEOUT={resolved.transport_connect_timeout})") + timeout = resolved.transport_connect_timeout + formatted_timeout = str(int(timeout)) if isinstance(timeout, float) and timeout.is_integer() else str(timeout) + desc_parts.append(f"(TRANSPORT_CONNECT_TIMEOUT={formatted_timeout})") for endpoints in resolved.address_lists or (resolved.endpoints,): addr_parts: list[str] = [] @@ -3127,7 +3125,7 @@ def execute_session_checkout_sync( if plsql is not None: cursor.execute(plsql, binds) if binds else cursor.execute(plsql) except oracledb_module.DatabaseError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def execute_session_release_sync(connection: Any, config: OracleSessionContextConfig | Mapping[str, Any]) -> None: @@ -3142,7 +3140,7 @@ def execute_session_release_sync(connection: Any, config: OracleSessionContextCo if deferred is not None: deferred() except oracledb_module.DatabaseError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc async def execute_session_checkout_async( @@ -3167,7 +3165,7 @@ async def execute_session_checkout_async( if plsql is not None: await cursor.execute(plsql, binds) if binds else await cursor.execute(plsql) except oracledb_module.DatabaseError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc async def execute_session_release_async( @@ -3184,7 +3182,7 @@ async def execute_session_release_async( if deferred is not None: deferred() except oracledb_module.DatabaseError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def sanitize_oracle_text_query(raw_query: str, *, operator: str = "ACCUM") -> str: @@ -3468,7 +3466,7 @@ def apply_oracle_transaction_priority(connection: Any, priority: Any) -> None: if not has_oracle_transaction_priority(connection, resolved): connection.transaction_priority = resolved except oracledb_module.Error as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc class OracleAcquireOptions: @@ -4513,7 +4511,7 @@ def _write_private_wallet_file(target_path: Path, data: bytes) -> None: except OSError as exc: if target_path.is_symlink(): msg = f"Unsafe symlink for Oracle wallet file: {target_path.name!r}" - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc raise try: fchmod = getattr(os, "fchmod", None) @@ -4542,7 +4540,7 @@ def _convert_p12_to_pem_bytes(p12_bytes: bytes, wallet_password: "str | bytes | parsed_bundle = pkcs12_mod.load_key_and_certificates(p12_bytes, b"") except (ValueError, TypeError) as exc: msg = "Oracle wallet contains encrypted 'ewallet.p12'; provide a valid 'wallet_password'" - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc else: pw_bytes = wallet_password.encode("utf-8") if isinstance(wallet_password, str) else bytes(wallet_password) try: @@ -4553,10 +4551,10 @@ def _convert_p12_to_pem_bytes(p12_bytes: bytes, wallet_password: "str | bytes | parsed_bundle = pkcs12_mod.load_key_and_certificates(p12_bytes, None) except (ValueError, TypeError) as inner_exc: msg = "Failed to decrypt Oracle 'ewallet.p12' with the provided 'wallet_password'" - raise ImproperConfigurationError(msg) from inner_exc + raise with_cause(ImproperConfigurationError(msg), inner_exc) from inner_exc else: msg = "Failed to decrypt Oracle 'ewallet.p12' with the provided 'wallet_password'" - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc private_key, certificate, additional_certs = parsed_bundle if private_key is None and certificate is None: @@ -4702,7 +4700,7 @@ def _extract_wallet_bytes_into( _write_private_wallet_file(dest_path, src_file.read()) except zipfile.BadZipFile as exc: msg = f"Invalid Oracle wallet zip archive: {exc}" - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc _ensure_wallet_pem(extract_dir, wallet_password=wallet_password) if not (extract_dir / "ewallet.pem").is_file() and not (extract_dir / "tnsnames.ora").is_file(): diff --git a/sqlspec/adapters/oracledb/driver.py b/sqlspec/adapters/oracledb/driver.py index 3a0f4eed5..4329042c9 100644 --- a/sqlspec/adapters/oracledb/driver.py +++ b/sqlspec/adapters/oracledb/driver.py @@ -4,7 +4,9 @@ from itertools import chain from pathlib import Path from time import perf_counter -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, Protocol, cast, overload +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast, overload + +from mypy_extensions import mypyc_attr from sqlspec.adapters.oracledb._typing import ( DB_TYPE_BLOB, @@ -63,42 +65,36 @@ OracleVersionCache, resolve_oracle_connection_major, ) -from sqlspec.core import ( - SQL, - SQLResult, - StackResult, - StatementConfig, - StatementStack, - build_arrow_result_from_table, - create_arrow_result, - get_cache_config, - register_driver_profile, -) +from sqlspec.core.cache import get_cache_config from sqlspec.core.explain import ( ORACLE_EXPLAIN_DISPLAY_CURSOR_META_KEY, ORACLE_EXPLAIN_PREFIX, ORACLE_EXPLAIN_XPLAN_FORMAT_META_KEY, ORACLE_MANAGED_EXPLAIN_META_KEY, ) +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.result._base import SQLResult, StackResult, build_arrow_result_from_table, create_arrow_result from sqlspec.core.sqlcommenter import format_wire_sqlcomment, strip_sqlcomment -from sqlspec.driver import ( - AsyncDriverAdapterBase, - AsyncRowStream, - BaseAsyncExceptionHandler, - BaseSyncExceptionHandler, +from sqlspec.core.stack import StatementStack +from sqlspec.core.statement import SQL, StatementConfig +from sqlspec.driver._async import AsyncDriverAdapterBase +from sqlspec.driver._common import ( StackExecutionObserver, - SyncDriverAdapterBase, - SyncRowStream, + cache_hit_hooks_unchanged, describe_stack_statement, hash_stack_operations, + validate_load_from_storage_kwargs, validate_savepoint_name, ) -from sqlspec.driver._common import CommonDriverAttributesMixin, cache_hit_hooks_unchanged -from sqlspec.exceptions import ImproperConfigurationError, SQLSpecError, StackExecutionError +from sqlspec.driver._exception_handler import BaseAsyncExceptionHandler, BaseSyncExceptionHandler +from sqlspec.driver._stream import AsyncRowStream, SyncRowStream +from sqlspec.driver._sync import SyncDriverAdapterBase +from sqlspec.exceptions import ImproperConfigurationError, SQLSpecError, StackExecutionError, with_cause from sqlspec.storage._utils import import_pyarrow from sqlspec.utils.arrow_helpers import coerce_oracle_df_to_arrow_table from sqlspec.utils.logging import get_logger, log_with_context from sqlspec.utils.module_loader import ensure_pyarrow +from sqlspec.utils.sync_tools import exit_async_context from sqlspec.utils.text import normalize_identifier, quote_identifier, split_qualified_identifier from sqlspec.utils.type_guards import has_pipeline_capability, is_async_readable from sqlspec.utils.uuids import uuid4 @@ -106,13 +102,17 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator, Generator, Mapping, Sequence - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, Statement, StatementFilter + from sqlglot.dialects.dialect import DialectType + + from sqlspec.builder._base import QueryBuilder from sqlspec.core.cache import CachedQuery + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult from sqlspec.core.stack import StackOperation - from sqlspec.driver import ExecutionResult - from sqlspec.observability import ObservabilityRuntime - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.core.statement import Statement + from sqlspec.driver._common import ExecutionResult + from sqlspec.observability._runtime import ObservabilityRuntime + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry from sqlspec.typing import ArrowRecordBatch, ArrowReturnFormat, ArrowSchema, SchemaT, StatementParameters __all__ = ( @@ -138,24 +138,6 @@ PLAN_CLEANUP_SQL: Final[str] = "DELETE FROM plan_table WHERE statement_id = :statement_id" -class OraclePipelineDriver(Protocol): - """Protocol for Oracle pipeline driver methods used in stack execution.""" - - statement_config: "StatementConfig" - driver_features: "dict[str, Any]" - - def prepare_statement( - self, - statement: "str | Statement | QueryBuilder", - parameters: "tuple[Any, ...] | dict[str, Any] | None", - *, - statement_config: "StatementConfig | None" = None, - kwargs: "dict[str, Any] | None" = None, - ) -> "SQL": ... - - def _compiled_sql(self, statement: "SQL", statement_config: "StatementConfig") -> "tuple[str, Any]": ... - - PIPELINE_MIN_DRIVER_VERSION: "tuple[int, int, int]" = (2, 4, 0) """Lowest python-oracledb version that supports pipelining.""" @@ -168,211 +150,254 @@ def _compiled_sql(self, statement: "SQL", statement_config: "StatementConfig") - """ -class OraclePipelineMixin(CommonDriverAttributesMixin): - """Shared helpers for Oracle pipeline execution.""" - - __slots__ = () +def _oracle_apply_session_tracing(driver: Any, connection: Any = None) -> None: + if not (getattr(driver, "_session_tracing_enabled", False) or getattr(driver, "_otel_tracing_enabled", False)): + return + target_conn = connection if connection is not None else getattr(driver, "connection", None) + if target_conn is None: + return + apply_session_tracing( + target_conn, + driver.statement_config, + driver.driver_features, + enable_session_attributes=getattr(driver, "_session_tracing_enabled", False), + enable_otel_trace_context=getattr(driver, "_otel_tracing_enabled", False), + ) - @property - def _oracle_version_cache(self) -> "OracleVersionCache | None": - """Pool-scoped Oracle version cache attached to this session.""" - return cast("OracleVersionCache | None", cast("OracleCacheHitState", self._cache_hit_state).version_cache) - @_oracle_version_cache.setter - def _oracle_version_cache(self, value: "OracleVersionCache | None") -> None: - cast("OracleCacheHitState", self._cache_hit_state).version_cache = value +def _oracle_format_wire_sql(driver: Any, sql: str, config: "StatementConfig | None" = None) -> str: + effective_config = config or driver.statement_config + if getattr(driver, "_session_tracing_enabled", False): + return strip_sqlcomment(sql) + if not effective_config.enable_sqlcommenter: + return sql + return format_wire_sqlcomment(sql, effective_config) + + +def _oracle_stack_native_blocker(stack: "StatementStack") -> "str | None": + for operation in stack.operations: + if operation.method == "execute_arrow": + return "arrow_operation" + if operation.method == "execute_script": + return "script_operation" + return None + + +def _oracle_log_pipeline_skip(driver: Any, reason: str, stack: "StatementStack") -> None: + log_level = logging.INFO if reason == "env_override" else logging.DEBUG + log_with_context( + logger, + log_level, + "stack.native_pipeline.skip", + driver=type(driver).__name__, + reason=reason, + hashed_operations=hash_stack_operations(stack), + ) - @property - def _otel_tracing_enabled(self) -> bool: - """Whether W3C trace context is propagated to the connection before each statement.""" - return cast("OracleCacheHitState", self._cache_hit_state).otel_tracing_enabled - @_otel_tracing_enabled.setter - def _otel_tracing_enabled(self, value: bool) -> None: - cast("OracleCacheHitState", self._cache_hit_state).otel_tracing_enabled = bool(value) +def _oracle_prepare_pipeline_operation( + driver: "OracleSyncDriver | OracleAsyncDriver", operation: "StackOperation" +) -> "_CompiledStackOperation": + kwargs = dict(operation.keyword_arguments) if operation.keyword_arguments else {} + statement_config = kwargs.pop("statement_config", None) + config = statement_config or driver.statement_config - def attach_observability(self, runtime: "ObservabilityRuntime") -> None: - """Refresh trace propagation when the session receives its runtime.""" - super().attach_observability(runtime) - driver = cast("OracleSyncDriver | OracleAsyncDriver", self) - driver._otel_tracing_enabled = ( - bool(driver.driver_features.get("enable_otel_trace_context", True)) and runtime.span_manager.is_enabled + if operation.method == "execute": + sql_statement = driver.prepare_statement( + operation.statement, operation.arguments, statement_config=config, kwargs=kwargs ) + elif operation.method == "execute_many": + if not operation.arguments: + msg = "execute_many stack operation requires parameter sets" + raise ValueError(msg) + parameter_sets = operation.arguments[0] + filters = operation.arguments[1:] + if isinstance(operation.statement, SQL): + statement_seed = operation.statement.raw_expression or operation.statement.raw_sql + sql_statement = SQL(statement_seed, parameter_sets, statement_config=config, is_many=True, **kwargs) + else: + base_statement = driver.prepare_statement( + operation.statement, filters, statement_config=config, kwargs=kwargs + ) + statement_seed = base_statement.raw_expression or base_statement.raw_sql + sql_statement = SQL(statement_seed, parameter_sets, statement_config=config, is_many=True, **kwargs) + else: + msg = f"Unsupported stack operation method: {operation.method}" + raise ValueError(msg) - def _apply_session_tracing(self, connection: Any = None) -> None: - if not (getattr(self, "_session_tracing_enabled", False) or getattr(self, "_otel_tracing_enabled", False)): - return - driver = cast("OraclePipelineDriver", self) - target_conn = connection if connection is not None else getattr(self, "connection", None) - if target_conn is None: - return - apply_session_tracing( - target_conn, - driver.statement_config, - driver.driver_features, - enable_session_attributes=getattr(self, "_session_tracing_enabled", False), - enable_otel_trace_context=getattr(self, "_otel_tracing_enabled", False), - ) + compiled_sql, prepared_parameters = driver._compiled_sql(sql_statement, config) + summary = describe_stack_statement(operation.statement) + return _CompiledStackOperation( + statement=sql_statement, + sql=compiled_sql, + parameters=prepared_parameters, + method=operation.method, + returns_rows=sql_statement.returns_rows() + and sql_statement.operation_type not in {"INSERT", "UPDATE", "DELETE"}, + summary=summary, + ) - def _format_wire_sql(self, sql: str, config: "StatementConfig | None" = None) -> str: - driver = cast("OraclePipelineDriver", self) - effective_config = config or driver.statement_config - if getattr(self, "_session_tracing_enabled", False): - return strip_sqlcomment(sql) - if not effective_config.enable_sqlcommenter: - return sql - return format_wire_sqlcomment(sql, effective_config) - def _compiled_sql( - self, statement: "SQL", statement_config: "StatementConfig", flatten_single_parameters: bool = False - ) -> "tuple[str, Any]": - compiled_sql, prepared_parameters = super()._compiled_sql( - statement, statement_config, flatten_single_parameters=flatten_single_parameters +def _oracle_add_pipeline_operation( + driver: "OracleSyncDriver | OracleAsyncDriver", pipeline: Any, operation: "_CompiledStackOperation" +) -> None: + parameters = operation.parameters or [] + if operation.method == "execute": + if operation.returns_rows: + fetch_kwargs = build_fetch_kwargs(driver.driver_features) + arraysize = driver.driver_features.get("arraysize") + if arraysize is not None: + fetch_kwargs["arraysize"] = arraysize + pipeline.add_fetchall(operation.sql, parameters, **fetch_kwargs) + else: + pipeline.add_execute(operation.sql, parameters) + return + + if operation.method == "execute_many": + pipeline.add_executemany(operation.sql, parameters) + return + + msg = f"Unsupported pipeline operation: {operation.method}" + raise ValueError(msg) + + +def _oracle_build_stack_results_from_pipeline( + driver: "OracleSyncDriver | OracleAsyncDriver", + compiled_operations: "Sequence[_CompiledStackOperation]", + pipeline_results: "Sequence[Any]", + continue_on_error: bool, + observer: StackExecutionObserver, + *, + coerced_rows_by_index: "Mapping[int, Sequence[Any] | None] | None" = None, +) -> "list[StackResult]": + stack_results: list[StackResult] = [] + for index, (compiled, result) in enumerate(zip(compiled_operations, pipeline_results, strict=False)): + try: + error = result.error + except AttributeError: + error = None + if error is not None: + mapped_error = create_mapped_exception(error) if not isinstance(error, SQLSpecError) else error + stack_error = StackExecutionError( + index, + compiled.summary, + mapped_error, + adapter=type(driver).__name__, + mode="continue-on-error" if continue_on_error else "fail-fast", + ) + if continue_on_error: + observer.record_operation_error(stack_error) + stack_results.append(StackResult.from_error(stack_error)) + continue + raise stack_error + + stack_results.append( + build_pipeline_stack_result( + compiled.statement, + compiled.method, + compiled.returns_rows, + compiled.parameters, + result, + driver.driver_features, + coerced_rows=coerced_rows_by_index.get(index) if coerced_rows_by_index is not None else None, + ) ) - if getattr(self, "_session_tracing_enabled", False): - return strip_sqlcomment(compiled_sql), prepared_parameters - return compiled_sql, prepared_parameters + return stack_results - def _stack_native_blocker(self, stack: "StatementStack") -> "str | None": - for operation in stack.operations: - if operation.method == "execute_arrow": - return "arrow_operation" - if operation.method == "execute_script": - return "script_operation" - return None - def _log_pipeline_skip(self, reason: str, stack: "StatementStack") -> None: - log_level = logging.INFO if reason == "env_override" else logging.DEBUG - log_with_context( - logger, - log_level, - "stack.native_pipeline.skip", - driver=type(self).__name__, - reason=reason, - hashed_operations=hash_stack_operations(stack), - ) +def _oracle_check_pipeline_commit_result(result: Any) -> None: + error = getattr(result, "error", None) + if error is not None: + raise create_mapped_exception(error) - def _prepare_pipeline_operation(self, operation: "StackOperation") -> "_CompiledStackOperation": - driver = cast("OraclePipelineDriver", self) - kwargs = dict(operation.keyword_arguments) if operation.keyword_arguments else {} - statement_config = kwargs.pop("statement_config", None) - config = statement_config or driver.statement_config - - if operation.method == "execute": - sql_statement = driver.prepare_statement( - operation.statement, operation.arguments, statement_config=config, kwargs=kwargs - ) - elif operation.method == "execute_many": - if not operation.arguments: - msg = "execute_many stack operation requires parameter sets" - raise ValueError(msg) - parameter_sets = operation.arguments[0] - filters = operation.arguments[1:] - if isinstance(operation.statement, SQL): - statement_seed = operation.statement.raw_expression or operation.statement.raw_sql - sql_statement = SQL(statement_seed, parameter_sets, statement_config=config, is_many=True, **kwargs) - else: - base_statement = driver.prepare_statement( - operation.statement, filters, statement_config=config, kwargs=kwargs - ) - statement_seed = base_statement.raw_expression or base_statement.raw_sql - sql_statement = SQL(statement_seed, parameter_sets, statement_config=config, is_many=True, **kwargs) - else: - msg = f"Unsupported stack operation method: {operation.method}" - raise ValueError(msg) - compiled_sql, prepared_parameters = self._compiled_sql(sql_statement, config) - summary = describe_stack_statement(operation.statement) - return _CompiledStackOperation( - statement=sql_statement, - sql=compiled_sql, - parameters=prepared_parameters, - method=operation.method, - returns_rows=sql_statement.returns_rows() - and sql_statement.operation_type not in {"INSERT", "UPDATE", "DELETE"}, - summary=summary, - ) +def _oracle_wrap_pipeline_error(driver: Any, error: Exception, continue_on_error: bool) -> StackExecutionError: + mode = "continue-on-error" if continue_on_error else "fail-fast" + mapped_error = create_mapped_exception(error) if not isinstance(error, SQLSpecError) else error + wrapped = StackExecutionError( + -1, "Oracle pipeline execution failed", mapped_error, adapter=type(driver).__name__, mode=mode + ) + wrapped.__cause__ = error + return wrapped - def _add_pipeline_operation(self, pipeline: Any, operation: "_CompiledStackOperation") -> None: - parameters = operation.parameters or [] - if operation.method == "execute": - if operation.returns_rows: - driver = cast("OraclePipelineDriver", self) - fetch_kwargs = build_fetch_kwargs(driver.driver_features) - arraysize = driver.driver_features.get("arraysize") - if arraysize is not None: - fetch_kwargs["arraysize"] = arraysize - pipeline.add_fetchall(operation.sql, parameters, **fetch_kwargs) - else: - pipeline.add_execute(operation.sql, parameters) - return - if operation.method == "execute_many": - pipeline.add_executemany(operation.sql, parameters) - return +class _OracleAsyncArrowBatchIterator: + """Async iterator yielding Oracle DataFrame batches as Arrow record batches.""" - msg = f"Unsupported pipeline operation: {operation.method}" - raise ValueError(msg) + __slots__ = ("_arrow_schema", "_closed", "_driver_features", "_oracle_dfs", "_pending_batches", "_yielded") - def _build_stack_results_from_pipeline( + def __init__( self, - compiled_operations: "Sequence[_CompiledStackOperation]", - pipeline_results: "Sequence[Any]", - continue_on_error: bool, - observer: StackExecutionObserver, - *, - coerced_rows_by_index: "Mapping[int, Sequence[Any] | None] | None" = None, - ) -> "list[StackResult]": - driver = cast("OraclePipelineDriver", self) - stack_results: list[StackResult] = [] - for index, (compiled, result) in enumerate(zip(compiled_operations, pipeline_results, strict=False)): - try: - error = result.error - except AttributeError: - error = None - if error is not None: - mapped_error = create_mapped_exception(error) if not isinstance(error, SQLSpecError) else error - stack_error = StackExecutionError( - index, - compiled.summary, - mapped_error, - adapter=type(self).__name__, - mode="continue-on-error" if continue_on_error else "fail-fast", - ) - if continue_on_error: - observer.record_operation_error(stack_error) - stack_results.append(StackResult.from_error(stack_error)) - continue - raise stack_error - - stack_results.append( - build_pipeline_stack_result( - compiled.statement, - compiled.method, - compiled.returns_rows, - compiled.parameters, - result, - driver.driver_features, - coerced_rows=coerced_rows_by_index.get(index) if coerced_rows_by_index is not None else None, - ) - ) - return stack_results + connection: Any, + sql: str, + parameters: "StatementParameters | None", + batch_size: int | None, + driver_features: Any, + arrow_schema: "ArrowSchema | None", + ) -> None: + self._oracle_dfs = _fetch_dataframe_batches( + connection, sql, parameters, batch_size, driver_features, arrow_schema + ) + self._driver_features = driver_features + self._arrow_schema = arrow_schema + self._pending_batches: list[Any] = [] + self._yielded = False + self._closed = False - def _check_pipeline_commit_result(self, result: Any) -> None: - error = getattr(result, "error", None) - if error is not None: - raise create_mapped_exception(error) + def __aiter__(self) -> "_OracleAsyncArrowBatchIterator": + return self - def _wrap_pipeline_error( - self, error: Exception, stack: "StatementStack", continue_on_error: bool - ) -> StackExecutionError: - mode = "continue-on-error" if continue_on_error else "fail-fast" - mapped_error = create_mapped_exception(error) if not isinstance(error, SQLSpecError) else error - return StackExecutionError( - -1, "Oracle pipeline execution failed", mapped_error, adapter=type(self).__name__, mode=mode - ) + async def aclose(self) -> None: + if self._closed: + return + self._closed = True + aclose_dfs = getattr(self._oracle_dfs, "aclose", None) + if callable(aclose_dfs): + await cast("Any", aclose_dfs)() + return + close_dfs = getattr(self._oracle_dfs, "close", None) + if callable(close_dfs): + close_dfs() + + async def __anext__(self) -> "ArrowRecordBatch": + while not self._pending_batches: + next_error: BaseException | None = None + oracle_df: Any = None + stop_iteration = False + try: + oracle_df = await self._oracle_dfs.__anext__() + except StopAsyncIteration: + stop_iteration = True + except OracleError as exc: + next_error = exc + except BaseException as exc: + next_error = exc + if next_error is not None: + await self.aclose() + if isinstance(next_error, OracleError): + raise with_cause(create_mapped_exception(next_error), next_error) from next_error + raise next_error + if stop_iteration: + await self.aclose() + if not self._yielded and self._arrow_schema is not None: + self._yielded = True + return cast( + "ArrowRecordBatch", import_pyarrow().RecordBatch.from_pylist([], schema=self._arrow_schema) + ) + raise StopAsyncIteration + batch_table = _coerce_oracle_arrow_batch_table(oracle_df, self._driver_features, self._arrow_schema) + if batch_table.num_rows: + batches = list(batch_table.to_batches()) + batches.reverse() + self._pending_batches = batches + else: + self._yielded = True + return cast("ArrowRecordBatch", import_pyarrow().RecordBatch.from_pylist([], schema=batch_table.schema)) + self._yielded = True + return cast("ArrowRecordBatch", self._pending_batches.pop()) +@mypyc_attr(allow_interpreted_subclasses=True) class OracleSyncExceptionHandler(BaseSyncExceptionHandler): """Synchronous context manager for handling Oracle database exceptions. @@ -392,6 +417,7 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False +@mypyc_attr(allow_interpreted_subclasses=True) class OracleAsyncExceptionHandler(BaseAsyncExceptionHandler): """Async context manager for handling Oracle database exceptions. @@ -411,7 +437,8 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False -class OracleSyncDriver(OraclePipelineMixin, SyncDriverAdapterBase): +@mypyc_attr(allow_interpreted_subclasses=True) +class OracleSyncDriver(SyncDriverAdapterBase): """Synchronous Oracle Database driver. Provides Oracle Database connectivity with parameter style conversion, @@ -428,7 +455,7 @@ class OracleSyncDriver(OraclePipelineMixin, SyncDriverAdapterBase): "_sessionless_transaction_id", "_transaction_active", ) - dialect = "oracle" + dialect: "DialectType | None" = "oracle" export_integers_may_widen: ClassVar[bool] = True native_arrow_export: ClassVar[bool] = True _pipeline_support: ClassVar[bool] = False @@ -468,6 +495,83 @@ def sessionless_transaction_id(self) -> bytes | None: """Return the sessionless transaction currently attached to this connection.""" return self._sessionless_transaction_id + @property + def _oracle_version_cache(self) -> "OracleVersionCache | None": + return cast("OracleVersionCache | None", cast("OracleCacheHitState", self._cache_hit_state).version_cache) + + @_oracle_version_cache.setter + def _oracle_version_cache(self, value: "OracleVersionCache | None") -> None: + cast("OracleCacheHitState", self._cache_hit_state).version_cache = value + + @property + def _otel_tracing_enabled(self) -> bool: + return cast("OracleCacheHitState", self._cache_hit_state).otel_tracing_enabled + + @_otel_tracing_enabled.setter + def _otel_tracing_enabled(self, value: bool) -> None: + cast("OracleCacheHitState", self._cache_hit_state).otel_tracing_enabled = bool(value) + + def attach_observability(self, runtime: "ObservabilityRuntime") -> None: + """Refresh trace propagation when the session receives its runtime.""" + super().attach_observability(runtime) + self._otel_tracing_enabled = ( + bool(self.driver_features.get("enable_otel_trace_context", True)) and runtime.span_manager.is_enabled + ) + + def _apply_session_tracing(self, connection: Any = None) -> None: + _oracle_apply_session_tracing(self, connection) + + def _format_wire_sql(self, sql: str, config: "StatementConfig | None" = None) -> str: + return _oracle_format_wire_sql(self, sql, config) + + def _compiled_sql( + self, statement: "SQL", statement_config: "StatementConfig", flatten_single_parameters: bool = False + ) -> "tuple[str, Any]": + compiled_sql, prepared_parameters = super()._compiled_sql( + statement, statement_config, flatten_single_parameters=flatten_single_parameters + ) + if self._session_tracing_enabled: + return strip_sqlcomment(compiled_sql), prepared_parameters + return compiled_sql, prepared_parameters + + def _stack_native_blocker(self, stack: "StatementStack") -> "str | None": + return _oracle_stack_native_blocker(stack) + + def _log_pipeline_skip(self, reason: str, stack: "StatementStack") -> None: + _oracle_log_pipeline_skip(self, reason, stack) + + def _prepare_pipeline_operation(self, operation: "StackOperation") -> "_CompiledStackOperation": + return _oracle_prepare_pipeline_operation(self, operation) + + def _add_pipeline_operation(self, pipeline: Any, operation: "_CompiledStackOperation") -> None: + _oracle_add_pipeline_operation(self, pipeline, operation) + + def _build_stack_results_from_pipeline( + self, + compiled_operations: "Sequence[_CompiledStackOperation]", + pipeline_results: "Sequence[Any]", + continue_on_error: bool, + observer: StackExecutionObserver, + *, + coerced_rows_by_index: "Mapping[int, Sequence[Any] | None] | None" = None, + ) -> "list[StackResult]": + return _oracle_build_stack_results_from_pipeline( + self, + compiled_operations, + pipeline_results, + continue_on_error, + observer, + coerced_rows_by_index=coerced_rows_by_index, + ) + + def _check_pipeline_commit_result(self, result: Any) -> None: + _oracle_check_pipeline_commit_result(result) + + def _wrap_pipeline_error( + self, error: Exception, stack: "StatementStack", continue_on_error: bool + ) -> StackExecutionError: + return _oracle_wrap_pipeline_error(self, error, continue_on_error) + @property def data_dictionary(self) -> "OracledbSyncDataDictionary": """Get the data dictionary for this driver. @@ -497,7 +601,7 @@ def begin_sessionless_transaction( try: txid = method(resolved, timeout=timeout, defer_round_trip=defer_round_trip) except OracleError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc self._sessionless_transaction_id = cast("bytes", txid) return self._sessionless_transaction_id @@ -515,7 +619,7 @@ def resume_sessionless_transaction( try: txid = method(cast("bytes", resolved), timeout=timeout, defer_round_trip=defer_round_trip) except OracleError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc self._sessionless_transaction_id = cast("bytes", txid) return self._sessionless_transaction_id @@ -532,6 +636,7 @@ def suspend_sessionless_transaction(self) -> None: if self._sessionless_transaction_id is not None: self._sessionless_cleanup_error = pending if pending is not exc: + pending.__cause__ = exc raise pending from exc raise else: @@ -815,6 +920,7 @@ def commit(self) -> None: if self._sessionless_transaction_id is not None: self._sessionless_cleanup_error = pending if pending is not exc: + pending.__cause__ = exc raise pending from exc raise else: @@ -836,6 +942,7 @@ def rollback(self) -> None: if self._sessionless_transaction_id is not None: self._sessionless_cleanup_error = pending if pending is not exc: + pending.__cause__ = exc raise pending from exc raise else: @@ -1056,7 +1163,9 @@ def select_to_arrow( except AttributeError as exc: if native_only: msg = "Oracle native Arrow support is not available for this connection." - raise ImproperConfigurationError(msg) from exc + error = ImproperConfigurationError(msg) + error.__cause__ = exc + raise error from exc return super().select_to_arrow( prepared_statement, statement_config=config, @@ -1281,8 +1390,27 @@ def _run_dbms_cloud() -> None: self._attach_partition_telemetry(telemetry_payload, partitioner) return self._storage_job(telemetry_payload) if kwargs: + validate_load_from_storage_kwargs(kwargs) return super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs + table, + source, + file_format=file_format, + partitioner=partitioner, + overwrite=overwrite, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) arrow_table, inbound = self._read_storage_arrow(source, file_format=file_format) return self.load_from_arrow(table, arrow_table, partitioner=partitioner, overwrite=overwrite, telemetry=inbound) @@ -1460,7 +1588,7 @@ def _iter_arrow_record_batches( else: yield import_pyarrow().RecordBatch.from_pylist([], schema=batch_table.schema) except OracleError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc finally: close_dfs = getattr(oracle_dfs, "close", None) if callable(close_dfs): @@ -1485,7 +1613,7 @@ def _fetch_arrow_record_batches( batch_schema = batch_table.schema record_batches.extend(batch_table.to_batches()) except OracleError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc finally: close_dfs = getattr(oracle_dfs, "close", None) if callable(close_dfs): @@ -1493,7 +1621,8 @@ def _fetch_arrow_record_batches( return record_batches, batch_schema -class OracleAsyncDriver(OraclePipelineMixin, AsyncDriverAdapterBase): +@mypyc_attr(allow_interpreted_subclasses=True) +class OracleAsyncDriver(AsyncDriverAdapterBase): """Asynchronous Oracle Database driver. Provides Oracle Database connectivity with parameter style conversion, @@ -1512,7 +1641,7 @@ class OracleAsyncDriver(OraclePipelineMixin, AsyncDriverAdapterBase): "_sessionless_transaction_id", "_transaction_active", ) - dialect = "oracle" + dialect: "DialectType | None" = "oracle" export_integers_may_widen: ClassVar[bool] = True def __init__( @@ -1551,6 +1680,83 @@ def sessionless_transaction_id(self) -> bytes | None: """Return the sessionless transaction currently attached to this connection.""" return self._sessionless_transaction_id + @property + def _oracle_version_cache(self) -> "OracleVersionCache | None": + return cast("OracleVersionCache | None", cast("OracleCacheHitState", self._cache_hit_state).version_cache) + + @_oracle_version_cache.setter + def _oracle_version_cache(self, value: "OracleVersionCache | None") -> None: + cast("OracleCacheHitState", self._cache_hit_state).version_cache = value + + @property + def _otel_tracing_enabled(self) -> bool: + return cast("OracleCacheHitState", self._cache_hit_state).otel_tracing_enabled + + @_otel_tracing_enabled.setter + def _otel_tracing_enabled(self, value: bool) -> None: + cast("OracleCacheHitState", self._cache_hit_state).otel_tracing_enabled = bool(value) + + def attach_observability(self, runtime: "ObservabilityRuntime") -> None: + """Refresh trace propagation when the session receives its runtime.""" + super().attach_observability(runtime) + self._otel_tracing_enabled = ( + bool(self.driver_features.get("enable_otel_trace_context", True)) and runtime.span_manager.is_enabled + ) + + def _apply_session_tracing(self, connection: Any = None) -> None: + _oracle_apply_session_tracing(self, connection) + + def _format_wire_sql(self, sql: str, config: "StatementConfig | None" = None) -> str: + return _oracle_format_wire_sql(self, sql, config) + + def _compiled_sql( + self, statement: "SQL", statement_config: "StatementConfig", flatten_single_parameters: bool = False + ) -> "tuple[str, Any]": + compiled_sql, prepared_parameters = super()._compiled_sql( + statement, statement_config, flatten_single_parameters=flatten_single_parameters + ) + if self._session_tracing_enabled: + return strip_sqlcomment(compiled_sql), prepared_parameters + return compiled_sql, prepared_parameters + + def _stack_native_blocker(self, stack: "StatementStack") -> "str | None": + return _oracle_stack_native_blocker(stack) + + def _log_pipeline_skip(self, reason: str, stack: "StatementStack") -> None: + _oracle_log_pipeline_skip(self, reason, stack) + + def _prepare_pipeline_operation(self, operation: "StackOperation") -> "_CompiledStackOperation": + return _oracle_prepare_pipeline_operation(self, operation) + + def _add_pipeline_operation(self, pipeline: Any, operation: "_CompiledStackOperation") -> None: + _oracle_add_pipeline_operation(self, pipeline, operation) + + def _build_stack_results_from_pipeline( + self, + compiled_operations: "Sequence[_CompiledStackOperation]", + pipeline_results: "Sequence[Any]", + continue_on_error: bool, + observer: StackExecutionObserver, + *, + coerced_rows_by_index: "Mapping[int, Sequence[Any] | None] | None" = None, + ) -> "list[StackResult]": + return _oracle_build_stack_results_from_pipeline( + self, + compiled_operations, + pipeline_results, + continue_on_error, + observer, + coerced_rows_by_index=coerced_rows_by_index, + ) + + def _check_pipeline_commit_result(self, result: Any) -> None: + _oracle_check_pipeline_commit_result(result) + + def _wrap_pipeline_error( + self, error: Exception, stack: "StatementStack", continue_on_error: bool + ) -> StackExecutionError: + return _oracle_wrap_pipeline_error(self, error, continue_on_error) + @property def data_dictionary(self) -> "OracledbAsyncDataDictionary": """Get the data dictionary for this driver. @@ -1580,7 +1786,7 @@ async def begin_sessionless_transaction( try: txid = await cast("Any", method)(resolved, timeout=timeout, defer_round_trip=defer_round_trip) except OracleError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc self._sessionless_transaction_id = cast("bytes", txid) return self._sessionless_transaction_id @@ -1600,7 +1806,7 @@ async def resume_sessionless_transaction( cast("bytes", resolved), timeout=timeout, defer_round_trip=defer_round_trip ) except OracleError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc self._sessionless_transaction_id = cast("bytes", txid) return self._sessionless_transaction_id @@ -1617,6 +1823,7 @@ async def suspend_sessionless_transaction(self) -> None: if self._sessionless_transaction_id is not None: self._sessionless_cleanup_error = pending if pending is not exc: + pending.__cause__ = exc raise pending from exc raise else: @@ -1902,6 +2109,7 @@ async def commit(self) -> None: if self._sessionless_transaction_id is not None: self._sessionless_cleanup_error = pending if pending is not exc: + pending.__cause__ = exc raise pending from exc raise else: @@ -1923,6 +2131,7 @@ async def rollback(self) -> None: if self._sessionless_transaction_id is not None: self._sessionless_cleanup_error = pending if pending is not exc: + pending.__cause__ = exc raise pending from exc raise else: @@ -2105,6 +2314,8 @@ async def select_to_arrow( prepared_parameters = cast("list[Any] | tuple[Any, ...] | dict[Any, Any] | None", prepared_parameters) exc_handler = self.handle_database_exceptions() + arrow_fallback_error: AttributeError | None = None + oracle_df: Any = None try: self._apply_session_tracing(self.connection) if return_format in {"batches", "reader"} and supports_df_batches(self.connection): @@ -2136,9 +2347,13 @@ async def select_to_arrow( exc_handler, self._execute_arrow_dataframe, sql, prepared_parameters, batch_size ) except AttributeError as exc: + arrow_fallback_error = exc + if arrow_fallback_error is not None: if native_only: msg = "Oracle native Arrow support is not available for this connection." - raise ImproperConfigurationError(msg) from exc + error = ImproperConfigurationError(msg) + error.__cause__ = arrow_fallback_error + raise error from arrow_fallback_error return await super().select_to_arrow( prepared_statement, statement_config=config, @@ -2224,8 +2439,12 @@ async def load_from_arrow( """Asynchronously load Arrow data into Oracle.""" self._require_capability("arrow_import_enabled") if drop_indexes: - async with self.without_indexes(table, concurrent=concurrent_indexes): - return await self.load_from_arrow( + index_context = self.without_indexes(table, concurrent=concurrent_indexes) + await index_context.__aenter__() + index_error: BaseException | None = None + index_job: StorageBridgeJob | None = None + try: + index_job = await self.load_from_arrow( table, source, partitioner=partitioner, @@ -2240,6 +2459,11 @@ async def load_from_arrow( concurrent_indexes=concurrent_indexes, type_casts=type_casts, ) + except BaseException as exc: + index_error = exc + await exit_async_context(index_context, index_error) + assert index_job is not None + return index_job if stage_table or conflict_keys: return await self._execute_staged_load_async( table, @@ -2370,8 +2594,27 @@ async def _run_dbms_cloud_async() -> None: self._attach_partition_telemetry(telemetry_payload, partitioner) return self._storage_job(telemetry_payload) if kwargs: + validate_load_from_storage_kwargs(kwargs) return await super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs + table, + source, + file_format=file_format, + partitioner=partitioner, + overwrite=overwrite, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) arrow_table, inbound = await self._read_storage_arrow(source, file_format=file_format) return await self.load_from_arrow( @@ -2523,6 +2766,8 @@ async def _dispatch_explain_plan( await cursor.execute(_tagged_explain_plan_sql(statement_id, explained_sql), parameters or {}) display_sql = PLAN_DISPLAY_SQL if xplan_format == "TYPICAL" else _format_plan_display_sql(xplan_format) + display_error: Exception | None = None + result: ExecutionResult | None = None try: await cursor.execute(display_sql, {"statement_id": statement_id}, **self._fetch_kwargs) fetched_data = await cursor.fetchall() @@ -2542,14 +2787,16 @@ async def _dispatch_explain_plan( is_select_result=True, row_format="tuple", ) - except Exception: + except Exception as exc: + display_error = exc + if display_error is not None: try: await cursor.execute(PLAN_CLEANUP_SQL, {"statement_id": statement_id}) except Exception: logger.warning("Failed to clean up Oracle plan rows after plan retrieval failed", exc_info=True) - raise - else: - await cursor.execute(PLAN_CLEANUP_SQL, {"statement_id": statement_id}) + raise display_error + await cursor.execute(PLAN_CLEANUP_SQL, {"statement_id": statement_id}) + assert result is not None return result async def _execute_arrow_dataframe(self, sql: str, parameters: "Any", batch_size: int | None) -> "Any": @@ -2560,56 +2807,40 @@ async def _execute_arrow_dataframe(self, sql: str, parameters: "Any", batch_size except AttributeError: execute_df = None if execute_df is not None: + fallback_to_plain = False try: return await execute_df(sql, params, arraysize=batch_size or 1000) except TypeError: + fallback_to_plain = True + if fallback_to_plain: return await execute_df(sql, params) fetch_kwargs = build_arrow_fetch_kwargs(self.driver_features) + retry_kwargs: dict[str, Any] | None = None + fetch_error: TypeError | None = None try: return await self.connection.fetch_df_all( statement=sql, parameters=params, arraysize=batch_size or 1000, **fetch_kwargs ) except TypeError as exc: + fetch_error = exc retry_kwargs = _retry_arrow_without_fetch_lobs(exc, fetch_kwargs) - if retry_kwargs is None: - raise - return await self.connection.fetch_df_all( - statement=sql, parameters=params, arraysize=batch_size or 1000, **retry_kwargs - ) + if retry_kwargs is None: + assert fetch_error is not None + raise fetch_error + return await self.connection.fetch_df_all( + statement=sql, parameters=params, arraysize=batch_size or 1000, **retry_kwargs + ) - async def _iter_arrow_record_batches( + def _iter_arrow_record_batches( self, sql: str, parameters: "StatementParameters | None", batch_size: int | None, arrow_schema: "ArrowSchema | None", ) -> "AsyncIterator[ArrowRecordBatch]": - oracle_dfs = _fetch_dataframe_batches( + return _OracleAsyncArrowBatchIterator( self.connection, sql, parameters, batch_size, self.driver_features, arrow_schema ) - yielded = False - try: - async for oracle_df in oracle_dfs: - batch_table = _coerce_oracle_arrow_batch_table(oracle_df, self.driver_features, arrow_schema) - if batch_table.num_rows: - for batch in batch_table.to_batches(): - yielded = True - yield batch - else: - yielded = True - yield import_pyarrow().RecordBatch.from_pylist([], schema=batch_table.schema) - if not yielded and arrow_schema is not None: - yield import_pyarrow().RecordBatch.from_pylist([], schema=arrow_schema) - except OracleError as exc: - raise create_mapped_exception(exc) from exc - finally: - aclose_dfs = getattr(oracle_dfs, "aclose", None) - if callable(aclose_dfs): - await cast("Any", aclose_dfs)() - else: - close_dfs = getattr(oracle_dfs, "close", None) - if callable(close_dfs): - close_dfs() async def _fetch_arrow_record_batches( self, @@ -2623,22 +2854,26 @@ async def _fetch_arrow_record_batches( oracle_dfs = _fetch_dataframe_batches( self.connection, sql, parameters, batch_size, self.driver_features, arrow_schema ) + fetch_error: BaseException | None = None try: async for oracle_df in oracle_dfs: batch_table = _coerce_oracle_arrow_batch_table(oracle_df, self.driver_features, arrow_schema) if batch_schema is None: batch_schema = batch_table.schema record_batches.extend(batch_table.to_batches()) - except OracleError as exc: - raise create_mapped_exception(exc) from exc - finally: - aclose_dfs = getattr(oracle_dfs, "aclose", None) - if callable(aclose_dfs): - await cast("Any", aclose_dfs)() - else: - close_dfs = getattr(oracle_dfs, "close", None) - if callable(close_dfs): - close_dfs() + except BaseException as exc: + fetch_error = exc + aclose_dfs = getattr(oracle_dfs, "aclose", None) + if callable(aclose_dfs): + await cast("Any", aclose_dfs)() + else: + close_dfs = getattr(oracle_dfs, "close", None) + if callable(close_dfs): + close_dfs() + if fetch_error is not None: + if isinstance(fetch_error, OracleError): + raise with_cause(create_mapped_exception(fetch_error), fetch_error) from fetch_error + raise fetch_error return record_batches, batch_schema async def _execute_stack_native( @@ -2679,6 +2914,7 @@ async def _execute_stack_native( owns_transaction = not self._connection_in_transaction() with StackExecutionObserver(self, stack, continue_on_error, native_pipeline=True) as observer: + pipeline_error: Exception | None = None try: pipelined_commit = owns_transaction and not continue_on_error if owns_transaction: @@ -2713,14 +2949,18 @@ async def _execute_stack_native( else: await self.rollback() except Exception as exc: + pipeline_error = exc + if pipeline_error is not None: if owns_transaction: try: await self.rollback() except Exception as rollback_error: logger.debug("Rollback after pipeline failure failed: %s", rollback_error) - if isinstance(exc, StackExecutionError): - raise - raise self._wrap_pipeline_error(exc, stack, continue_on_error) from exc + if isinstance(pipeline_error, StackExecutionError): + raise pipeline_error + raise with_cause( + self._wrap_pipeline_error(pipeline_error, stack, continue_on_error), pipeline_error + ) from pipeline_error return tuple(results) @@ -3019,7 +3259,7 @@ def _fetch_dataframe_batches( statement=sql, parameters=params, size=batch_size or 1000, **retry_kwargs ) except OracleError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def _coerce_oracle_arrow_batch_table(oracle_df: Any, driver_features: Any, arrow_schema: "ArrowSchema | None") -> Any: diff --git a/sqlspec/adapters/oracledb/events/_hub.py b/sqlspec/adapters/oracledb/events/_hub.py index 387ed3476..3c4e75a4c 100644 --- a/sqlspec/adapters/oracledb/events/_hub.py +++ b/sqlspec/adapters/oracledb/events/_hub.py @@ -15,7 +15,7 @@ from sqlspec.adapters.oracledb._typing import DatabaseError as OracleDatabaseError from sqlspec.exceptions import EventChannelError -from sqlspec.extensions.events import normalize_event_channel_name +from sqlspec.extensions.events._store import normalize_event_channel_name from sqlspec.utils.logging import get_logger, log_with_context if TYPE_CHECKING: diff --git a/sqlspec/adapters/oracledb/tenant.py b/sqlspec/adapters/oracledb/tenant.py index fd20a3e65..87b71025c 100644 --- a/sqlspec/adapters/oracledb/tenant.py +++ b/sqlspec/adapters/oracledb/tenant.py @@ -33,7 +33,7 @@ from sqlspec.core.tenant.base import TenantLeaseState, with_cause from sqlspec.driver._tenant import AsyncTenantLease, SyncTenantLease, await_owned_disposal from sqlspec.exceptions import ImproperConfigurationError, TenantCleanupError, TenantContextError -from sqlspec.utils.tenant import TenantContext, TenantContextPayload +from sqlspec.utils.tenant._context import TenantContext, TenantContextPayload from sqlspec.utils.type_guards import has_pipeline_capability if TYPE_CHECKING: diff --git a/sqlspec/adapters/psqlpy/config.py b/sqlspec/adapters/psqlpy/config.py index 7f32dbf45..25f8622ba 100644 --- a/sqlspec/adapters/psqlpy/config.py +++ b/sqlspec/adapters/psqlpy/config.py @@ -504,12 +504,14 @@ def provide_session( return PsqlpySessionContext( acquire_connection=factory.acquire_connection, release_connection=factory.release_connection, - statement_config=statement_config - or (lambda: resolve_runtime_statement_config(None, self.statement_config, default_statement_config)), + statement_config=statement_config or self._default_runtime_statement_config, driver_features=self.driver_features, prepare_driver=self._prepare_driver, ) + def _default_runtime_statement_config(self) -> "StatementConfig": + return resolve_runtime_statement_config(None, self.statement_config, default_statement_config) + async def provide_pool(self, *args: Any, **kwargs: Any) -> "ConnectionPool": """Provide async pool instance. diff --git a/sqlspec/adapters/psqlpy/core.py b/sqlspec/adapters/psqlpy/core.py index 411dc1f38..7455f134a 100644 --- a/sqlspec/adapters/psqlpy/core.py +++ b/sqlspec/adapters/psqlpy/core.py @@ -29,23 +29,18 @@ TargetSessionAttrs, ) from sqlspec.adapters.psqlpy.type_converter import coerce_pgvector -from sqlspec.core import ( - DriverParameterProfile, - ParameterStyle, - ParameterStyleConfig, - StatementConfig, - build_statement_config_from_profile, -) +from sqlspec.core.cache import CachedQuery from sqlspec.core.config_runtime import ( build_postgres_extension_probe_names, is_postgres_extension_active, resolve_postgres_extension_state, resolve_runtime_statement_config, ) -from sqlspec.core.parameters import type_coercion_dispatcher, type_coercion_fallbacks +from sqlspec.core.parameters._processor import type_coercion_dispatcher, type_coercion_fallbacks +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle, ParameterStyleConfig from sqlspec.core.result._base import DMLResult, SQLResult from sqlspec.core.sqlcommenter import split_sqlcomment -from sqlspec.driver import CachedQuery from sqlspec.exceptions import ( CheckViolationError, ConnectionTimeoutError, @@ -64,10 +59,11 @@ SQLSpecError, UniqueViolationError, _classify_timeout_or_cancellation, + with_cause, ) from sqlspec.typing import PGVECTOR_INSTALLED, Empty from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import quote_identifier, split_qualified_identifier from sqlspec.utils.type_converters import build_nested_decimal_normalizer, build_uuid_coercions from sqlspec.utils.type_guards import has_query_result_metadata @@ -76,7 +72,7 @@ if TYPE_CHECKING: from collections.abc import Callable - from sqlspec.core import SQL + from sqlspec.core.statement import SQL, StatementConfig __all__ = ( "PsqlpyStreamSource", @@ -567,7 +563,7 @@ def create_mapped_exception(error: Any, *, logger: Any | None = None) -> SQLSpec ): return _create_postgres_error(error, ReadOnlyTransactionError, "read-only transaction") - if isinstance(error, PsqlpyDataError): + if isinstance(error, (PsqlpyDataError, TypeError)): return _create_postgres_error(error, DataError, "data error") if "syntax error" in error_msg or "parse error" in error_msg: @@ -704,7 +700,7 @@ def _coerce_json_parameter(value: Any, cast_type: str, serializer: "Callable[[An serialized_value = serializer(value) except Exception as error: msg = "Failed to serialize JSON parameter for psqlpy." - raise SQLSpecError(msg) from error + raise with_cause(SQLSpecError(msg), error) from error return serialized_value @@ -718,7 +714,7 @@ def _coerce_uuid_parameter(value: Any) -> Any: return uuid_from_string(value) except ValueError as error: msg = "Invalid UUID parameter for psqlpy." - raise SQLSpecError(msg) from error + raise with_cause(SQLSpecError(msg), error) from error return value @@ -733,7 +729,7 @@ def _coerce_timestamp_parameter(value: Any) -> Any: return dt.datetime.fromisoformat(normalized_value) except ValueError as error: msg = "Invalid ISO timestamp parameter for psqlpy." - raise SQLSpecError(msg) from error + raise with_cause(SQLSpecError(msg), error) from error return value diff --git a/sqlspec/adapters/psqlpy/driver.py b/sqlspec/adapters/psqlpy/driver.py index 25fae7a95..732be76b6 100644 --- a/sqlspec/adapters/psqlpy/driver.py +++ b/sqlspec/adapters/psqlpy/driver.py @@ -38,24 +38,33 @@ split_schema_and_table, ) from sqlspec.adapters.psqlpy.data_dictionary import PsqlpyDataDictionary -from sqlspec.core import SQL, DMLResult, StatementConfig, get_cache_config, register_driver_profile -from sqlspec.driver import AsyncDriverAdapterBase, AsyncRowStream, BaseAsyncExceptionHandler +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.result._base import DMLResult +from sqlspec.core.statement import SQL, StatementConfig +from sqlspec.driver._async import AsyncDriverAdapterBase from sqlspec.driver._common import cache_hit_hooks_unchanged -from sqlspec.utils.serializers import from_json +from sqlspec.driver._exception_handler import BaseAsyncExceptionHandler +from sqlspec.driver._stream import AsyncRowStream +from sqlspec.exceptions import with_cause +from sqlspec.utils.serializers._json import from_json from sqlspec.utils.text import normalize_identifier, quote_identifier if TYPE_CHECKING: from collections.abc import Callable, Mapping, Sequence + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.psqlpy._typing import PsqlpyConnection - from sqlspec.core import ArrowResult, SQLResult - from sqlspec.driver import CachedQuery, ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.core.cache import CachedQuery + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry __all__ = ("PsqlpyCursor", "PsqlpyDriver", "PsqlpyExceptionHandler", "PsqlpySessionContext") -@mypyc_attr(allow_interpreted_subclasses=False) +@mypyc_attr(allow_interpreted_subclasses=True) class PsqlpyExceptionHandler(BaseAsyncExceptionHandler): """Async context manager for handling psqlpy database exceptions. @@ -69,13 +78,13 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba if exc_type is None: return False - if issubclass(exc_type, (PsqlpyDatabaseError, PsqlpyConnectionExecuteError, PsqlpyError)): + if issubclass(exc_type, (PsqlpyDatabaseError, PsqlpyConnectionExecuteError, PsqlpyError, TypeError)): self.pending_exception = create_mapped_exception(exc_val) return True return False -@mypyc_attr(allow_interpreted_subclasses=False) +@mypyc_attr(allow_interpreted_subclasses=True) class PsqlpyDriver(AsyncDriverAdapterBase): """PostgreSQL driver implementation using psqlpy. @@ -84,7 +93,7 @@ class PsqlpyDriver(AsyncDriverAdapterBase): """ __slots__ = ("_data_dictionary", "_pooler_unprepared", "_transaction_active") - dialect = "postgres" + dialect: "DialectType | None" = "postgres" def __init__( self, @@ -220,7 +229,7 @@ async def dispatch_execute_script(self, cursor: "PsqlpyConnection", statement: S await cursor.execute(stmt, stmt_params, prepared=use_prepared) if has_observers: stmt_duration = perf_counter() - stmt_started - stmt_profile = self._pending_statement_profile + stmt_profile: dict[str, Any] | None = self._pending_statement_profile self._pending_statement_profile = None self._emit_script_statement_event( sql=stmt, @@ -248,14 +257,17 @@ async def begin(self) -> None: try: await self.connection.execute("BEGIN") except (PsqlpyDatabaseError, PsqlpyConnectionExecuteError, PsqlpyError, Exception) as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e + columnar_error: Exception | None = None try: if self._should_set_local_columnar(): await self.connection.execute("SET LOCAL google_columnar_engine.enable_columnar_scan = 'on'") except Exception as e: + columnar_error = e + if columnar_error is not None: with suppress(Exception): await self.connection.execute("ROLLBACK") - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(columnar_error), columnar_error) from columnar_error self._transaction_active = True async def commit(self) -> None: @@ -266,7 +278,7 @@ async def commit(self) -> None: try: await self.connection.execute("COMMIT") except (PsqlpyDatabaseError, PsqlpyConnectionExecuteError, PsqlpyError, Exception) as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e finally: self._transaction_active = False @@ -278,7 +290,7 @@ async def rollback(self) -> None: try: await self.connection.execute("ROLLBACK") except (PsqlpyDatabaseError, PsqlpyConnectionExecuteError, PsqlpyError, Exception) as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e finally: self._transaction_active = False @@ -396,6 +408,8 @@ async def load_from_arrow( type_casts=type_casts, ) if overwrite: + if self._auto_begin_on_write and not await self._connection_in_transaction_async(): + await self.begin() qualified = format_table_identifier(table) exc_handler = self.handle_database_exceptions() async with exc_handler, self.with_cursor(self.connection) as cursor: @@ -406,6 +420,8 @@ async def load_from_arrow( first_batch = next((b for b in batch_iter if b.num_rows > 0), None) metrics: dict[str, int] = {"rows_processed": 0, "bytes_processed": 0} if first_batch is not None: + if self._auto_begin_on_write and not await self._connection_in_transaction_async(): + await self.begin() all_batches = chain((first_batch,), (b for b in batch_iter if b.num_rows > 0)) schema_name, table_name = split_schema_and_table(table) use_binary_wire = bool(self.driver_features.get("enable_pgcopy_binary", False)) @@ -450,22 +466,6 @@ async def load_from_arrow( self._attach_partition_telemetry(telemetry_payload, partitioner) return self._storage_job(telemetry_payload, telemetry) - async def load_from_storage( - self, - table: str, - source: "StorageDestination", - *, - file_format: "StorageFormat", - partitioner: "dict[str, object] | None" = None, - overwrite: bool = False, - **kwargs: Any, - ) -> "StorageBridgeJob": - """Load staged artifacts from storage using the storage bridge pipeline.""" - - return await super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs - ) - def prepare_driver_parameters( self, parameters: Any, diff --git a/sqlspec/adapters/psqlpy/events/_hub.py b/sqlspec/adapters/psqlpy/events/_hub.py index f570cd158..5cafdf8a9 100644 --- a/sqlspec/adapters/psqlpy/events/_hub.py +++ b/sqlspec/adapters/psqlpy/events/_hub.py @@ -17,10 +17,10 @@ from typing import TYPE_CHECKING, Any, cast from weakref import WeakKeyDictionary -from sqlspec.core import SQL +from sqlspec.core.statement import SQL from sqlspec.exceptions import ImproperConfigurationError -from sqlspec.extensions.events import normalize_event_channel_name from sqlspec.extensions.events._buffer import enqueue_with_capacity, resolve_listener_queue_capacity +from sqlspec.extensions.events._store import normalize_event_channel_name from sqlspec.utils.logging import get_logger, log_with_context from sqlspec.utils.sync_tools import exit_async_context from sqlspec.utils.type_guards import is_notification diff --git a/sqlspec/adapters/psqlpy/tenant.py b/sqlspec/adapters/psqlpy/tenant.py index cd09f9c11..dda6bb6db 100644 --- a/sqlspec/adapters/psqlpy/tenant.py +++ b/sqlspec/adapters/psqlpy/tenant.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from sqlspec.adapters.psqlpy._typing import PsqlpyConnection from sqlspec.core.tenant.base import AsyncTenantStrategy - from sqlspec.utils.tenant import TenantContextPayload + from sqlspec.utils.tenant._context import TenantContextPayload PSQLPY_DISPOSAL_TIMEOUT_S: Final = 2.0 _IDLE_PROBE: Final = "SELECT transaction_timestamp() = statement_timestamp()" diff --git a/sqlspec/adapters/psycopg/config.py b/sqlspec/adapters/psycopg/config.py index 4eba53fbd..5b0dde90a 100644 --- a/sqlspec/adapters/psycopg/config.py +++ b/sqlspec/adapters/psycopg/config.py @@ -606,12 +606,14 @@ def provide_session( return PsycopgSyncSessionContext( acquire_connection=handler.acquire_connection, release_connection=handler.release_connection, - statement_config=statement_config - or (lambda: resolve_runtime_statement_config(None, self.statement_config, default_statement_config)), + statement_config=statement_config or self._default_runtime_statement_config, driver_features=self.driver_features, prepare_driver=self._prepare_driver, ) + def _default_runtime_statement_config(self) -> "StatementConfig": + return resolve_runtime_statement_config(None, self.statement_config, default_statement_config) + def provide_pool(self, *args: Any, **kwargs: Any) -> "ConnectionPool": """Provide pool instance. @@ -1184,12 +1186,14 @@ def provide_session( return PsycopgAsyncSessionContext( acquire_connection=handler.acquire_connection, release_connection=handler.release_connection, - statement_config=statement_config - or (lambda: resolve_runtime_statement_config(None, self.statement_config, default_statement_config)), + statement_config=statement_config or self._default_runtime_statement_config, driver_features=self.driver_features, prepare_driver=self._prepare_driver, ) + def _default_runtime_statement_config(self) -> "StatementConfig": + return resolve_runtime_statement_config(None, self.statement_config, default_statement_config) + async def provide_pool(self, *args: Any, **kwargs: Any) -> "AsyncConnectionPool": """Provide async pool instance. diff --git a/sqlspec/adapters/psycopg/core.py b/sqlspec/adapters/psycopg/core.py index 0dc224c2c..da786aee4 100644 --- a/sqlspec/adapters/psycopg/core.py +++ b/sqlspec/adapters/psycopg/core.py @@ -8,16 +8,11 @@ from sqlspec.adapters.psycopg._typing import PsycopgComposed, PsycopgIdentifier, PsycopgSQL from sqlspec.adapters.psycopg._typing import psycopg_errors as pg_errors from sqlspec.adapters.psycopg._typing import psycopg_module as psycopg -from sqlspec.core import ( - SQL, - DriverParameterProfile, - ParameterStyle, - ParameterValidator, - StatementConfig, - build_statement_config_from_profile, -) +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult -from sqlspec.driver import ExecutionResult, rows_to_dicts +from sqlspec.driver._common import ExecutionResult +from sqlspec.driver._stream import rows_to_dicts from sqlspec.exceptions import ( CheckViolationError, ConnectionTimeoutError, @@ -42,7 +37,7 @@ map_sqlstate_to_exception, ) from sqlspec.typing import PGVECTOR_INSTALLED -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import split_qualified_identifier from sqlspec.utils.type_converters import build_json_list_converter, build_json_tuple_converter, build_uuid_coercions from sqlspec.utils.type_guards import has_rowcount, has_sqlstate, resolve_row_format @@ -51,8 +46,11 @@ if TYPE_CHECKING: from collections.abc import Callable, Mapping - from sqlspec.core import ParameterStyleConfig, StackOperation from sqlspec.core.cache import CachedQuery + from sqlspec.core.parameters._types import ParameterStyleConfig + from sqlspec.core.parameters._validator import ParameterValidator + from sqlspec.core.stack import StackOperation + from sqlspec.core.statement import SQL, StatementConfig __all__ = ( "OPEN_TRANSACTION_STATUSES", @@ -106,7 +104,7 @@ class PreparedStackOperation(NamedTuple): operation: "StackOperation" statement: "SQL" sql: "LiteralString | PsycopgSQL | PsycopgComposed" - parameters: "tuple[Any, ...] | dict[str, Any] | None" + parameters: "tuple[Any, ...] | list[Any] | dict[str, Any] | None" class PipelineCursorEntry(NamedTuple): @@ -669,11 +667,12 @@ def build_pipeline_execution_result( if statement.returns_rows() or cursor.description is not None: fetched_data = cast("list[Any] | None", cursor.fetchall()) or [] + description = cursor.description if column_name_resolver is None: - fetched_data, column_names = collect_rows(fetched_data, cursor.description) + fetched_data, column_names = collect_rows(fetched_data, description) else: - column_names = column_name_resolver(cursor.description) - column_types = column_types_resolver(cursor.description) if column_types_resolver is not None else None + column_names = column_name_resolver(description) + column_types = column_types_resolver(description) if column_types_resolver is not None else None row_format = resolve_row_format(fetched_data) return ExecutionResult( cursor_result=cursor, @@ -730,11 +729,12 @@ async def build_async_pipeline_execution_result( if statement.returns_rows() or cursor.description is not None: fetched_data = cast("list[Any] | None", await cursor.fetchall()) or [] + description = cursor.description if column_name_resolver is None: - fetched_data, column_names = collect_rows(fetched_data, cursor.description) + fetched_data, column_names = collect_rows(fetched_data, description) else: - column_names = column_name_resolver(cursor.description) - column_types = column_types_resolver(cursor.description) if column_types_resolver is not None else None + column_names = column_name_resolver(description) + column_types = column_types_resolver(description) if column_types_resolver is not None else None row_format = resolve_row_format(fetched_data) return ExecutionResult( cursor_result=cursor, @@ -842,6 +842,8 @@ def source_uses_pyformat(raw_sql: str, validator: "ParameterValidator") -> bool: Returns: ``True`` when the text contains a pyformat placeholder outside literals and comments. """ + if "%s" not in raw_sql and "%S" not in raw_sql and "%(" not in raw_sql: + return False return any( info.style in {ParameterStyle.POSITIONAL_PYFORMAT, ParameterStyle.NAMED_PYFORMAT} for info in validator.extract_parameters(raw_sql, dialect="postgres") @@ -867,15 +869,22 @@ def escape_literal_percent( return sql keep = {info.position for info in validator.extract_parameters(sql, dialect="postgres")} segments: list[str] = [] - index = 0 - while index < len(sql): - if preserve_percent_escapes and sql[index : index + 2] == "%%": - segments.append("%%") + copied = 0 + index = sql.find("%") + while index != -1: + if preserve_percent_escapes and sql.startswith("%%", index): index += 2 - continue - char = sql[index] - segments.append("%%" if char == "%" and index not in keep else char) - index += 1 + elif index in keep: + index += 1 + else: + segments.append(sql[copied:index]) + segments.append("%%") + index += 1 + copied = index + index = sql.find("%", index) + if not segments: + return sql + segments.append(sql[copied:]) return "".join(segments) diff --git a/sqlspec/adapters/psycopg/driver.py b/sqlspec/adapters/psycopg/driver.py index 5f1782649..0f16cdf5e 100644 --- a/sqlspec/adapters/psycopg/driver.py +++ b/sqlspec/adapters/psycopg/driver.py @@ -6,6 +6,8 @@ from time import perf_counter from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, LiteralString, cast +from mypy_extensions import mypyc_attr + from sqlspec.adapters.psycopg._typing import ( PsycopgAsyncConnection, PsycopgAsyncCursor, @@ -43,41 +45,33 @@ source_uses_pyformat, ) from sqlspec.adapters.psycopg.data_dictionary import PsycopgAsyncDataDictionary, PsycopgSyncDataDictionary -from sqlspec.core import ( - SQL, - SQLResult, - StackResult, - StatementConfig, - StatementStack, - get_cache_config, - is_copy_from_operation, - is_copy_operation, - is_copy_to_operation, - register_driver_profile, -) -from sqlspec.driver import ( - AsyncDriverAdapterBase, - AsyncRowStream, - BaseAsyncExceptionHandler, - BaseSyncExceptionHandler, - StackExecutionObserver, - SyncDriverAdapterBase, - SyncRowStream, - describe_stack_statement, -) -from sqlspec.driver._common import cache_hit_hooks_unchanged -from sqlspec.exceptions import ImproperConfigurationError, SQLSpecError, StackExecutionError +from sqlspec.core.cache import get_cache_config +from sqlspec.core.compiler import is_copy_from_operation, is_copy_operation, is_copy_to_operation +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.result._base import SQLResult, StackResult +from sqlspec.core.stack import StatementStack +from sqlspec.driver._async import AsyncDriverAdapterBase +from sqlspec.driver._common import StackExecutionObserver, cache_hit_hooks_unchanged, describe_stack_statement +from sqlspec.driver._exception_handler import BaseAsyncExceptionHandler, BaseSyncExceptionHandler +from sqlspec.driver._stream import AsyncRowStream, SyncRowStream +from sqlspec.driver._sync import SyncDriverAdapterBase +from sqlspec.exceptions import ImproperConfigurationError, SQLSpecError, StackExecutionError, with_cause from sqlspec.utils.logging import get_logger +from sqlspec.utils.sync_tools import exit_async_context from sqlspec.utils.text import normalize_identifier from sqlspec.utils.type_guards import is_readable, resolve_row_format if TYPE_CHECKING: from collections import abc + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.psycopg._typing import PsycopgPipelineDriver - from sqlspec.core import ArrowResult - from sqlspec.driver import CachedQuery, ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.core.cache import CachedQuery + from sqlspec.core.result._base import ArrowResult + from sqlspec.core.statement import SQL, StatementConfig + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry from sqlspec.typing import ArrowRecordBatch @@ -132,59 +126,54 @@ def pipeline_operation_failed(cursor: Any, statement: "SQL") -> bool: return isinstance(rowcount, int) and rowcount < 0 -class PsycopgPipelineMixin: - """Shared helpers for psycopg sync/async pipeline execution.""" +def _psycopg_prepare_records_for_arrow( + driver: "PsycopgPipelineDriver", records: "abc.Sequence[abc.Mapping[str, Any]] | abc.Sequence[abc.Sequence[Any]]" +) -> "abc.Sequence[abc.Mapping[str, Any]] | abc.Sequence[abc.Sequence[Any]]": + return cast( + "abc.Sequence[abc.Mapping[str, Any]] | abc.Sequence[abc.Sequence[Any]]", + driver.prepare_driver_parameters(records, driver.statement_config, is_many=True), + ) - __slots__ = () - def _prepare_records_for_arrow( - self, records: "abc.Sequence[abc.Mapping[str, Any]] | abc.Sequence[abc.Sequence[Any]]" - ) -> "abc.Sequence[abc.Mapping[str, Any]] | abc.Sequence[abc.Sequence[Any]]": - driver = cast("PsycopgPipelineDriver", self) - return cast( - "abc.Sequence[abc.Mapping[str, Any]] | abc.Sequence[abc.Sequence[Any]]", - driver.prepare_driver_parameters(records, driver.statement_config, is_many=True), - ) +def _psycopg_prepare_pipeline_operations( + driver: "PsycopgPipelineDriver", stack: "StatementStack" +) -> "list[PreparedStackOperation] | None": + prepared: list[PreparedStackOperation] = [] + for index, operation in enumerate(stack.operations): + if operation.method != "execute": + return None - def _prepare_pipeline_operations(self, stack: "StatementStack") -> "list[PreparedStackOperation] | None": - prepared: list[PreparedStackOperation] = [] - for index, operation in enumerate(stack.operations): - if operation.method != "execute": - return None - - kwargs = dict(operation.keyword_arguments) if operation.keyword_arguments else {} - statement_config = kwargs.pop("statement_config", None) - driver = cast("PsycopgPipelineDriver", self) - config = statement_config or driver.statement_config - - sql_statement = driver.prepare_statement( - operation.statement, operation.arguments, statement_config=config, kwargs=kwargs - ) + kwargs = dict(operation.keyword_arguments) if operation.keyword_arguments else {} + statement_config = kwargs.pop("statement_config", None) + config = statement_config or driver.statement_config - if sql_statement.is_script or sql_statement.is_many: - return None + sql_statement = driver.prepare_statement( + operation.statement, operation.arguments, statement_config=config, kwargs=kwargs + ) - sql_text, prepared_parameters = driver._compiled_sql( # pyright: ignore[reportPrivateUsage] - sql_statement, config - ) - sql_text = escape_literal_percent( - sql_text, - prepared_parameters, - config.parameter_validator, - preserve_percent_escapes=source_uses_pyformat(sql_statement.raw_sql, config.parameter_validator), - ) - prepared.append( - PreparedStackOperation( - operation_index=index, - operation=operation, - statement=sql_statement, - sql=cast("LiteralString | PsycopgSQL | PsycopgComposed", sql_text), - parameters=prepared_parameters, - ) + if sql_statement.is_script or sql_statement.is_many: + return None + + sql_text, prepared_parameters = driver._compiled_sql(sql_statement, config) + sql_text = escape_literal_percent( + sql_text, + prepared_parameters, + config.parameter_validator, + preserve_percent_escapes=source_uses_pyformat(sql_statement.raw_sql, config.parameter_validator), + ) + prepared.append( + PreparedStackOperation( + operation_index=index, + operation=operation, + statement=sql_statement, + sql=cast("LiteralString | PsycopgSQL | PsycopgComposed", sql_text), + parameters=prepared_parameters, ) - return prepared + ) + return prepared +@mypyc_attr(allow_interpreted_subclasses=True) class PsycopgSyncExceptionHandler(BaseSyncExceptionHandler): """Context manager for handling PostgreSQL psycopg database exceptions. @@ -203,7 +192,8 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False -class PsycopgSyncDriver(PsycopgPipelineMixin, SyncDriverAdapterBase): +@mypyc_attr(allow_interpreted_subclasses=True) +class PsycopgSyncDriver(SyncDriverAdapterBase): """PostgreSQL psycopg synchronous driver. Provides synchronous database operations for PostgreSQL using psycopg3. @@ -214,7 +204,7 @@ class PsycopgSyncDriver(PsycopgPipelineMixin, SyncDriverAdapterBase): """ __slots__ = ("_data_dictionary", "_restore_autocommit", "_transaction_active") - dialect = "postgres" + dialect: "DialectType | None" = "postgres" cache_hit_function: "ClassVar[Any]" = staticmethod(execute_cache_hit) def __init__( @@ -270,8 +260,9 @@ def dispatch_execute(self, cursor: Any, statement: "SQL") -> "ExecutionResult": if statement.returns_rows() or cursor.description is not None: fetched_data = cursor.fetchall() data = cast("list[Any] | None", fetched_data) or [] - column_names = self._resolve_column_names(cursor.description) - column_types = _resolve_column_types(cursor.description) + description = cursor.description + column_names = self._resolve_column_names(description) + column_types = _resolve_column_types(description) row_format = resolve_row_format(data) return self.create_execution_result( @@ -351,7 +342,7 @@ def dispatch_execute_script(self, cursor: Any, statement: "SQL") -> "ExecutionRe execute_with_optional_parameters(cursor, stmt_sql, prepared_parameters) if has_observers: stmt_duration = perf_counter() - stmt_started - stmt_profile = self._pending_statement_profile + stmt_profile: dict[str, Any] | None = self._pending_statement_profile self._pending_statement_profile = None self._emit_script_statement_event( sql=stmt, @@ -454,9 +445,12 @@ def begin(self) -> None: self.connection.autocommit = True mapped = create_mapped_exception(e) if type(mapped) is not SQLSpecError: + mapped.__cause__ = e raise mapped from e msg = f"Failed to begin transaction: {e}" - raise SQLSpecError(msg) from e + error = SQLSpecError(msg) + error.__cause__ = e + raise error from e self._restore_autocommit = restore_autocommit self._transaction_active = True @@ -467,9 +461,12 @@ def commit(self) -> None: except psycopg.Error as e: mapped = create_mapped_exception(e) if type(mapped) is not SQLSpecError: + mapped.__cause__ = e raise mapped from e msg = f"Failed to commit transaction: {e}" - raise SQLSpecError(msg) from e + error = SQLSpecError(msg) + error.__cause__ = e + raise error from e self._transaction_active = False self._restore_original_autocommit() @@ -480,9 +477,12 @@ def rollback(self) -> None: except psycopg.Error as e: mapped = create_mapped_exception(e) if type(mapped) is not SQLSpecError: + mapped.__cause__ = e raise mapped from e msg = f"Failed to rollback transaction: {e}" - raise SQLSpecError(msg) from e + error = SQLSpecError(msg) + error.__cause__ = e + raise error from e self._transaction_active = False self._restore_original_autocommit() @@ -566,6 +566,14 @@ def handle_database_exceptions(self) -> "PsycopgSyncExceptionHandler": """Handle database-specific exceptions and wrap them appropriately.""" return PsycopgSyncExceptionHandler() + def _prepare_records_for_arrow( + self, records: "abc.Sequence[abc.Mapping[str, Any]] | abc.Sequence[abc.Sequence[Any]]" + ) -> "abc.Sequence[abc.Mapping[str, Any]] | abc.Sequence[abc.Sequence[Any]]": + return _psycopg_prepare_records_for_arrow(self, records) + + def _prepare_pipeline_operations(self, stack: "StatementStack") -> "list[PreparedStackOperation] | None": + return _psycopg_prepare_pipeline_operations(self, stack) + def execute_stack(self, stack: "StatementStack", *, continue_on_error: bool = False) -> "tuple[StackResult, ...]": """Execute a StatementStack using psycopg pipeline mode when supported.""" @@ -739,7 +747,7 @@ def _execute_stack_pipeline( cursor.execute(sql) except Exception as exc: owner = _attribute_pipeline_failure(pending, prepared) - raise _stack_operation_error(type(self).__name__, owner, exc) from exc + raise with_cause(_stack_operation_error(type(self).__name__, owner, exc), exc) from exc pending.append(PipelineCursorEntry(prepared=prepared, cursor=cursor)) @@ -772,7 +780,9 @@ def _collect_pipeline_results( statement = entry.prepared.statement cursor = entry.cursor if sync_error is not None and pipeline_operation_failed(cursor, statement): - raise _stack_operation_error(type(self).__name__, entry.prepared, sync_error) from sync_error + raise with_cause( + _stack_operation_error(type(self).__name__, entry.prepared, sync_error), sync_error + ) from sync_error try: execution_result = build_pipeline_execution_result( statement, @@ -781,12 +791,12 @@ def _collect_pipeline_results( column_types_resolver=_resolve_column_types, ) except Exception as exc: - raise _stack_operation_error(type(self).__name__, entry.prepared, exc) from exc + raise with_cause(_stack_operation_error(type(self).__name__, entry.prepared, exc), exc) from exc sql_result = self.build_statement_result(statement, execution_result) results.append(StackResult.from_sql_result(sql_result)) if sync_error is not None: - raise _stack_pipeline_sync_error(type(self).__name__, sync_error) from sync_error + raise with_cause(_stack_pipeline_sync_error(type(self).__name__, sync_error), sync_error) from sync_error return results def _resolve_column_names(self, description: Any) -> list[str]: @@ -811,9 +821,12 @@ def _restore_original_autocommit(self) -> None: except psycopg.Error as e: mapped = create_mapped_exception(e) if type(mapped) is not SQLSpecError: + mapped.__cause__ = e raise mapped from e msg = f"Failed to restore autocommit: {e}" - raise SQLSpecError(msg) from e + error = SQLSpecError(msg) + error.__cause__ = e + raise error from e def _should_set_local_columnar(self) -> bool: """Return True when AlloyDB columnar GUC must be set transaction-locally.""" @@ -822,6 +835,7 @@ def _should_set_local_columnar(self) -> bool: return self.driver_features.get("pooler_mode") in {"transaction", "statement"} +@mypyc_attr(allow_interpreted_subclasses=True) class PsycopgAsyncExceptionHandler(BaseAsyncExceptionHandler): """Async context manager for handling PostgreSQL psycopg database exceptions. @@ -840,7 +854,8 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False -class PsycopgAsyncDriver(PsycopgPipelineMixin, AsyncDriverAdapterBase): +@mypyc_attr(allow_interpreted_subclasses=True) +class PsycopgAsyncDriver(AsyncDriverAdapterBase): """PostgreSQL psycopg asynchronous driver. Provides asynchronous database operations for PostgreSQL using psycopg3. @@ -852,7 +867,7 @@ class PsycopgAsyncDriver(PsycopgPipelineMixin, AsyncDriverAdapterBase): """ __slots__ = ("_data_dictionary", "_restore_autocommit", "_transaction_active") - dialect = "postgres" + dialect: "DialectType | None" = "postgres" cache_hit_function: "ClassVar[Any]" = staticmethod(execute_cache_hit_async) def __init__( @@ -908,8 +923,9 @@ async def dispatch_execute(self, cursor: Any, statement: "SQL") -> "ExecutionRes if statement.returns_rows() or cursor.description is not None: fetched_data = await cursor.fetchall() data = cast("list[Any] | None", fetched_data) or [] - column_names = self._resolve_column_names(cursor.description) - column_types = _resolve_column_types(cursor.description) + description = cursor.description + column_names = self._resolve_column_names(description) + column_types = _resolve_column_types(description) row_format = resolve_row_format(data) return self.create_execution_result( @@ -989,7 +1005,7 @@ async def dispatch_execute_script(self, cursor: Any, statement: "SQL") -> "Execu await execute_with_optional_parameters_async(cursor, stmt_sql, prepared_parameters) if has_observers: stmt_duration = perf_counter() - stmt_started - stmt_profile = self._pending_statement_profile + stmt_profile: dict[str, Any] | None = self._pending_statement_profile self._pending_statement_profile = None self._emit_script_statement_event( sql=stmt, @@ -1029,8 +1045,11 @@ async def dispatch_special_handling(self, cursor: Any, statement: "SQL") -> "SQL if isinstance(copy_data, list) and len(copy_data) == 1: copy_data = copy_data[0] + copy_error: BaseException | None = None if is_copy_from_operation(operation_type): - async with cursor.copy(sql) as copy_ctx: + copy_context = cursor.copy(sql) + copy_ctx = await copy_context.__aenter__() + try: if is_readable(copy_data): chunk_size = 65536 while chunk := copy_data.read(chunk_size): @@ -1042,6 +1061,9 @@ async def dispatch_special_handling(self, cursor: Any, statement: "SQL") -> "SQL if isinstance(data_to_write, str): data_to_write = data_to_write.encode("utf-8") await copy_ctx.write(data_to_write) + except BaseException as exc: + copy_error = exc + await exit_async_context(copy_context, copy_error) rows_affected = max(cursor.rowcount, 0) @@ -1051,11 +1073,16 @@ async def dispatch_special_handling(self, cursor: Any, statement: "SQL") -> "SQL if is_copy_to_operation(operation_type): output_data: list[str] = [] - async with cursor.copy(sql) as copy_ctx: + copy_context = cursor.copy(sql) + copy_ctx = await copy_context.__aenter__() + try: output_data.extend([ bytes(row).decode("utf-8") if isinstance(row, (bytes, bytearray, memoryview)) else str(row) async for row in copy_ctx ]) + except BaseException as exc: + copy_error = exc + await exit_async_context(copy_context, copy_error) exported_data = "".join(output_data) rows_affected = max(cursor.rowcount, 0) @@ -1081,6 +1108,7 @@ async def begin(self) -> None: if self._connection_in_transaction(): return restore_autocommit = False + begin_error: psycopg.Error | None = None try: restore_autocommit = self.connection.autocommit if restore_autocommit: @@ -1088,14 +1116,19 @@ async def begin(self) -> None: if self._should_set_local_columnar(): await self.connection.execute("SET LOCAL google_columnar_engine.enable_columnar_scan = 'on'") except psycopg.Error as e: + begin_error = e + if begin_error is not None: if restore_autocommit: with suppress(Exception): await self.connection.set_autocommit(True) - mapped = create_mapped_exception(e) + mapped = create_mapped_exception(begin_error) if type(mapped) is not SQLSpecError: - raise mapped from e - msg = f"Failed to begin transaction: {e}" - raise SQLSpecError(msg) from e + mapped.__cause__ = begin_error + raise mapped from begin_error + msg = f"Failed to begin transaction: {begin_error}" + error = SQLSpecError(msg) + error.__cause__ = begin_error + raise error from begin_error self._restore_autocommit = restore_autocommit self._transaction_active = True @@ -1106,9 +1139,12 @@ async def commit(self) -> None: except psycopg.Error as e: mapped = create_mapped_exception(e) if type(mapped) is not SQLSpecError: + mapped.__cause__ = e raise mapped from e msg = f"Failed to commit transaction: {e}" - raise SQLSpecError(msg) from e + error = SQLSpecError(msg) + error.__cause__ = e + raise error from e self._transaction_active = False await self._restore_original_autocommit() @@ -1119,9 +1155,12 @@ async def rollback(self) -> None: except psycopg.Error as e: mapped = create_mapped_exception(e) if type(mapped) is not SQLSpecError: + mapped.__cause__ = e raise mapped from e msg = f"Failed to rollback transaction: {e}" - raise SQLSpecError(msg) from e + error = SQLSpecError(msg) + error.__cause__ = e + raise error from e self._transaction_active = False await self._restore_original_autocommit() @@ -1323,21 +1362,35 @@ async def _copy_arrow_batches( metrics: dict[str, int], use_binary: bool, ) -> None: - async with self.with_cursor(self.connection) as cursor, cursor.copy(copy_sql) as copy_ctx: - if use_binary: - for chunk in self._iter_pgcopy_binary_chunks(batches, columns=columns, metrics=metrics): - await copy_ctx.write(chunk) - else: - for batch in batches: - metrics["rows_processed"] += int(batch.num_rows) - metrics["bytes_processed"] += int(batch.nbytes) - records = self._arrow_batch_to_rows(batch, columns) - if self._arrow_rows_need_preparation(batch): - records = cast( - "list[Any]", self.prepare_driver_parameters(records, self.statement_config, is_many=True) - ) - for record in records: - await copy_ctx.write_row(record) + cursor_context = self.with_cursor(self.connection) + cursor = await cursor_context.__aenter__() + cursor_error: BaseException | None = None + try: + copy_context = cursor.copy(copy_sql) + copy_ctx = await copy_context.__aenter__() + copy_error: BaseException | None = None + try: + if use_binary: + for chunk in self._iter_pgcopy_binary_chunks(batches, columns=columns, metrics=metrics): + await copy_ctx.write(chunk) + else: + for batch in batches: + metrics["rows_processed"] += int(batch.num_rows) + metrics["bytes_processed"] += int(batch.nbytes) + records = self._arrow_batch_to_rows(batch, columns) + if self._arrow_rows_need_preparation(batch): + records = cast( + "list[Any]", + self.prepare_driver_parameters(records, self.statement_config, is_many=True), + ) + for record in records: + await copy_ctx.write_row(record) + except BaseException as exc: + copy_error = exc + await exit_async_context(copy_context, copy_error) + except BaseException as exc: + cursor_error = exc + await exit_async_context(cursor_context, cursor_error) def collect_rows(self, cursor: Any, fetched: "list[Any]") -> "tuple[list[Any], list[str], int]": """Collect psycopg async rows for the direct execution path.""" @@ -1360,6 +1413,14 @@ async def _execute_cache_hit( return await self._execute_cached_statement(statement) return await super()._execute_cache_hit(sql, params, cached) + def _prepare_records_for_arrow( + self, records: "abc.Sequence[abc.Mapping[str, Any]] | abc.Sequence[abc.Sequence[Any]]" + ) -> "abc.Sequence[abc.Mapping[str, Any]] | abc.Sequence[abc.Sequence[Any]]": + return _psycopg_prepare_records_for_arrow(self, records) + + def _prepare_pipeline_operations(self, stack: "StatementStack") -> "list[PreparedStackOperation] | None": + return _psycopg_prepare_pipeline_operations(self, stack) + async def _execute_stack_pipeline( self, stack: "StatementStack", prepared_ops: "list[PreparedStackOperation]" ) -> "tuple[StackResult, ...]": @@ -1367,12 +1428,16 @@ async def _execute_stack_pipeline( started_transaction = False with StackExecutionObserver(self, stack, continue_on_error=False, native_pipeline=True): + pipeline_error: Exception | None = None try: if not self._connection_in_transaction(): await self.begin() started_transaction = True - async with AsyncExitStack() as resource_stack: + resource_stack = AsyncExitStack() + await resource_stack.__aenter__() + resource_error: BaseException | None = None + try: pipeline = await resource_stack.enter_async_context(self.connection.pipeline()) pending: list[PipelineCursorEntry] = [] @@ -1387,7 +1452,7 @@ async def _execute_stack_pipeline( await cursor.execute(sql) except Exception as exc: owner = _attribute_pipeline_failure(pending, prepared) - raise _stack_operation_error(type(self).__name__, owner, exc) from exc + raise with_cause(_stack_operation_error(type(self).__name__, owner, exc), exc) from exc pending.append(PipelineCursorEntry(prepared=prepared, cursor=cursor)) @@ -1398,16 +1463,21 @@ async def _execute_stack_pipeline( sync_error = exc results = await self._collect_pipeline_results(pending, sync_error) + except BaseException as caught: + resource_error = caught + await exit_async_context(resource_stack, resource_error) if started_transaction: await self.commit() - except Exception: + except Exception as exc: + pipeline_error = exc + if pipeline_error is not None: if started_transaction: try: await self.rollback() - except Exception as rollback_error: # pragma: no cover + except Exception as rollback_error: logger.debug("Rollback after psycopg pipeline failure failed: %s", rollback_error) - raise + raise pipeline_error return tuple(results) @@ -1420,7 +1490,9 @@ async def _collect_pipeline_results( statement = entry.prepared.statement cursor = entry.cursor if sync_error is not None and pipeline_operation_failed(cursor, statement): - raise _stack_operation_error(type(self).__name__, entry.prepared, sync_error) from sync_error + raise with_cause( + _stack_operation_error(type(self).__name__, entry.prepared, sync_error), sync_error + ) from sync_error try: execution_result = await build_async_pipeline_execution_result( statement, @@ -1429,12 +1501,12 @@ async def _collect_pipeline_results( column_types_resolver=_resolve_column_types, ) except Exception as exc: - raise _stack_operation_error(type(self).__name__, entry.prepared, exc) from exc + raise with_cause(_stack_operation_error(type(self).__name__, entry.prepared, exc), exc) from exc sql_result = self.build_statement_result(statement, execution_result) results.append(StackResult.from_sql_result(sql_result)) if sync_error is not None: - raise _stack_pipeline_sync_error(type(self).__name__, sync_error) from sync_error + raise with_cause(_stack_pipeline_sync_error(type(self).__name__, sync_error), sync_error) from sync_error return results def _resolve_column_names(self, description: Any) -> list[str]: @@ -1459,9 +1531,12 @@ async def _restore_original_autocommit(self) -> None: except psycopg.Error as e: mapped = create_mapped_exception(e) if type(mapped) is not SQLSpecError: + mapped.__cause__ = e raise mapped from e msg = f"Failed to restore autocommit: {e}" - raise SQLSpecError(msg) from e + error = SQLSpecError(msg) + error.__cause__ = e + raise error from e def _should_set_local_columnar(self) -> bool: """Return True when AlloyDB columnar GUC must be set transaction-locally.""" @@ -1492,7 +1567,7 @@ def _stack_operation_error( ) -> "StackExecutionError": """Build a fail-fast StackExecutionError attributed to a specific pipeline operation.""" mapped_cause = create_mapped_exception(cause) if isinstance(cause, psycopg.Error) else cause - return StackExecutionError( + wrapped = StackExecutionError( prepared.operation_index, describe_stack_statement(prepared.operation.statement), mapped_cause, @@ -1500,14 +1575,18 @@ def _stack_operation_error( mode="fail-fast", native_pipeline=True, ) + wrapped.__cause__ = cause + return wrapped def _stack_pipeline_sync_error(adapter: str, cause: "Exception") -> "StackExecutionError": """Build a fail-fast StackExecutionError for an unattributed pipeline sync failure.""" mapped_cause = create_mapped_exception(cause) if isinstance(cause, psycopg.Error) else cause - return StackExecutionError( + wrapped = StackExecutionError( -1, "psycopg pipeline sync failed", mapped_cause, adapter=adapter, mode="fail-fast", native_pipeline=True ) + wrapped.__cause__ = cause + return wrapped register_driver_profile("psycopg", driver_profile) diff --git a/sqlspec/adapters/psycopg/events/_hub.py b/sqlspec/adapters/psycopg/events/_hub.py index e87a6df11..9c1c3fd5b 100644 --- a/sqlspec/adapters/psycopg/events/_hub.py +++ b/sqlspec/adapters/psycopg/events/_hub.py @@ -22,8 +22,8 @@ from sqlspec.adapters.psycopg._typing import PsycopgComposed, PsycopgIdentifier, PsycopgSQL from sqlspec.exceptions import ImproperConfigurationError -from sqlspec.extensions.events import normalize_event_channel_name from sqlspec.extensions.events._buffer import enqueue_with_capacity, resolve_listener_queue_capacity +from sqlspec.extensions.events._store import normalize_event_channel_name from sqlspec.utils.logging import get_logger, log_with_context from sqlspec.utils.type_guards import has_notifies diff --git a/sqlspec/adapters/psycopg/tenant.py b/sqlspec/adapters/psycopg/tenant.py index bb3397fe0..02ec68e8f 100644 --- a/sqlspec/adapters/psycopg/tenant.py +++ b/sqlspec/adapters/psycopg/tenant.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: from sqlspec.adapters.psycopg._typing import PsycopgAsyncConnection, PsycopgSyncConnection from sqlspec.core.tenant.base import AsyncTenantStrategy, SyncTenantStrategy - from sqlspec.utils.tenant import TenantContextPayload + from sqlspec.utils.tenant._context import TenantContextPayload __all__ = ( "PsycopgAsyncRLSStrategy", diff --git a/sqlspec/adapters/pymssql/core.py b/sqlspec/adapters/pymssql/core.py index b121663e5..d43e9c729 100644 --- a/sqlspec/adapters/pymssql/core.py +++ b/sqlspec/adapters/pymssql/core.py @@ -5,7 +5,8 @@ from typing import TYPE_CHECKING, Any, Final, Literal from sqlspec.adapters.pymssql._typing import PymssqlError -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult from sqlspec.exceptions import ( CheckViolationError, @@ -28,7 +29,7 @@ UniqueViolationError, ) from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import split_qualified_identifier from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_rowcount, resolve_row_format @@ -38,7 +39,8 @@ from logging import Logger from sqlspec.core.cache import CachedQuery - from sqlspec.core.parameters import ParameterValidator + from sqlspec.core.parameters._validator import ParameterValidator + from sqlspec.core.statement import StatementConfig __all__ = ( "apply_driver_features", diff --git a/sqlspec/adapters/pymssql/driver.py b/sqlspec/adapters/pymssql/driver.py index 9cb41e596..6cb755deb 100644 --- a/sqlspec/adapters/pymssql/driver.py +++ b/sqlspec/adapters/pymssql/driver.py @@ -4,6 +4,8 @@ from collections.abc import Sized from typing import TYPE_CHECKING, Any, Literal, cast +from mypy_extensions import mypyc_attr + from sqlspec.adapters.pymssql._typing import ( PymssqlConnection, PymssqlCursor, @@ -28,27 +30,32 @@ resolve_rowcount, ) from sqlspec.adapters.pymssql.data_dictionary import PymssqlSyncDataDictionary -from sqlspec.core import SQL, StatementConfig, get_cache_config, register_driver_profile -from sqlspec.driver import ( - BaseSyncExceptionHandler, +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.driver._common import ( ExecutionResult, - SyncDriverAdapterBase, - SyncRowStream, - rows_to_dicts, + cache_hit_hooks_unchanged, + validate_load_from_storage_kwargs, validate_savepoint_name, ) -from sqlspec.driver._common import cache_hit_hooks_unchanged +from sqlspec.driver._exception_handler import BaseSyncExceptionHandler +from sqlspec.driver._stream import SyncRowStream, rows_to_dicts +from sqlspec.driver._sync import SyncDriverAdapterBase from sqlspec.exceptions import SQLSpecError from sqlspec.utils.logging import get_logger if TYPE_CHECKING: from collections.abc import Mapping, Sequence + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.pymssql._typing import PymssqlQueryParams as QueryParams - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, SQLResult, Statement, StatementFilter - from sqlspec.driver import CachedQuery - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.cache import CachedQuery + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.core.statement import SQL, Statement, StatementConfig + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry from sqlspec.typing import StatementParameters __all__ = ("PymssqlCursor", "PymssqlDriver", "PymssqlExceptionHandler", "PymssqlSessionContext") @@ -56,6 +63,7 @@ logger = get_logger("sqlspec.adapters.pymssql") +@mypyc_attr(allow_interpreted_subclasses=True) class PymssqlExceptionHandler(BaseSyncExceptionHandler): """Context manager for handling pymssql exceptions.""" @@ -127,6 +135,7 @@ def close(self, error: bool = False) -> None: cursor_manager.__exit__(None, None, None) +@mypyc_attr(allow_interpreted_subclasses=True) class PymssqlDriver(SyncDriverAdapterBase): """SQL Server database driver using pymssql.""" @@ -137,7 +146,7 @@ class PymssqlDriver(SyncDriverAdapterBase): "_migration_schema_restore", "_transaction_active", ) - dialect = "tsql" + dialect: "DialectType | None" = "tsql" def __init__( self, @@ -234,7 +243,9 @@ def begin(self) -> None: self._transaction_active = True except PymssqlError as exc: msg = f"Failed to begin SQL Server transaction: {exc}" - raise SQLSpecError(msg) from exc + error = SQLSpecError(msg) + error.__cause__ = exc + raise error from exc def commit(self) -> None: try: @@ -247,7 +258,9 @@ def commit(self) -> None: self._transaction_active = False except PymssqlError as exc: msg = f"Failed to commit SQL Server transaction: {exc}" - raise SQLSpecError(msg) from exc + error = SQLSpecError(msg) + error.__cause__ = exc + raise error from exc def rollback(self) -> None: try: @@ -258,7 +271,9 @@ def rollback(self) -> None: self.connection.rollback() except PymssqlError as exc: msg = f"Failed to rollback SQL Server transaction: {exc}" - raise SQLSpecError(msg) from exc + error = SQLSpecError(msg) + error.__cause__ = exc + raise error from exc finally: self._explicit_transaction = False self._transaction_active = False @@ -443,8 +458,27 @@ def load_from_storage( ) -> "StorageBridgeJob": """Load staged artifacts from storage into the target SQL Server table.""" if kwargs: + validate_load_from_storage_kwargs(kwargs) return super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs + table, + source, + file_format=file_format, + partitioner=partitioner, + overwrite=overwrite, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) arrow_table, inbound = self._read_storage_arrow(source, file_format=file_format) return self.load_from_arrow(table, arrow_table, partitioner=partitioner, overwrite=overwrite, telemetry=inbound) diff --git a/sqlspec/adapters/pymssql/tenant.py b/sqlspec/adapters/pymssql/tenant.py index b5c655b30..2e43e0479 100644 --- a/sqlspec/adapters/pymssql/tenant.py +++ b/sqlspec/adapters/pymssql/tenant.py @@ -12,7 +12,7 @@ if TYPE_CHECKING: from sqlspec.core.tenant.base import SyncTenantStrategy - from sqlspec.utils.tenant import TenantContextPayload + from sqlspec.utils.tenant._context import TenantContextPayload __all__ = ("PymssqlSessionContextStrategy", "PymssqlTenantLease") diff --git a/sqlspec/adapters/pymysql/_typing.py b/sqlspec/adapters/pymysql/_typing.py index ca0c43959..9da57ab57 100644 --- a/sqlspec/adapters/pymysql/_typing.py +++ b/sqlspec/adapters/pymysql/_typing.py @@ -17,17 +17,15 @@ from pymysql.cursors import DictCursor as PyMysqlDictCursor from pymysql.cursors import SSCursor as PyMysqlSSCursor +from sqlspec.driver._session import SyncDriverSessionContext from sqlspec.typing import import_optional_attr if TYPE_CHECKING: - from collections.abc import Callable - from types import TracebackType from typing import Protocol, TypeAlias from google.cloud.sql.connector import Connector as PyMysqlCloudSqlConnector from sqlspec.adapters.pymysql.driver import PyMysqlDriver - from sqlspec.core import StatementConfig class PyMysqlFieldTypeProtocol(Protocol): JSON: int @@ -69,9 +67,6 @@ def _pymysql_cloud_sql_connector(*args: Any, **kwargs: Any) -> Any: ) -_resolved_driver_type: "type[PyMysqlDriver] | None" = None - - class PyMysqlCursor: """Context manager for PyMySQL cursor operations.""" @@ -95,78 +90,16 @@ def __exit__(self, *_: object) -> None: self.cursor.close() -class PyMysqlSessionContext: +class PyMysqlSessionContext(SyncDriverSessionContext): """Sync context manager for PyMySQL sessions.""" - __slots__ = ( - "_acquire_connection", - "_connection", - "_driver", - "_driver_features", - "_prepare_driver", - "_release_connection", - "_statement_config", - ) - - def __init__( - self, - acquire_connection: "Callable[[], PyMysqlConnection]", - release_connection: "Callable[..., Any]", - statement_config: "StatementConfig", - driver_features: "dict[str, Any]", - prepare_driver: "Callable[[PyMysqlDriver], PyMysqlDriver]", - ) -> None: - self._acquire_connection = acquire_connection - self._release_connection = release_connection - self._statement_config = statement_config - self._driver_features = driver_features - self._prepare_driver = prepare_driver - self._connection: PyMysqlConnection | None = None - self._driver: PyMysqlDriver | None = None - - def __enter__(self) -> "PyMysqlDriver": - driver_cls = _get_pymysql_driver_type() - self._connection = self._acquire_connection() - try: - driver = driver_cls( - connection=self._connection, - statement_config=self._statement_config, - driver_features=self._driver_features, - ) - self._driver = driver - return self._prepare_driver(driver) - except BaseException as exc: - with contextlib.suppress(Exception): - self.__exit__(type(exc), exc, exc.__traceback__) - raise - - def __exit__( - self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None" - ) -> "bool | None": - driver = self._driver - try: - if driver is not None: - driver._close_open_streams(error=exc_type is not None) - except BaseException as close_exc: - if exc_type is None: - exc_type, exc_val, exc_tb = type(close_exc), close_exc, close_exc.__traceback__ - raise - finally: - try: - if self._connection is not None: - self._release_connection(self._connection, exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) - self._connection = None - finally: - self._driver = None - if driver is not None: - driver._mark_closed() - return None - - -def _get_pymysql_driver_type() -> "type[PyMysqlDriver]": - global _resolved_driver_type - driver_cls = _resolved_driver_type - if driver_cls is None: - driver_cls = import_optional_attr("sqlspec.adapters.pymysql.driver", "PyMysqlDriver") - _resolved_driver_type = driver_cls - return driver_cls + __slots__ = () + + if TYPE_CHECKING: + + def __enter__(self) -> "PyMysqlDriver": ... + + def _resolve_driver_class(self) -> "type[PyMysqlDriver]": + from sqlspec.adapters.pymysql.driver import PyMysqlDriver + + return PyMysqlDriver diff --git a/sqlspec/adapters/pymysql/core.py b/sqlspec/adapters/pymysql/core.py index bb52dba42..30856c6de 100644 --- a/sqlspec/adapters/pymysql/core.py +++ b/sqlspec/adapters/pymysql/core.py @@ -7,9 +7,10 @@ from typing import TYPE_CHECKING, Any, Final, Literal from sqlspec.adapters.pymysql._typing import PyMysqlMySQLError, PyMysqlSSCursor -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult -from sqlspec.driver import rows_to_dicts +from sqlspec.driver._stream import rows_to_dicts from sqlspec.exceptions import ( CheckViolationError, ConnectionTimeoutError, @@ -31,14 +32,15 @@ ) from sqlspec.utils.config_tools import parse_mysql_dsn from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import quote_backtick_identifier, split_qualified_identifier from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_lastrowid, has_rowcount, has_sqlstate, has_type_code, resolve_row_format if TYPE_CHECKING: from sqlspec.core.cache import CachedQuery - from sqlspec.core.parameters import ParameterValidator + from sqlspec.core.parameters._validator import ParameterValidator + from sqlspec.core.statement import StatementConfig __all__ = ( "PyMysqlStreamSource", diff --git a/sqlspec/adapters/pymysql/driver.py b/sqlspec/adapters/pymysql/driver.py index fd2975a5a..30039accd 100644 --- a/sqlspec/adapters/pymysql/driver.py +++ b/sqlspec/adapters/pymysql/driver.py @@ -6,6 +6,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Final, Literal, cast +from mypy_extensions import mypyc_attr + from sqlspec.adapters.pymysql._typing import ( PYMYSQL_INSERT_VALUES_PATTERN, PyMysqlCursor, @@ -37,20 +39,28 @@ resolve_rowcount, ) from sqlspec.adapters.pymysql.data_dictionary import PyMysqlDataDictionary -from sqlspec.core import ArrowResult, get_cache_config, register_driver_profile -from sqlspec.driver import BaseSyncExceptionHandler, SyncDriverAdapterBase, SyncRowStream +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile from sqlspec.driver._common import cache_hit_hooks_unchanged +from sqlspec.driver._exception_handler import BaseSyncExceptionHandler +from sqlspec.driver._stream import SyncRowStream +from sqlspec.driver._sync import SyncDriverAdapterBase +from sqlspec.exceptions import with_cause from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json +from sqlspec.utils.serializers._json import from_json from sqlspec.utils.type_guards import supports_json_type if TYPE_CHECKING: from collections.abc import Callable, Mapping, Sequence + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.pymysql._typing import PyMysqlConnection - from sqlspec.core import SQL, SQLResult, StatementConfig - from sqlspec.driver import CachedQuery, ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.core.cache import CachedQuery + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.core.statement import SQL, StatementConfig + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry __all__ = ("PyMysqlCursor", "PyMysqlDriver", "PyMysqlExceptionHandler", "PyMysqlSessionContext") @@ -60,6 +70,7 @@ PYMYSQL_JSON_TYPE_CODES: Final[set[int]] = {json_type_value} if json_type_value is not None else set() +@mypyc_attr(allow_interpreted_subclasses=True) class PyMysqlExceptionHandler(BaseSyncExceptionHandler): """Context manager for handling PyMySQL exceptions.""" @@ -74,11 +85,12 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False +@mypyc_attr(allow_interpreted_subclasses=True) class PyMysqlDriver(SyncDriverAdapterBase): """MySQL/MariaDB database driver using PyMySQL.""" __slots__ = ("_data_dictionary", "_json_deserializer") - dialect = "mysql" + dialect: "DialectType | None" = "mysql" def __init__( self, @@ -181,19 +193,19 @@ def begin(self) -> None: try: self.connection.begin() except PyMysqlMySQLError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def commit(self) -> None: try: self.connection.commit() except PyMysqlMySQLError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def rollback(self) -> None: try: self.connection.rollback() except PyMysqlMySQLError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def with_cursor(self, connection: "PyMysqlConnection") -> "PyMysqlCursor": return PyMysqlCursor(connection) @@ -310,7 +322,7 @@ def load_from_arrow( self._attach_partition_telemetry(telemetry_payload, partitioner) return self._storage_job(telemetry_payload, telemetry) - def collect_rows(self, cursor: Any, fetched: "list[Any]") -> "tuple[list[Any], list[str], int]": + def collect_rows(self, cursor: Any, fetched: "Sequence[Any]") -> "tuple[list[Any], list[str], int]": """Collect PyMySQL rows for the direct execution path.""" description = cursor.description or None row_plan = resolve_row_plan(description, PYMYSQL_JSON_TYPE_CODES) diff --git a/sqlspec/adapters/snowflake/_typing.py b/sqlspec/adapters/snowflake/_typing.py index 27f451526..7753bd499 100644 --- a/sqlspec/adapters/snowflake/_typing.py +++ b/sqlspec/adapters/snowflake/_typing.py @@ -1,18 +1,17 @@ """Snowflake cursor and session context types.""" import contextlib -from collections.abc import Callable -from types import TracebackType from typing import TYPE_CHECKING, Any, TypeAlias from mypy_extensions import mypyc_attr +from sqlspec.driver._session import SyncDriverSessionContext + if TYPE_CHECKING: from snowflake.connector.connection import SnowflakeConnection as _SnowflakeConnection from snowflake.connector.cursor import SnowflakeCursor as _SnowflakeRawCursor from sqlspec.adapters.snowflake.driver import SnowflakeDriver - from sqlspec.core import StatementConfig SnowflakeConnection: TypeAlias = _SnowflakeConnection SnowflakeRawCursor: TypeAlias = _SnowflakeRawCursor @@ -43,79 +42,21 @@ def __exit__(self, *_: Any) -> None: cursor.close() -@mypyc_attr(allow_interpreted_subclasses=True, native_class=False) -class SnowflakeSessionContext: +@mypyc_attr(allow_interpreted_subclasses=True) +class SnowflakeSessionContext(SyncDriverSessionContext): """Acquire, prepare, and release a synchronous Snowflake driver session.""" - __slots__ = ( - "_acquire_connection", - "_connection", - "_driver", - "_driver_features", - "_prepare_driver", - "_release_connection", - "_statement_config", - ) - - def __init__( - self, - acquire_connection: "Callable[[], SnowflakeConnection]", - release_connection: Callable[..., Any], - statement_config: "StatementConfig", - driver_features: dict[str, Any], - prepare_driver: "Callable[[SnowflakeDriver], SnowflakeDriver]", - ) -> None: - self._acquire_connection = acquire_connection - self._release_connection = release_connection - self._statement_config = statement_config - self._driver_features = driver_features - self._prepare_driver = prepare_driver - self._connection: SnowflakeConnection | None = None - self._driver: SnowflakeDriver | None = None - - def __enter__(self) -> "SnowflakeDriver": + __slots__ = () + + if TYPE_CHECKING: + + def __enter__(self) -> "SnowflakeDriver": ... + + def _resolve_driver_class(self) -> "type[SnowflakeDriver]": from sqlspec.adapters.snowflake.driver import SnowflakeDriver - connection = self._acquire_connection() - self._connection = connection - try: - self._driver = SnowflakeDriver( - connection=connection, statement_config=self._statement_config, driver_features=self._driver_features - ) - return self._prepare_driver(self._driver) - except BaseException as exc: - driver = self._driver - try: - if driver is not None: - driver._close_open_streams(error=True) - with contextlib.suppress(Exception): - driver.close() - finally: - self._driver = None - self._connection = None - try: - self._release_connection(connection, exc_type=type(exc), exc_val=exc, exc_tb=exc.__traceback__) - finally: - if driver is not None: - driver._mark_closed() - raise - - def __exit__( - self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None - ) -> None: - driver, self._driver = self._driver, None - connection, self._connection = self._connection, None - try: - if driver is not None: - driver._close_open_streams(error=exc_type is not None) - driver.close() - except BaseException as exc: - if connection is not None: - self._release_connection(connection, exc_type=type(exc), exc_val=exc, exc_tb=exc.__traceback__) - raise - else: - if connection is not None: - self._release_connection(connection, exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) - finally: - if driver is not None: - driver._mark_closed() + return SnowflakeDriver + + def _cleanup_driver(self, driver: Any, *, error: bool) -> None: + driver._close_open_streams(error=error) + driver.close() diff --git a/sqlspec/adapters/snowflake/core.py b/sqlspec/adapters/snowflake/core.py index dd4318e2d..9c9d52b6a 100644 --- a/sqlspec/adapters/snowflake/core.py +++ b/sqlspec/adapters/snowflake/core.py @@ -10,7 +10,9 @@ from urllib.parse import urlsplit from sqlspec.adapters.snowflake.type_converter import SnowflakeOutputConverter -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle +from sqlspec.core.statement import StatementConfig from sqlspec.exceptions import ( CheckViolationError, DatabaseConnectionError, @@ -30,7 +32,7 @@ UniqueViolationError, ) from sqlspec.typing import import_optional -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import normalize_identifier, quote_identifier from sqlspec.utils.type_converters import build_uuid_coercions diff --git a/sqlspec/adapters/snowflake/driver.py b/sqlspec/adapters/snowflake/driver.py index 24a536de2..ab23f1b70 100644 --- a/sqlspec/adapters/snowflake/driver.py +++ b/sqlspec/adapters/snowflake/driver.py @@ -8,6 +8,8 @@ from time import perf_counter from typing import TYPE_CHECKING, Any, ClassVar, Literal +from mypy_extensions import mypyc_attr + from sqlspec.adapters.snowflake._typing import SnowflakeCursor from sqlspec.adapters.snowflake.core import ( DEFAULT_STAGE_PARQUET_THRESHOLD_ROWS, @@ -26,31 +28,35 @@ normalize_many_parameters, resolve_rowcount, ) -from sqlspec.core import ( - SQL, - StatementConfig, - build_arrow_result_from_reader, - build_arrow_result_from_table, - get_cache_config, - register_driver_profile, -) -from sqlspec.driver import BaseSyncExceptionHandler, SyncDriverAdapterBase, SyncRowStream +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.result._base import build_arrow_result_from_reader, build_arrow_result_from_table +from sqlspec.core.statement import SQL, StatementConfig +from sqlspec.driver._exception_handler import BaseSyncExceptionHandler +from sqlspec.driver._stream import SyncRowStream +from sqlspec.driver._sync import SyncDriverAdapterBase from sqlspec.exceptions import SQLSpecError from sqlspec.storage._staging_sql import build_create_table_from_arrow_schema +from sqlspec.storage._utils import import_pyarrow, import_pyarrow_parquet from sqlspec.typing import ArrowReturnFormat, StatementParameters from sqlspec.utils.arrow_helpers import arrow_reader_with_deferred_close from sqlspec.utils.correlation import CorrelationContext from sqlspec.utils.module_loader import ensure_pyarrow -from sqlspec.utils.serializers import to_json as encode_json +from sqlspec.utils.serializers._json import to_json from sqlspec.utils.text import normalize_identifier, quote_identifier from sqlspec.utils.uuids import uuid4 if TYPE_CHECKING: + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.snowflake.data_dictionary import SnowflakeSyncDataDictionary - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, CachedQuery, SQLResult, Statement, StatementFilter - from sqlspec.driver import ExecutionResult - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.cache import CachedQuery + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.core.statement import Statement + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry __all__ = ("SnowflakeDriver", "SnowflakeExceptionHandler") @@ -106,8 +112,7 @@ def _iter_arrow_tables(cursor: Any, features: Mapping[str, Any]) -> Iterator[Any except NotSupportedError: row_fallback = True if row_fallback: - ensure_pyarrow() - import pyarrow as pa + pa = import_pyarrow() raw_names = [descriptor_column_name(column) for column in cursor.description or ()] names = normalize_column_names( @@ -131,8 +136,7 @@ def _iter_arrow_tables(cursor: Any, features: Mapping[str, Any]) -> Iterator[Any def materialize_arrow_table_from_batches(cursor: Any, driver_features: Mapping[str, Any]) -> Any: """Read native Arrow batches, preserving metadata for empty and JSON results.""" - ensure_pyarrow() - import pyarrow as pa + pa = import_pyarrow() tables = list(_iter_arrow_tables(cursor, driver_features)) if not tables: @@ -148,8 +152,7 @@ def build_arrow_reader_from_cursor( batch_size: int | None = None, ) -> Any: """Stream Arrow batches and close the cursor on exhaustion or early close.""" - ensure_pyarrow() - import pyarrow as pa + pa = import_pyarrow() tables = iter(_iter_arrow_tables(cursor, driver_features)) try: @@ -174,6 +177,7 @@ def batches() -> Iterator[Any]: return arrow_reader_with_deferred_close(reader, cursor_close_callback) +@mypyc_attr(allow_interpreted_subclasses=True) class SnowflakeExceptionHandler(BaseSyncExceptionHandler): """Capture mapped connector exceptions outside compiled context-manager frames.""" @@ -186,6 +190,7 @@ def _handle_exception(self, exc_type: type[BaseException] | None, exc_val: BaseE return False +@mypyc_attr(allow_interpreted_subclasses=True) class SnowflakeDriver(SyncDriverAdapterBase): """Execute Snowflake SQL with explicit transactions and zero-pandas Arrow I/O.""" @@ -198,7 +203,7 @@ class SnowflakeDriver(SyncDriverAdapterBase): "_paramstyle", "_session_query_tag", ) - dialect = "snowflake" + dialect: "DialectType | None" = "snowflake" native_arrow_export: ClassVar[bool] = True def __init__( @@ -297,7 +302,7 @@ def _execute_cache_hit_with_cursor( ) -> "SQLResult": """Apply statement correlation and Snowflake row semantics on cache hits.""" statement = self._cached_statement( - sql, params, cached, params, params_are_simple=True, compiled_sql=cached.compiled_sql + sql, params, cached, params, params_are_simple=True, compiled_sql=self._format_wire_sql(cached.compiled_sql) ) try: with self.with_cursor(self.connection) as cursor: @@ -717,10 +722,10 @@ def _load_from_arrow_physical( ): self._load_parquet_stage(formatted, arrow_table, overwrite=overwrite) elif arrow_table.num_rows or overwrite: - import pyarrow as pa + pa = import_pyarrow() nested = {field.name for field in arrow_table.schema if pa.types.is_nested(field.type)} - serializer = self.driver_features.get("json_serializer", encode_json) + serializer = self.driver_features.get("json_serializer", to_json) columns = ", ".join(quote_identifier(name) for name in arrow_table.column_names) names = [f"p{index}" for index in range(arrow_table.num_columns)] values = ", ".join( @@ -775,7 +780,7 @@ def _run_in_load_transaction(self, operation: Callable[[], None]) -> None: def _load_parquet_stage(self, table: str, arrow_table: Any, *, overwrite: bool) -> None: """Upload Parquet to a user stage, then replace or append rows and copy them in one DML transaction.""" - import pyarrow.parquet as pq + pq = import_pyarrow_parquet() stage = f"@~/sqlspec_stage/{uuid4().hex}/" handler = self.handle_database_exceptions() @@ -848,9 +853,9 @@ def fetch_chunk(self) -> list[dict[str, Any]]: if batch is not None: rows = batch.to_pylist() if self._variant_names and self._driver.driver_features.get("enable_json_deserialization", True): - from sqlspec.utils.serializers import from_json as decode_json + from sqlspec.utils.serializers._json import from_json - decoder = self._driver.driver_features.get("json_deserializer", decode_json) + decoder = self._driver.driver_features.get("json_deserializer", from_json) for row in rows: for name in self._variant_names: if isinstance(row.get(name), str): diff --git a/sqlspec/adapters/snowflake/migrations.py b/sqlspec/adapters/snowflake/migrations.py index 916826815..246e601b4 100644 --- a/sqlspec/adapters/snowflake/migrations.py +++ b/sqlspec/adapters/snowflake/migrations.py @@ -1,12 +1,16 @@ """Snowflake migration tracking and reversible session schema switches.""" -from sqlspec.builder import CreateTable, Select +from mypy_extensions import mypyc_attr + +from sqlspec.builder._ddl import CreateTable +from sqlspec.builder._select import Select from sqlspec.migrations.tracker import SyncMigrationTracker from sqlspec.utils.text import normalize_identifier, quote_identifier, split_qualified_identifier __all__ = ("SnowflakeSyncMigrationTracker",) +@mypyc_attr(allow_interpreted_subclasses=True) class SnowflakeSyncMigrationTracker(SyncMigrationTracker): """Track migrations with Snowflake types and explicit lowercase result aliases.""" diff --git a/sqlspec/adapters/snowflake/pool.py b/sqlspec/adapters/snowflake/pool.py index 93f4aacec..1a70d9327 100644 --- a/sqlspec/adapters/snowflake/pool.py +++ b/sqlspec/adapters/snowflake/pool.py @@ -12,12 +12,12 @@ from typing import TYPE_CHECKING, Any, cast from sqlspec.adapters.snowflake.core import create_mapped_exception, is_snowflake_error, normalize_connection_params -from sqlspec.exceptions import DatabaseConnectionError, ImproperConfigurationError, MissingDependencyError +from sqlspec.exceptions import DatabaseConnectionError, ImproperConfigurationError, MissingDependencyError, with_cause from sqlspec.typing import import_optional if TYPE_CHECKING: from sqlspec.adapters.snowflake._typing import SnowflakeConnection - from sqlspec.observability import ObservabilityRuntime + from sqlspec.observability._runtime import ObservabilityRuntime __all__ = ("SnowflakeConnectionPool",) @@ -338,7 +338,7 @@ def _open_connection(self) -> "SnowflakeConnection": ) except Exception as exc: if is_snowflake_error(exc): - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc raise def _create_physical_connection(self) -> "SnowflakeConnection": diff --git a/sqlspec/adapters/snowflake/tenant.py b/sqlspec/adapters/snowflake/tenant.py index 20d5bab5d..fd5d96406 100644 --- a/sqlspec/adapters/snowflake/tenant.py +++ b/sqlspec/adapters/snowflake/tenant.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: from sqlspec.core.tenant.base import SyncTenantStrategy - from sqlspec.utils.tenant import TenantContextPayload + from sqlspec.utils.tenant._context import TenantContextPayload __all__ = ( "DEFAULT_SNOWFLAKE_TENANT_ATTRIBUTE", diff --git a/sqlspec/adapters/snowflake/type_converter.py b/sqlspec/adapters/snowflake/type_converter.py index a4da60ab8..6405a3753 100644 --- a/sqlspec/adapters/snowflake/type_converter.py +++ b/sqlspec/adapters/snowflake/type_converter.py @@ -3,7 +3,7 @@ from collections.abc import Callable from typing import Any, Final -from sqlspec.utils.serializers import from_json +from sqlspec.utils.serializers._json import from_json SNOWFLAKE_VARIANT_TYPE_CODES: Final[frozenset[int]] = frozenset({5, 9, 10}) diff --git a/sqlspec/adapters/spanner/core.py b/sqlspec/adapters/spanner/core.py index 5d8c56ba8..a07a06145 100644 --- a/sqlspec/adapters/spanner/core.py +++ b/sqlspec/adapters/spanner/core.py @@ -17,15 +17,11 @@ from sqlspec.adapters.spanner._typing import spanner_exceptions as api_exceptions from sqlspec.adapters.spanner._typing import spanner_param_types from sqlspec.adapters.spanner.type_converter import coerce_params_for_spanner, infer_spanner_param_types -from sqlspec.core import ( - DriverParameterProfile, - ParameterStyle, - StatementConfig, - TypedParameter, - build_statement_config_from_profile, -) -from sqlspec.driver import rows_to_dicts +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle, TypedParameter +from sqlspec.core.statement import StatementConfig from sqlspec.driver._exception_handler import _run_with_async_exception_handler, _run_with_sync_exception_handler +from sqlspec.driver._stream import rows_to_dicts from sqlspec.exceptions import ( DataError, DeadlockError, @@ -40,10 +36,11 @@ SQLParsingError, SQLSpecError, UniqueViolationError, + with_cause, ) from sqlspec.utils.arrow_helpers import convert_dict_to_arrow from sqlspec.utils.routing import RoutingContext -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.sync_tools import exit_async_context if TYPE_CHECKING: @@ -629,6 +626,7 @@ def __call__(self, transaction: Any, *args: Any, **kwargs: Any) -> Any: except DeadlockError as exc: cause = exc.__cause__ if isinstance(cause, api_exceptions.Aborted): + cause.__cause__ = None raise cause from None raise @@ -649,6 +647,7 @@ async def __call__(self, transaction: Any, *args: Any, **kwargs: Any) -> Any: except DeadlockError as exc: cause = exc.__cause__ if isinstance(cause, api_exceptions.Aborted): + cause.__cause__ = None raise cause from None raise @@ -678,7 +677,7 @@ def run_in_transaction_sync( try: return database.run_in_transaction(_SpannerSyncTransactionWork(build_driver, fn), *args, **kwargs) except api_exceptions.GoogleAPICallError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc async def run_in_transaction_async( @@ -706,7 +705,7 @@ async def run_in_transaction_async( try: return await database.run_in_transaction(_SpannerAsyncTransactionWork(build_driver, fn), *args, **kwargs) except api_exceptions.GoogleAPICallError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def list_existing_table_names_sync(database: Any) -> "set[str]": @@ -801,7 +800,7 @@ def execute_partitioned_dml_sync( database.execute_partitioned_dml(sql, params=coerced_params, param_types=resolved_types, **kwargs) or 0 ) except api_exceptions.GoogleAPICallError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc async def execute_partitioned_dml_async( @@ -843,7 +842,7 @@ async def execute_partitioned_dml_async( ) return int(result or 0) except api_exceptions.GoogleAPICallError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc class _SpannerStreamSourceBase: @@ -1046,7 +1045,11 @@ def resolve_snapshot_kwargs( if not supplied and default_staleness is not None and apply_default_staleness: supplied = dict(default_staleness) if multi_use is None: - multi_use = supplied.pop("multi_use", None) + raw_multi_use: Any = supplied.pop("multi_use", None) + if raw_multi_use is not None and not isinstance(raw_multi_use, bool): + msg = "Spanner multi_use must be a boolean" + raise ImproperConfigurationError(msg) + multi_use = raw_multi_use else: supplied.pop("multi_use", None) if len(supplied) > 1: @@ -1063,7 +1066,7 @@ def resolve_snapshot_kwargs( value = timedelta(seconds=value) except (ValueError, OverflowError) as exc: msg = "Spanner staleness must be finite and within the supported duration range" - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc if not isinstance(value, timedelta) or value.total_seconds() < 0: msg = "Spanner staleness must be a nonnegative duration or numeric seconds" raise ImproperConfigurationError(msg) @@ -1073,7 +1076,7 @@ def resolve_snapshot_kwargs( value = datetime.fromisoformat(value) except ValueError as exc: msg = "Spanner snapshot timestamps must be valid ISO-8601 values" - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None: msg = "Spanner snapshot timestamps must include a timezone" raise ImproperConfigurationError(msg) @@ -1189,7 +1192,7 @@ async def _run_read_async( read_error = caught await exit_async_context(snapshot_context, read_error) except api_exceptions.GoogleAPICallError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc return rows @@ -1209,7 +1212,7 @@ def _run_read( reader = cast("_SpannerReadProtocol", snapshot) return list(reader.execute_sql(sql, params=params, param_types=types)) except api_exceptions.GoogleAPICallError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc class _SpannerSyncUpdateJob: diff --git a/sqlspec/adapters/spanner/driver.py b/sqlspec/adapters/spanner/driver.py index bfb31cb8f..c772449f2 100644 --- a/sqlspec/adapters/spanner/driver.py +++ b/sqlspec/adapters/spanner/driver.py @@ -4,6 +4,7 @@ from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Any, Literal, Protocol, cast, overload +from mypy_extensions import mypyc_attr from sqlglot import exp as _sqlglot_exp from sqlspec.adapters.spanner._typing import ( @@ -46,18 +47,15 @@ supports_write, ) from sqlspec.adapters.spanner.data_dictionary import SpannerAsyncDataDictionary, SpannerSyncDataDictionary -from sqlspec.core import StatementConfig, register_driver_profile -from sqlspec.driver import ( - AsyncDriverAdapterBase, - AsyncRowStream, - BaseAsyncExceptionHandler, - BaseSyncExceptionHandler, - ExecutionResult, - SyncDriverAdapterBase, - SyncRowStream, -) -from sqlspec.exceptions import ReadOnlyTransactionError, SQLConversionError -from sqlspec.utils.serializers import from_json +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.driver._async import AsyncDriverAdapterBase +from sqlspec.driver._common import ExecutionResult, validate_load_from_storage_kwargs +from sqlspec.driver._exception_handler import BaseAsyncExceptionHandler, BaseSyncExceptionHandler +from sqlspec.driver._stream import AsyncRowStream, SyncRowStream +from sqlspec.driver._sync import SyncDriverAdapterBase +from sqlspec.exceptions import ReadOnlyTransactionError, SQLConversionError, with_cause +from sqlspec.utils.serializers._json import from_json +from sqlspec.utils.sync_tools import exit_async_context if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Mapping, Sequence @@ -65,10 +63,11 @@ from sqlglot.dialects.dialect import DialectType from sqlspec.adapters.spanner._typing import SpannerSyncConnection - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, SQLResult, Statement, StatementFilter - from sqlspec.core.statement import SQL - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.core.statement import SQL, Statement, StatementConfig + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry from sqlspec.typing import SchemaT, StatementParameters __all__ = ( @@ -110,6 +109,7 @@ ) +@mypyc_attr(allow_interpreted_subclasses=True) class SpannerSyncExceptionHandler(BaseSyncExceptionHandler): """Map Spanner client exceptions to SQLSpec exceptions stored on ``pending_exception``.""" @@ -125,6 +125,7 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False +@mypyc_attr(allow_interpreted_subclasses=True) class SpannerAsyncExceptionHandler(BaseAsyncExceptionHandler): """Map async Spanner client exceptions to SQLSpec exceptions stored on ``pending_exception``.""" @@ -140,10 +141,11 @@ def _handle_exception(self, exc_type: "type[BaseException] | None", exc_val: "Ba return False +@mypyc_attr(allow_interpreted_subclasses=True) class SpannerSyncDriver(SyncDriverAdapterBase): """Synchronous Spanner driver operating on Snapshot or Transaction contexts.""" - dialect: "DialectType" = "spanner" + dialect: "DialectType | None" = "spanner" __slots__ = ( "_data_dictionary", "_owns_transaction", @@ -188,9 +190,9 @@ def dispatch_execute(self, cursor: "SpannerSyncConnection", statement: "SQL") -> return self.create_execution_result(cursor, rowcount_override=0) if statement.is_modifying_operation() and not supports_write(cursor): raise ReadOnlyTransactionError(_READ_ONLY_SNAPSHOT_ERROR_MESSAGE) - params = cast("dict[str, Any] | None", params) - param_types_map = self._infer_param_types(params) - coerced_params = self._coerce_params(params) + dict_params = cast("dict[str, Any]", params) if isinstance(params, dict) else None + param_types_map = self._infer_param_types(dict_params) + coerced_params = self._coerce_params(dict_params) if statement.returns_rows(): reader = cast("_SpannerReadProtocol", cursor) @@ -229,9 +231,9 @@ def dispatch_select_stream(self, statement: "SQL", chunk_size: int) -> "SyncRowS if not statement.returns_rows(): return None sql, params = self._compiled_sql(statement, self.statement_config) - params = cast("dict[str, Any] | None", params) - param_types_map = self._infer_param_types(params) - coerced_params = self._coerce_params(params) + dict_params = cast("dict[str, Any]", params) if isinstance(params, dict) else None + param_types_map = self._infer_param_types(dict_params) + coerced_params = self._coerce_params(dict_params) return SyncRowStream( SpannerSyncStreamSource( self, sql, coerced_params, param_types_map, chunk_size, self._execute_kwargs(for_read=True) @@ -260,7 +262,7 @@ def dispatch_execute_many(self, cursor: "SpannerSyncConnection", statement: "SQL batch_args: list[tuple[str, dict[str, Any] | None, dict[str, Any]]] = [] append_batch_arg = batch_args.append for params in prepared_parameters: - raw_params = cast("dict[str, Any] | None", params) + raw_params = cast("dict[str, Any]", params) if isinstance(params, dict) else None coerced_params = _coerce(raw_params) if not coerced_params: append_batch_arg((sql, {}, empty_param_types)) @@ -311,7 +313,7 @@ def dispatch_execute_script(self, cursor: "SpannerSyncConnection", statement: "S pending_ddl = [] cursor = self.connection reader = cast("_SpannerReadProtocol", cursor) - script_params = cast("dict[str, Any] | None", params) + script_params = cast("dict[str, Any]", params) if isinstance(params, dict) else None param_types_map = self._infer_param_types(script_params) coerced_params = self._coerce_params(script_params) is_select = is_query_statement(stmt, dialect_str) @@ -646,8 +648,27 @@ def load_from_storage( ) -> "StorageBridgeJob": """Load artifacts from storage into Spanner table.""" if kwargs: + validate_load_from_storage_kwargs(kwargs) return super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs + table, + source, + file_format=file_format, + partitioner=partitioner, + overwrite=overwrite, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) arrow_table, inbound = self._read_storage_arrow(source, file_format=file_format) return self.load_from_arrow(table, arrow_table, partitioner=partitioner, overwrite=overwrite, telemetry=inbound) @@ -822,10 +843,11 @@ def _resolve_row_plan(self, fields: Any) -> "tuple[list[str], tuple[tuple[int, A return resolve_row_plan(fields, self._row_plan_cache, json_deserializer=json_deserializer) +@mypyc_attr(allow_interpreted_subclasses=True) class SpannerAsyncDriver(AsyncDriverAdapterBase): """Asynchronous Spanner driver operating on AsyncSnapshot or AsyncTransaction contexts.""" - dialect: "DialectType" = "spanner" + dialect: "DialectType | None" = "spanner" __slots__ = ( "_data_dictionary", "_owns_transaction", @@ -870,9 +892,9 @@ async def dispatch_execute(self, cursor: "SpannerAsyncConnection", statement: "S return self.create_execution_result(cursor, rowcount_override=0) if statement.is_modifying_operation() and not supports_write(cursor): raise ReadOnlyTransactionError(_READ_ONLY_ASYNC_SNAPSHOT_ERROR_MESSAGE) - params = cast("dict[str, Any] | None", params) - param_types_map = self._infer_param_types(params) - coerced_params = self._coerce_params(params) + dict_params = cast("dict[str, Any]", params) if isinstance(params, dict) else None + param_types_map = self._infer_param_types(dict_params) + coerced_params = self._coerce_params(dict_params) if statement.returns_rows(): reader = cast("_SpannerAsyncReadProtocol", cursor) @@ -915,9 +937,9 @@ def dispatch_select_stream(self, statement: "SQL", chunk_size: int) -> "AsyncRow if not statement.returns_rows(): return None sql, params = self._compiled_sql(statement, self.statement_config) - params = cast("dict[str, Any] | None", params) - param_types_map = self._infer_param_types(params) - coerced_params = self._coerce_params(params) + dict_params = cast("dict[str, Any]", params) if isinstance(params, dict) else None + param_types_map = self._infer_param_types(dict_params) + coerced_params = self._coerce_params(dict_params) return AsyncRowStream( SpannerAsyncStreamSource( self, sql, coerced_params, param_types_map, chunk_size, self._execute_kwargs(for_read=True) @@ -946,7 +968,7 @@ async def dispatch_execute_many(self, cursor: "SpannerAsyncConnection", statemen batch_args: list[tuple[str, dict[str, Any] | None, dict[str, Any]]] = [] append_batch_arg = batch_args.append for params in prepared_parameters: - raw_params = cast("dict[str, Any] | None", params) + raw_params = cast("dict[str, Any]", params) if isinstance(params, dict) else None coerced_params = _coerce(raw_params) if not coerced_params: append_batch_arg((sql, {}, empty_param_types)) @@ -997,7 +1019,7 @@ async def dispatch_execute_script(self, cursor: "SpannerAsyncConnection", statem pending_ddl = [] cursor = self.connection reader = cast("_SpannerAsyncReadProtocol", cursor) - script_params = cast("dict[str, Any] | None", params) + script_params = cast("dict[str, Any]", params) if isinstance(params, dict) else None param_types_map = self._infer_param_types(script_params) coerced_params = self._coerce_params(script_params) is_select = is_query_statement(stmt, dialect_str) @@ -1251,8 +1273,12 @@ async def load_from_arrow( """Load Arrow data into Spanner table via batch mutations.""" self._require_capability("arrow_import_enabled") if drop_indexes: - async with self.without_indexes(table, concurrent=concurrent_indexes): - return await self.load_from_arrow( + index_context = self.without_indexes(table, concurrent=concurrent_indexes) + await index_context.__aenter__() + index_error: BaseException | None = None + index_job: StorageBridgeJob | None = None + try: + index_job = await self.load_from_arrow( table, source, partitioner=partitioner, @@ -1267,6 +1293,11 @@ async def load_from_arrow( concurrent_indexes=concurrent_indexes, type_casts=type_casts, ) + except BaseException as exc: + index_error = exc + await exit_async_context(index_context, index_error) + assert index_job is not None + return index_job if stage_table or ( conflict_keys and on_conflict == "nothing" and not hasattr(self.connection, "insert_or_ignore") ): @@ -1340,8 +1371,27 @@ async def load_from_storage( ) -> "StorageBridgeJob": """Load artifacts from storage into Spanner table asynchronously.""" if kwargs: + validate_load_from_storage_kwargs(kwargs) return await super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs + table, + source, + file_format=file_format, + partitioner=partitioner, + overwrite=overwrite, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) arrow_table, inbound = await self._read_storage_arrow(source, file_format=file_format) return await self.load_from_arrow( @@ -1420,7 +1470,7 @@ def write_delete(self, table: str, keyset: Any) -> None: try: writer.delete(table, keyset) except SpannerGoogleAPICallError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def bind_external_transaction(self) -> None: """Mark the driver as bound to an externally managed transaction runner.""" @@ -1441,7 +1491,7 @@ def _apply_mutation( for chunk in chunks: mutation_fn(table, column_list, chunk) except SpannerGoogleAPICallError as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def _execute_kwargs(self, *, for_read: bool = False, for_batch: bool = False) -> dict[str, Any]: options = _pending_spanner_options.get() or self._pending_execute_options @@ -1456,15 +1506,27 @@ def _resolve_database(self) -> Any: async def _batch_write_mutations(self, table: str, columns: "list[str]", chunks: "list[list[list[Any]]]") -> None: """High-throughput async ingest via the Spanner Batch Write API (one mutation group per chunk).""" database = self._resolve_database() - async with database.mutation_groups() as mutation_groups: - for chunk in chunks: - group = mutation_groups.group() - group.insert_or_update(table, columns, chunk) - async for response in await mutation_groups.batch_write(): - status = response.status - if status is not None and status.code: - msg = f"Spanner batch_write group failed: {status.message}" - raise SQLConversionError(msg) + groups_context = database.mutation_groups() + mutation_groups = await groups_context.__aenter__() + groups_error: BaseException | None = None + try: + await self._write_mutation_groups(mutation_groups, table, columns, chunks) + except BaseException as exc: + groups_error = exc + await exit_async_context(groups_context, groups_error) + + async def _write_mutation_groups( + self, mutation_groups: Any, table: str, columns: "list[str]", chunks: "list[list[list[Any]]]" + ) -> None: + """Queue one mutation group per chunk, send them, and raise on the first failed group.""" + for chunk in chunks: + group = mutation_groups.group() + group.insert_or_update(table, columns, chunk) + async for response in await mutation_groups.batch_write(): + status = response.status + if status is not None and status.code: + msg = f"Spanner batch_write group failed: {status.message}" + raise SQLConversionError(msg) async def _commit_before_ddl(self) -> None: """Commit a begun session transaction before a schema change, as DDL does on MySQL or Oracle.""" diff --git a/sqlspec/adapters/spanner/type_converter.py b/sqlspec/adapters/spanner/type_converter.py index 256555b4b..dd038e698 100644 --- a/sqlspec/adapters/spanner/type_converter.py +++ b/sqlspec/adapters/spanner/type_converter.py @@ -25,9 +25,9 @@ from sqlspec.adapters.spanner._typing import SpannerInterval as Interval from sqlspec.adapters.spanner._typing import SpannerJsonObject as JsonObject from sqlspec.adapters.spanner._typing import spanner_param_types as param_types -from sqlspec.core import TypedParameter +from sqlspec.core.parameters._types import TypedParameter from sqlspec.utils.module_loader import import_optional_attr -from sqlspec.utils.serializers import from_json +from sqlspec.utils.serializers._json import from_json from sqlspec.utils.type_converters import should_json_encode_sequence from sqlspec.utils.uuids import uuid_from_bytes diff --git a/sqlspec/adapters/sqlanywhere/core.py b/sqlspec/adapters/sqlanywhere/core.py index 0feee5124..852671be6 100644 --- a/sqlspec/adapters/sqlanywhere/core.py +++ b/sqlspec/adapters/sqlanywhere/core.py @@ -5,7 +5,8 @@ from decimal import Decimal from typing import TYPE_CHECKING, Any, Final, Literal -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult from sqlspec.dialects.sybase import SQLAnywhere as _SQLAnywhereDialect from sqlspec.exceptions import ( @@ -25,9 +26,10 @@ SQLParsingError, SQLSpecError, UniqueViolationError, + with_cause, ) from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import split_qualified_identifier from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_rowcount, resolve_row_format @@ -37,6 +39,7 @@ from logging import Logger from sqlspec.core.cache import CachedQuery + from sqlspec.core.statement import StatementConfig __all__ = ( "DEFAULT_SQLANYWHERE_PORT", @@ -340,6 +343,7 @@ def execute_cache_hit( cached: "CachedQuery", statement_config: "StatementConfig", column_name_cache: "dict[int, tuple[Sequence[Any], list[str]]]", + wire_sql: "str | None" = None, ) -> "SQLResult": """Execute a cached statement on a sqlanydb connection and build its result. @@ -350,6 +354,7 @@ def execute_cache_hit( cached: Cached compilation of ``sql``. statement_config: Statement configuration the result uses to rebuild its statement. column_name_cache: Session-local cache of resolved column names. + wire_sql: Optional wire SQL override with sqlcommenter formatting applied. Returns: The statement result. @@ -358,7 +363,7 @@ def execute_cache_hit( SQLSpecError: Mapped from any other exception the statement raises. """ parameters = tuple(params) if isinstance(params, list) else params - compiled_sql = cached.compiled_sql + compiled_sql = cached.compiled_sql if wire_sql is None else wire_sql try: cursor = connection.cursor() try: @@ -515,7 +520,7 @@ def _normalize_raw_connection_config(connection_config: "Mapping[str, Any]") -> parsed_port = int(port_val) except (TypeError, ValueError) as exc: msg = "SQL Anywhere port must be an integer between 1 and 65535." - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc if not 1 <= parsed_port <= _MAX_SQLANYWHERE_PORT: msg = "SQL Anywhere port must be an integer between 1 and 65535." raise ImproperConfigurationError(msg) diff --git a/sqlspec/adapters/sqlanywhere/driver.py b/sqlspec/adapters/sqlanywhere/driver.py index d3c8f2e6e..a7e9f6b66 100644 --- a/sqlspec/adapters/sqlanywhere/driver.py +++ b/sqlspec/adapters/sqlanywhere/driver.py @@ -5,6 +5,8 @@ from collections.abc import Sized from typing import TYPE_CHECKING, Any, Literal, cast +from mypy_extensions import mypyc_attr + from sqlspec.adapters.sqlanywhere._typing import ( SqlanywhereConnection, SqlanywhereCursor, @@ -26,27 +28,32 @@ resolve_rowcount, ) from sqlspec.adapters.sqlanywhere.data_dictionary import SqlanywhereSyncDataDictionary -from sqlspec.core import SQL, StatementConfig, get_cache_config, register_driver_profile -from sqlspec.driver import ( - BaseSyncExceptionHandler, +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.driver._common import ( ExecutionResult, - SyncDriverAdapterBase, - SyncRowStream, - rows_to_dicts, + cache_hit_hooks_unchanged, + validate_load_from_storage_kwargs, validate_savepoint_name, ) -from sqlspec.driver._common import cache_hit_hooks_unchanged +from sqlspec.driver._exception_handler import BaseSyncExceptionHandler +from sqlspec.driver._stream import SyncRowStream, rows_to_dicts +from sqlspec.driver._sync import SyncDriverAdapterBase from sqlspec.exceptions import SQLSpecError, TransactionError from sqlspec.utils.logging import get_logger if TYPE_CHECKING: from collections.abc import Mapping, Sequence + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.sqlanywhere._typing import SqlanywhereQueryParams as QueryParams - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, SQLResult, Statement, StatementFilter - from sqlspec.driver import CachedQuery - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.cache import CachedQuery + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.core.statement import SQL, Statement, StatementConfig + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry from sqlspec.typing import StatementParameters __all__ = ( @@ -60,6 +67,7 @@ logger = get_logger("sqlspec.adapters.sqlanywhere") +@mypyc_attr(allow_interpreted_subclasses=True) class SqlanywhereExceptionHandler(BaseSyncExceptionHandler): """Context manager for handling SAP SQL Anywhere sqlanydb exceptions.""" @@ -142,11 +150,12 @@ def close(self, error: bool = False) -> None: cursor_manager.__exit__(None, None, None) +@mypyc_attr(allow_interpreted_subclasses=True) class SqlanywhereDriver(SyncDriverAdapterBase): """SAP SQL Anywhere database driver using sqlanydb.""" __slots__ = ("_column_name_cache", "_data_dictionary", "_transaction_active") - dialect = "sqlanywhere" + dialect: "DialectType | None" = "sqlanywhere" def __init__( self, @@ -236,14 +245,18 @@ def commit(self) -> None: self._transaction_active = False except Exception as exc: msg = f"Failed to commit SQL Anywhere transaction: {exc}" - raise TransactionError(msg) from exc + error = TransactionError(msg) + error.__cause__ = exc + raise error from exc def rollback(self) -> None: try: self.connection.rollback() except Exception as exc: msg = f"Failed to rollback SQL Anywhere transaction: {exc}" - raise TransactionError(msg) from exc + error = TransactionError(msg) + error.__cause__ = exc + raise error from exc finally: self._transaction_active = False @@ -413,8 +426,27 @@ def load_from_storage( ) -> "StorageBridgeJob": """Load staged artifacts from storage into the target SQL Anywhere table.""" if kwargs: + validate_load_from_storage_kwargs(kwargs) return super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs + table, + source, + file_format=file_format, + partitioner=partitioner, + overwrite=overwrite, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) arrow_table, inbound = self._read_storage_arrow(source, file_format=file_format) return self.load_from_arrow(table, arrow_table, partitioner=partitioner, overwrite=overwrite, telemetry=inbound) @@ -435,7 +467,10 @@ def _execute_cache_hit( self, sql: str, params: "tuple[Any, ...] | list[Any] | dict[str, Any]", cached: "CachedQuery" ) -> "SQLResult": """Execute a cached statement through the compiled SQL Anywhere cache-hit executor.""" - return execute_cache_hit(self.connection, sql, params, cached, self.statement_config, self._column_name_cache) + wire_sql = self._format_wire_sql(cached.compiled_sql) if self.statement_config.enable_sqlcommenter else None + return execute_cache_hit( + self.connection, sql, params, cached, self.statement_config, self._column_name_cache, wire_sql + ) def _connection_in_transaction(self) -> bool: """Return whether a transaction opened by this driver remains active.""" diff --git a/sqlspec/adapters/sqlite/_typing.py b/sqlspec/adapters/sqlite/_typing.py index 9fd7c2209..a7ba20b71 100644 --- a/sqlspec/adapters/sqlite/_typing.py +++ b/sqlspec/adapters/sqlite/_typing.py @@ -4,19 +4,17 @@ compilation to avoid ABI boundary issues. """ -import contextlib import sqlite3 as sqlite_module -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any + +from sqlspec.driver._session import SyncDriverSessionContext _SqliteConnection = sqlite_module.Connection if TYPE_CHECKING: - from collections.abc import Callable - from types import TracebackType from typing import TypeAlias from sqlspec.adapters.sqlite.driver import SqliteDriver - from sqlspec.core import StatementConfig SqliteConnection: TypeAlias = _SqliteConnection SqliteConnectionFactory: TypeAlias = type[sqlite_module.Connection] @@ -70,90 +68,24 @@ def __exit__(self, *_: Any) -> None: exc_val: Exception value if an exception occurred exc_tb: Exception traceback if an exception occurred """ - if self.cursor is not None: - with contextlib.suppress(Exception): - self.cursor.close() + cursor = self.cursor + if cursor is not None: + try: + cursor.close() + except Exception: + return -class SqliteSessionContext: - """Sync context manager for SQLite sessions. +class SqliteSessionContext(SyncDriverSessionContext): + """Sync context manager for SQLite sessions.""" - This class is intentionally excluded from mypyc compilation to avoid ABI - boundary issues. It receives callables from uncompiled config classes and - instantiates compiled Driver objects, acting as a bridge between compiled - and uncompiled code. + __slots__ = () - Uses callable-based connection management to decouple from config implementation. - """ + if TYPE_CHECKING: - __slots__ = ( - "_acquire_connection", - "_connection", - "_driver", - "_driver_class", - "_driver_features", - "_prepare_driver", - "_release_connection", - "_statement_config", - ) - - _default_driver_class: ClassVar["type[SqliteDriver] | None"] = None - - def __init__( - self, - acquire_connection: "Callable[[], Any]", - release_connection: "Callable[..., Any]", - statement_config: "StatementConfig", - driver_features: "dict[str, Any]", - prepare_driver: "Callable[[SqliteDriver], SqliteDriver]", - driver_class: "type[SqliteDriver] | None" = None, - ) -> None: - self._acquire_connection = acquire_connection - self._release_connection = release_connection - self._statement_config = statement_config - self._driver_features = driver_features - self._prepare_driver = prepare_driver - self._driver_class = driver_class or self._default_driver_class - self._connection: Any = None - self._driver: SqliteDriver | None = None - - def __enter__(self) -> "SqliteDriver": - driver_cls = self._driver_class - if driver_cls is None: - from sqlspec.adapters.sqlite.driver import SqliteDriver - - driver_cls = SqliteDriver - - self._connection = self._acquire_connection() - try: - self._driver = driver_cls( - connection=self._connection, - statement_config=self._statement_config, - driver_features=self._driver_features, - ) - return self._prepare_driver(self._driver) - except BaseException as exc: - self.__exit__(type(exc), exc, exc.__traceback__) - raise - - def __exit__( - self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None" - ) -> "bool | None": - driver = self._driver - try: - if driver is not None: - driver._close_open_streams(error=exc_type is not None) - except BaseException as close_exc: - if exc_type is None: - exc_type, exc_val, exc_tb = type(close_exc), close_exc, close_exc.__traceback__ - raise - finally: - try: - if self._connection is not None: - self._release_connection(self._connection, exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) - self._connection = None - finally: - self._driver = None - if driver is not None: - driver._mark_closed() - return None + def __enter__(self) -> "SqliteDriver": ... + + def _resolve_driver_class(self) -> "type[SqliteDriver]": + from sqlspec.adapters.sqlite.driver import SqliteDriver + + return SqliteDriver diff --git a/sqlspec/adapters/sqlite/core.py b/sqlspec/adapters/sqlite/core.py index 95414f8d0..afb5d22af 100644 --- a/sqlspec/adapters/sqlite/core.py +++ b/sqlspec/adapters/sqlite/core.py @@ -13,9 +13,10 @@ from sqlglot import exp from sqlspec.adapters.sqlite._typing import sqlite_module as sqlite3 -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult -from sqlspec.driver import rows_to_dicts +from sqlspec.driver._stream import rows_to_dicts from sqlspec.exceptions import ( CheckViolationError, DatabaseConnectionError, @@ -32,8 +33,9 @@ SQLParsingError, SQLSpecError, UniqueViolationError, + with_cause, ) -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import quote_identifier, split_qualified_identifier from sqlspec.utils.type_converters import build_decimal_converter, build_uuid_coercions, time_iso_convert from sqlspec.utils.type_guards import has_sqlite_error, resolve_row_format @@ -44,6 +46,7 @@ from sqlspec.adapters.sqlite._typing import SqliteConnection from sqlspec.core.cache import CachedQuery from sqlspec.core.compiler import OperationType + from sqlspec.core.statement import StatementConfig __all__ = ( "JULIAN_EPOCH", @@ -190,7 +193,7 @@ def extension_pragma_statements(config: Any, extension_name: str) -> "tuple[str, msg = str(exc).replace( "driver_features['pragmas']", f"extension_config['{extension_name}']['pragma_overrides']" ) - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc return tuple(statements) @@ -307,7 +310,8 @@ def resolve_lastrowid( operation_type: "OperationType", rowcount: int, expression: Any, - eligibility_cache: "dict[tuple[str | None, str], bool]", + eligibility_cache: "dict[Any, bool]", + compiled_sql: "str | None" = None, ) -> "int | None": """Resolve lastrowid only when compiled target metadata proves rowid eligibility. @@ -321,23 +325,45 @@ def resolve_lastrowid( operation_type: Compiled statement operation type. rowcount: Normalized affected-row count. expression: Compiled SQLGlot expression. - eligibility_cache: Driver-local target eligibility cache. + eligibility_cache: Driver-local eligibility cache, keyed by ``(schema, table)`` target and + by compiled SQL text. + compiled_sql: Optional compiled SQL text used to key eligibility lookups without AST walks. + Text that ends with a comment is resolved by target and never stored, so per-request + comments do not add entries. Returns: Integer lastrowid when the current target is a rowid table, otherwise None. """ if operation_type != "INSERT" or rowcount <= 0: return None + if compiled_sql is not None: + supports_rowid = eligibility_cache.get(compiled_sql) + if supports_rowid is None: + supports_rowid = _target_rowid_eligibility(connection, expression, eligibility_cache) + if supports_rowid is not None and not compiled_sql.endswith("*/"): + if len(eligibility_cache) >= _ROWID_ELIGIBILITY_CACHE_SIZE: + eligibility_cache.clear() + eligibility_cache[compiled_sql] = supports_rowid + if not supports_rowid: + return None + return normalize_lastrowid(cursor, operation_type, rowcount) + if not _target_rowid_eligibility(connection, expression, eligibility_cache): + return None + return normalize_lastrowid(cursor, operation_type, rowcount) + + +def _target_rowid_eligibility(connection: Any, expression: Any, eligibility_cache: "dict[Any, bool]") -> "bool | None": + """Return whether the INSERT target of ``expression`` is a rowid table, or None without a target.""" target = _resolve_insert_target(expression) if target is None: return None supports_rowid = eligibility_cache.get(target) if supports_rowid is None: supports_rowid = _target_supports_rowid(connection, target) + if len(eligibility_cache) >= _ROWID_ELIGIBILITY_CACHE_SIZE: + eligibility_cache.clear() eligibility_cache[target] = supports_rowid - if not supports_rowid: - return None - return normalize_lastrowid(cursor, operation_type, rowcount) + return supports_rowid def _close_cursor(cursor: Any) -> None: @@ -354,7 +380,8 @@ def execute_cache_hit( params: Any, cached: "CachedQuery", statement_config: "StatementConfig", - rowid_target_cache: "dict[tuple[str | None, str], bool]", + rowid_target_cache: "dict[Any, bool]", + wire_sql: "str | None" = None, ) -> "SQLResult": """Execute a cached statement on a SQLite connection and build its result. @@ -365,6 +392,7 @@ def execute_cache_hit( cached: Cached compilation of ``sql``. statement_config: Statement configuration the result uses to rebuild its statement. rowid_target_cache: Driver-local rowid eligibility cache, cleared when the statement can change schema. + wire_sql: Optional wire SQL override with sqlcommenter formatting applied. Returns: The statement result. @@ -375,8 +403,9 @@ def execute_cache_hit( operation_type = cached.operation_type if operation_type not in ROWID_SAFE_OPERATIONS: rowid_target_cache.clear() + executed_sql = cached.compiled_sql if wire_sql is None else wire_sql try: - cursor = connection.execute(cached.compiled_sql, params or ()) + cursor = connection.execute(executed_sql, params or ()) except sqlite3.Error as exc: mapped_error = create_mapped_exception(exc) mapped_error.__cause__ = exc @@ -397,6 +426,7 @@ def execute_cache_hit( affected_rows, cached.processed_state.parsed_expression, rowid_target_cache, + cached.compiled_sql, ), ) fetched_data: list[Any] = cursor.fetchall() @@ -410,6 +440,7 @@ def execute_cache_hit( effective_rowcount, cached.processed_state.parsed_expression, rowid_target_cache, + cached.compiled_sql, ) description = cursor.description column_names: list[str] = [column[0] for column in description] if description else [] @@ -417,7 +448,7 @@ def execute_cache_hit( sql, statement_config, cached.processed_state, - cached.compiled_sql, + executed_sql, params, fetched_data, column_names, @@ -512,6 +543,7 @@ def normalize_execute_many_parameters(parameters: Any) -> Any: _INSERT_TARGET_CACHE_SIZE: Final[int] = 512 +_ROWID_ELIGIBILITY_CACHE_SIZE: Final[int] = 512 _INSERT_TARGET_CACHE: "dict[int, tuple[Any, tuple[str | None, str] | None]]" = {} _EXCLUDED_CONNECTION_KEYS: Final[frozenset[str]] = frozenset({ "query_only", diff --git a/sqlspec/adapters/sqlite/driver.py b/sqlspec/adapters/sqlite/driver.py index fe960bd8f..13a24ff0f 100644 --- a/sqlspec/adapters/sqlite/driver.py +++ b/sqlspec/adapters/sqlite/driver.py @@ -31,7 +31,7 @@ from sqlspec.driver._exception_handler import BaseSyncExceptionHandler from sqlspec.driver._stream import SyncRowStream from sqlspec.driver._sync import SyncDriverAdapterBase -from sqlspec.exceptions import ImproperConfigurationError +from sqlspec.exceptions import ImproperConfigurationError, with_cause from sqlspec.utils.type_guards import resolve_row_format if TYPE_CHECKING: @@ -100,7 +100,7 @@ def __init__( super().__init__(connection=connection, statement_config=statement_config, driver_features=driver_features) self._data_dictionary: SqliteDataDictionary | None = None - self._rowid_target_cache: dict[tuple[str | None, str], bool] = {} + self._rowid_target_cache: dict[Any, bool] = {} self._cache_hit_executor = execute_cache_hit if cache_hit_hooks_unchanged(type(self), SqliteDriver) else None self._cache_hit_state = self._rowid_target_cache @@ -142,6 +142,7 @@ def dispatch_execute(self, cursor: Any, statement: "SQL") -> "ExecutionResult": effective_rowcount, statement._get_shared_expression(), self._rowid_target_cache, + sql, ) return self.create_execution_result( @@ -162,6 +163,7 @@ def dispatch_execute(self, cursor: Any, statement: "SQL") -> "ExecutionResult": affected_rows, statement._get_shared_expression(), self._rowid_target_cache, + sql, ) return self.create_execution_result(cursor, rowcount_override=affected_rows, last_inserted_id=last_inserted_id) @@ -207,13 +209,11 @@ def begin(self, mode: "Literal['DEFERRED', 'IMMEDIATE', 'EXCLUSIVE'] | None" = N if not self.connection.in_transaction: stmt = f"BEGIN {transaction_mode}" if transaction_mode else "BEGIN" cursor = self.connection.execute(stmt) - with contextlib.suppress(AttributeError, TypeError): - self.connection.in_transaction = True if cursor is not None: with contextlib.suppress(Exception): cursor.close() except sqlite3.Error as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e def commit(self) -> None: """Commit the current transaction. @@ -223,10 +223,8 @@ def commit(self) -> None: """ try: end_transaction(self.connection, commit=True, supports_autocommit=self._in_autocommit_mode()) - with contextlib.suppress(AttributeError, TypeError): - self.connection.in_transaction = False except sqlite3.Error as e: - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e def rollback(self) -> None: """Rollback the current transaction. @@ -236,12 +234,10 @@ def rollback(self) -> None: """ try: end_transaction(self.connection, commit=False, supports_autocommit=self._in_autocommit_mode()) - with contextlib.suppress(AttributeError, TypeError): - self.connection.in_transaction = False self._rowid_target_cache.clear() except sqlite3.Error as e: self._rowid_target_cache.clear() - raise create_mapped_exception(e) from e + raise with_cause(create_mapped_exception(e), e) from e def create_savepoint(self, name: str) -> None: """Create a savepoint within the current transaction without script splitting.""" @@ -252,7 +248,7 @@ def create_savepoint(self, name: str) -> None: with contextlib.suppress(Exception): cursor.close() except sqlite3.Error as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def release_savepoint(self, name: str) -> None: """Release a savepoint without clearing rowid target cache or splitting script.""" @@ -263,7 +259,7 @@ def release_savepoint(self, name: str) -> None: with contextlib.suppress(Exception): cursor.close() except sqlite3.Error as exc: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc def with_cursor(self, connection: "SqliteConnection") -> "SqliteCursor": """Create context manager for SQLite cursor. @@ -386,7 +382,7 @@ def load_from_arrow( if owns_transaction: self.rollback() if isinstance(exc, sqlite3.Error): - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc raise telemetry_payload = self._ingest_telemetry(arrow_table) @@ -415,7 +411,10 @@ def _execute_cache_hit( self, sql: str, params: "tuple[Any, ...] | list[Any] | dict[str, Any]", cached: "CachedQuery" ) -> "SQLResult": """Execute a cached statement through the compiled SQLite cache-hit executor.""" - return execute_cache_hit(self.connection, sql, params, cached, self.statement_config, self._rowid_target_cache) + wire_sql = self._format_wire_sql(cached.compiled_sql) if self.statement_config.enable_sqlcommenter else None + return execute_cache_hit( + self.connection, sql, params, cached, self.statement_config, self._rowid_target_cache, wire_sql + ) def _invalidate_rowid_target_cache(self, operation_type: "OperationType") -> None: if operation_type not in ROWID_SAFE_OPERATIONS: diff --git a/sqlspec/adapters/sqlite/pool.py b/sqlspec/adapters/sqlite/pool.py index 6425392c4..1ae379b86 100644 --- a/sqlspec/adapters/sqlite/pool.py +++ b/sqlspec/adapters/sqlite/pool.py @@ -7,7 +7,6 @@ import time import weakref from collections.abc import Mapping -from contextlib import contextmanager from typing import TYPE_CHECKING, Any, Final, cast from urllib.parse import parse_qs, unquote, urlsplit @@ -16,12 +15,12 @@ from sqlspec.adapters.sqlite._typing import SqliteConnection from sqlspec.adapters.sqlite._typing import sqlite_module as sqlite3 from sqlspec.adapters.sqlite.core import create_mapped_exception, end_transaction, is_read_only_sqlite_config -from sqlspec.exceptions import ImproperConfigurationError +from sqlspec.exceptions import ImproperConfigurationError, with_cause from sqlspec.utils.logging import POOL_LOGGER_NAME, get_logger, log_with_context from sqlspec.utils.uuids import uuid4 if TYPE_CHECKING: - from collections.abc import Callable, Generator + from collections.abc import Callable __all__ = ("SqliteConnectionPool",) @@ -164,7 +163,7 @@ def new_connection(self) -> SqliteConnection: connection = sqlite3.connect(**self._connection_parameters) except sqlite3.Error as exc: if getattr(exc, "sqlite_errorcode", 0) & 0xFF in (sqlite3.SQLITE_CANTOPEN, 26): - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc raise try: @@ -198,18 +197,13 @@ def new_connection(self) -> SqliteConnection: return cast("SqliteConnection", connection) - @contextmanager - def get_connection(self) -> "Generator[SqliteConnection, None, None]": + def get_connection(self) -> "_SqliteConnectionLease": """Lease a connection, serializing writer leases when configured. - Yields: - A connection with its lease held until context exit. + Returns: + A slotted context manager holding the connection lease until exit. """ - with ( - self._write_lock if self._write_lock is not None else contextlib.nullcontext(), - self._get_connection_unlocked() as connection, - ): - yield connection + return _SqliteConnectionLease(self) def close(self) -> None: """Close every connection this pool opened, on any thread.""" @@ -306,50 +300,6 @@ def resolve_connection_identity(connection: Any, *configs: Any) -> object | None return token return None - @contextmanager - def _get_connection_unlocked(self) -> "Generator[SqliteConnection, None, None]": - """Get a thread-local connection. - - Yields: - SqliteConnection: A thread-local connection. - """ - connection = self._get_thread_connection() - with self._registry_lock: - depth = self._connection_depths.get(connection, 0) - self._connection_depths[connection] = depth + 1 - if depth == 0: - self._checked_out_count += 1 - outermost = depth == 0 - try: - yield connection - except BaseException: - if outermost: - try: - end_transaction(connection, commit=False) - except Exception: - self._retire_failed_connection(connection) - raise - else: - if outermost: - try: - end_transaction(connection, commit=True) - except BaseException: - try: - end_transaction(connection, commit=False) - except Exception: - self._retire_failed_connection(connection) - raise - self._reset_connection_or_retire(connection) - finally: - with self._registry_lock: - remaining = self._connection_depths.get(connection, 1) - 1 - if remaining > 0: - self._connection_depths[connection] = remaining - else: - self._connection_depths.pop(connection, None) - if outermost and self._checked_out_count > 0: - self._checked_out_count -= 1 - def _reset_connection(self, connection: SqliteConnection) -> None: if not self._reset_session_on_release: return @@ -719,6 +669,83 @@ def _close_thread_connection(self) -> None: holder.last_used = 0.0 +@mypyc_attr(allow_interpreted_subclasses=False) +class _SqliteConnectionLease: + """Slotted context manager for SQLite pooled connection leases.""" + + __slots__ = ("_connection", "_outermost", "_pool", "_write_lock_acquired") + + def __init__(self, pool: SqliteConnectionPool) -> None: + self._pool = pool + self._connection: SqliteConnection | None = None + self._outermost = False + self._write_lock_acquired = False + + def __enter__(self) -> SqliteConnection: + pool = self._pool + write_lock = pool._write_lock + if write_lock is not None: + write_lock.acquire() + self._write_lock_acquired = True + try: + connection = pool._get_thread_connection() + except BaseException: + if self._write_lock_acquired and write_lock is not None: + self._write_lock_acquired = False + write_lock.release() + raise + with pool._registry_lock: + depth = pool._connection_depths.get(connection, 0) + pool._connection_depths[connection] = depth + 1 + if depth == 0: + pool._checked_out_count += 1 + self._connection = connection + self._outermost = depth == 0 + return connection + + def __exit__( + self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: Any + ) -> "bool | None": + connection = self._connection + if connection is None: + return None + pool = self._pool + outermost = self._outermost + self._connection = None + try: + if outermost: + if exc_type is not None: + try: + end_transaction(connection, commit=False) + except Exception: + pool._retire_failed_connection(connection) + else: + try: + end_transaction(connection, commit=True) + except BaseException: + try: + end_transaction(connection, commit=False) + except Exception: + pool._retire_failed_connection(connection) + raise + pool._reset_connection_or_retire(connection) + finally: + try: + with pool._registry_lock: + remaining = pool._connection_depths.get(connection, 1) - 1 + if remaining > 0: + pool._connection_depths[connection] = remaining + else: + pool._connection_depths.pop(connection, None) + if outermost and pool._checked_out_count > 0: + pool._checked_out_count -= 1 + finally: + if self._write_lock_acquired and pool._write_lock is not None: + self._write_lock_acquired = False + pool._write_lock.release() + return None + + @mypyc_attr(allow_interpreted_subclasses=False) class _TaskConnectionHolder: """Per-task connection task_holder isolating concurrent asyncio tasks on one thread.""" diff --git a/sqlspec/adapters/sybase/core.py b/sqlspec/adapters/sybase/core.py index 3ae815632..98745060c 100644 --- a/sqlspec/adapters/sybase/core.py +++ b/sqlspec/adapters/sybase/core.py @@ -5,7 +5,8 @@ from typing import TYPE_CHECKING, Any, Final, Literal from urllib.parse import unquote, urlsplit -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle from sqlspec.core.result._base import DMLResult, SQLResult from sqlspec.dialects.sybase import Sybase as _SybaseDialect from sqlspec.exceptions import ( @@ -25,9 +26,10 @@ SQLParsingError, SQLSpecError, UniqueViolationError, + with_cause, ) from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import split_qualified_identifier from sqlspec.utils.type_converters import build_uuid_coercions from sqlspec.utils.type_guards import has_rowcount, resolve_row_format @@ -37,7 +39,8 @@ from logging import Logger from sqlspec.core.cache import CachedQuery - from sqlspec.core.parameters import ParameterValidator + from sqlspec.core.parameters._validator import ParameterValidator + from sqlspec.core.statement import StatementConfig __all__ = ( "DEFAULT_SYBASE_CONN_PROPERTIES", @@ -524,7 +527,7 @@ def parse_sybase_replica_endpoint(endpoint: str) -> dict[str, Any]: hostname, port = parsed.hostname, parsed.port except ValueError as exc: msg = "Invalid sybase replica endpoint" - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc if not hostname or (parsed.scheme and parsed.scheme != "sybase"): msg = "Invalid sybase replica endpoint" raise ImproperConfigurationError(msg) diff --git a/sqlspec/adapters/sybase/driver.py b/sqlspec/adapters/sybase/driver.py index 91681a612..5e39ff5c7 100644 --- a/sqlspec/adapters/sybase/driver.py +++ b/sqlspec/adapters/sybase/driver.py @@ -5,6 +5,8 @@ from collections.abc import Sized from typing import TYPE_CHECKING, Any, Literal, cast +from mypy_extensions import mypyc_attr + from sqlspec.adapters.sybase._typing import SybaseConnection, SybaseCursor, SybaseRawCursor, SybaseSessionContext from sqlspec.adapters.sybase.core import ( build_insert_statement, @@ -22,27 +24,32 @@ resolve_rowcount, ) from sqlspec.adapters.sybase.data_dictionary import SybaseSyncDataDictionary -from sqlspec.core import SQL, StatementConfig, get_cache_config, register_driver_profile -from sqlspec.driver import ( - BaseSyncExceptionHandler, +from sqlspec.core.cache import get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.driver._common import ( ExecutionResult, - SyncDriverAdapterBase, - SyncRowStream, - rows_to_dicts, + cache_hit_hooks_unchanged, + validate_load_from_storage_kwargs, validate_savepoint_name, ) -from sqlspec.driver._common import cache_hit_hooks_unchanged +from sqlspec.driver._exception_handler import BaseSyncExceptionHandler +from sqlspec.driver._stream import SyncRowStream, rows_to_dicts +from sqlspec.driver._sync import SyncDriverAdapterBase from sqlspec.exceptions import SQLSpecError from sqlspec.utils.logging import get_logger if TYPE_CHECKING: from collections.abc import Mapping, Sequence + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.sybase._typing import SybaseQueryParams as QueryParams - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, SQLResult, Statement, StatementFilter - from sqlspec.driver import CachedQuery - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.cache import CachedQuery + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.core.statement import SQL, Statement, StatementConfig + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry from sqlspec.typing import StatementParameters __all__ = ("SybaseCursor", "SybaseDriver", "SybaseExceptionHandler", "SybaseSessionContext", "SybaseStreamSource") @@ -50,6 +57,7 @@ logger = get_logger("sqlspec.adapters.sybase") +@mypyc_attr(allow_interpreted_subclasses=True) class SybaseExceptionHandler(BaseSyncExceptionHandler): """Context manager for handling Sybase pymssql exceptions.""" @@ -128,11 +136,12 @@ def close(self, error: bool = False) -> None: cursor_manager.__exit__(None, None, None) +@mypyc_attr(allow_interpreted_subclasses=True) class SybaseDriver(SyncDriverAdapterBase): """Sybase Adaptive Server Enterprise database driver using pymssql over FreeTDS TDS 5.0.""" __slots__ = ("_column_name_cache", "_data_dictionary", "_explicit_transaction", "_transaction_active") - dialect = "sybase" + dialect: "DialectType | None" = "sybase" def __init__( self, @@ -221,7 +230,9 @@ def begin(self) -> None: except Exception as exc: mapped = create_mapped_exception(exc, logger=logger) msg = f"Failed to begin Sybase ASE transaction: {exc}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc def commit(self) -> None: try: @@ -235,7 +246,9 @@ def commit(self) -> None: except Exception as exc: mapped = create_mapped_exception(exc, logger=logger) msg = f"Failed to commit Sybase ASE transaction: {exc}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc def rollback(self) -> None: try: @@ -247,7 +260,9 @@ def rollback(self) -> None: except Exception as exc: mapped = create_mapped_exception(exc, logger=logger) msg = f"Failed to rollback Sybase ASE transaction: {exc}" - raise type(mapped)(msg) from exc + error = type(mapped)(msg) + error.__cause__ = exc + raise error from exc finally: self._explicit_transaction = False self._transaction_active = False @@ -420,8 +435,27 @@ def load_from_storage( ) -> "StorageBridgeJob": """Load staged artifacts from storage into the target Sybase ASE table.""" if kwargs: + validate_load_from_storage_kwargs(kwargs) return super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs + table, + source, + file_format=file_format, + partitioner=partitioner, + overwrite=overwrite, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) arrow_table, inbound = self._read_storage_arrow(source, file_format=file_format) return self.load_from_arrow(table, arrow_table, partitioner=partitioner, overwrite=overwrite, telemetry=inbound) diff --git a/sqlspec/adapters/teradata/adk/store.py b/sqlspec/adapters/teradata/adk/store.py index 7f3aef529..78a5abd09 100644 --- a/sqlspec/adapters/teradata/adk/store.py +++ b/sqlspec/adapters/teradata/adk/store.py @@ -1,7 +1,7 @@ """Teradata Vantage ADK stores for Google Agent Development Kit session storage.""" import re -from collections.abc import Iterator +from collections.abc import Generator from contextlib import contextmanager from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, cast @@ -1255,7 +1255,7 @@ def _create_missing_index(driver: Any, table_name: str, index_name: str, columns @contextmanager -def _transaction(config: "TeradataConfig") -> Iterator[tuple[Any, Any]]: +def _transaction(config: "TeradataConfig") -> Generator[tuple[Any, Any], None, None]: """Keep compound DML atomic and restore autocommit. Yields: diff --git a/sqlspec/adapters/teradata/core.py b/sqlspec/adapters/teradata/core.py index 0ce96d741..4789484b3 100644 --- a/sqlspec/adapters/teradata/core.py +++ b/sqlspec/adapters/teradata/core.py @@ -6,7 +6,9 @@ from typing import Any, Literal from urllib.parse import unquote, urlsplit -from sqlspec.core import DriverParameterProfile, ParameterStyle, StatementConfig, build_statement_config_from_profile +from sqlspec.core.parameters._registry import build_statement_config_from_profile +from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyle +from sqlspec.core.statement import StatementConfig from sqlspec.dialects.teradata import Vantage as _VantageDialect from sqlspec.exceptions import ( CheckViolationError, @@ -24,8 +26,9 @@ SQLParsingError, SQLSpecError, UniqueViolationError, + with_cause, ) -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.text import split_qualified_identifier from sqlspec.utils.type_converters import build_uuid_coercions @@ -410,7 +413,7 @@ def parse_teradata_replica_endpoint(endpoint: str) -> dict[str, Any]: hostname, port = parsed.hostname, parsed.port except ValueError as exc: msg = "Invalid teradata replica endpoint" - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc if not hostname or (parsed.scheme and parsed.scheme != "teradata"): msg = "Invalid teradata replica endpoint" raise ImproperConfigurationError(msg) diff --git a/sqlspec/adapters/teradata/driver.py b/sqlspec/adapters/teradata/driver.py index 5ebaa309a..50125d7e7 100644 --- a/sqlspec/adapters/teradata/driver.py +++ b/sqlspec/adapters/teradata/driver.py @@ -5,6 +5,8 @@ import sys from typing import TYPE_CHECKING, Any, Literal, cast +from mypy_extensions import mypyc_attr + from sqlspec.adapters.teradata._typing import ( TeradataConnection, TeradataCursor, @@ -32,22 +34,13 @@ extract_arrow_column_types, prepare_teradata_parameters, ) -from sqlspec.core import ( - SQL, - SQLResult, - StatementConfig, - build_arrow_result_from_reader, - create_arrow_result, - get_cache_config, - register_driver_profile, -) -from sqlspec.driver import ( - BaseSyncExceptionHandler, - ExecutionResult, - SyncDriverAdapterBase, - SyncRowStream, - rows_to_dicts, -) +from sqlspec.core.cache import CachedQuery, get_cache_config +from sqlspec.core.parameters._registry import register_driver_profile +from sqlspec.core.result._base import DMLResult, SQLResult, build_arrow_result_from_reader, create_arrow_result +from sqlspec.driver._common import validate_load_from_storage_kwargs +from sqlspec.driver._exception_handler import BaseSyncExceptionHandler +from sqlspec.driver._stream import SyncRowStream, rows_to_dicts +from sqlspec.driver._sync import SyncDriverAdapterBase from sqlspec.exceptions import ImproperConfigurationError, SQLSpecError, TransactionError from sqlspec.utils.arrow_helpers import ( arrow_batches_from_rows, @@ -59,10 +52,15 @@ if TYPE_CHECKING: from collections.abc import Mapping, Sequence + from sqlglot.dialects.dialect import DialectType + from sqlspec.adapters.teradata._typing import TeradataQueryParams as QueryParams - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, CachedQuery, Statement, StatementFilter - from sqlspec.storage import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult + from sqlspec.core.statement import SQL, Statement, StatementConfig + from sqlspec.driver._common import ExecutionResult + from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry from sqlspec.typing import ArrowReturnFormat, StatementParameters __all__ = ( @@ -77,6 +75,7 @@ _SESSION_QUERY_BAND_SQL_PATTERN = re.compile(r"\bQUERY_BAND\b.*\bFOR\s+SESSION\b", re.IGNORECASE | re.DOTALL) +@mypyc_attr(allow_interpreted_subclasses=True) class TeradataExceptionHandler(BaseSyncExceptionHandler): """Context manager for handling Teradata Vantage teradatasql exceptions.""" @@ -186,6 +185,7 @@ def close(self, error: bool = False) -> None: cursor_manager.__exit__(None, None, None) +@mypyc_attr(allow_interpreted_subclasses=True) class TeradataDriver(SyncDriverAdapterBase): """Teradata Vantage database driver using teradatasql.""" @@ -197,7 +197,7 @@ class TeradataDriver(SyncDriverAdapterBase): "_stream_sources", "_transaction_active", ) - dialect = "vantage" + dialect: "DialectType | None" = "vantage" def __init__( self, @@ -312,7 +312,9 @@ def begin(self) -> None: self._transaction_active = True except Exception as exc: msg = "Failed to begin Teradata transaction" - raise TransactionError(msg) from exc + error = TransactionError(msg) + error.__cause__ = exc + raise error from exc def commit(self) -> None: """Commit manual-mode work and restore the autocommit state captured by ``begin()``.""" @@ -325,7 +327,9 @@ def commit(self) -> None: self._previous_autocommit = None except Exception as exc: msg = "Failed to commit Teradata transaction" - raise TransactionError(msg) from exc + error = TransactionError(msg) + error.__cause__ = exc + raise error from exc def rollback(self) -> None: """Roll back manual-mode work and restore the autocommit state captured by ``begin()``.""" @@ -338,7 +342,9 @@ def rollback(self) -> None: self._previous_autocommit = None except Exception as exc: msg = "Failed to rollback Teradata transaction" - raise TransactionError(msg) from exc + error = TransactionError(msg) + error.__cause__ = exc + raise error from exc def set_query_band(self, band: "Mapping[str, str] | str", *, transaction: bool = False) -> None: """Apply validated query-band metadata to the session or active transaction.""" @@ -417,7 +423,7 @@ def select_to_arrow( msg = "Teradata Arrow export uses chunked DB-API conversion" raise ImproperConfigurationError(msg) size = batch_size if batch_size is not None else int(self.driver_features.get("arraysize", 2048)) - if isinstance(size, bool) or size < 1: + if isinstance(batch_size, bool) or size < 1: msg = "batch_size must be >= 1" raise ImproperConfigurationError(msg) fast = bool(self.driver_features.get("enable_fastexport", False)) @@ -599,8 +605,27 @@ def load_from_storage( ) -> "StorageBridgeJob": """Load staged artifacts from storage into the target Teradata table.""" if kwargs: + validate_load_from_storage_kwargs(kwargs) return super().load_from_storage( - table, source, file_format=file_format, partitioner=partitioner, overwrite=overwrite, **kwargs + table, + source, + file_format=file_format, + partitioner=partitioner, + overwrite=overwrite, + storage_options=kwargs.get("storage_options"), + stage_table=kwargs.get("stage_table"), + conflict_keys=kwargs.get("conflict_keys"), + on_conflict=kwargs.get("on_conflict", "update"), + exclude_update_columns=kwargs.get("exclude_update_columns"), + drop_stage_table=kwargs.get("drop_stage_table", True), + drop_indexes=kwargs.get("drop_indexes", False), + concurrent_indexes=kwargs.get("concurrent_indexes", False), + checkpoint_table=kwargs.get("checkpoint_table"), + checkpoint_phase=kwargs.get("checkpoint_phase", "default"), + max_workers=kwargs.get("max_workers", 1), + executor=kwargs.get("executor", "thread"), + arrow_schema=kwargs.get("arrow_schema"), + type_casts=kwargs.get("type_casts"), ) arrow_table, inbound = self._read_storage_arrow(source, file_format=file_format) return self.load_from_arrow(table, arrow_table, partitioner=partitioner, overwrite=overwrite, telemetry=inbound) @@ -625,8 +650,7 @@ def _execute_cache_hit_with_cursor( self, sql: str, params: "tuple[Any, ...] | list[Any] | dict[str, Any]", cached: "CachedQuery" ) -> "SQLResult": """Execute a compiled cache hit while preserving FastExport, row coercion, and DDL commit rules.""" - _ = sql - compiled_sql = cached.compiled_sql + compiled_sql = self._format_wire_sql(cached.compiled_sql) returns_rows = cached.operation_profile.returns_rows exec_sql = ( maybe_prefix_fastexport( @@ -642,10 +666,8 @@ def _execute_cache_hit_with_cursor( names: list[str] = [] column_types: dict[str, str] = {} affected = 0 - exc_handler = self.handle_database_exceptions() - def _run(cursor: "TeradataRawCursor") -> None: - nonlocal names, column_types, affected + with self.with_cursor(self.connection) as cursor: cursor.arraysize = int(self.driver_features.get("arraysize", 2048)) if _sets_session_query_band(compiled_sql): self._session_query_band_dirty = True @@ -660,33 +682,25 @@ def _run(cursor: "TeradataRawCursor") -> None: while chunk := cursor.fetchmany(cursor.arraysize): rows.extend(coerce_teradata_row(row, description=description) for row in chunk) column_types = extract_arrow_column_types(description, column_names=names) - return - if getattr(self.connection, "autocommit", True) is False and is_ddl_or_session_band_sql(compiled_sql): - self.connection.commit() - affected = resolve_rowcount(cursor) - - with self.with_cursor(self.connection) as cursor: - if sys.exc_info()[1] is None: - with exc_handler: - _run(cursor) else: - self._run_with_exception_handler(exc_handler, _run, cursor) - self._check_pending_exception(exc_handler) + if getattr(self.connection, "autocommit", True) is False and is_ddl_or_session_band_sql(compiled_sql): + self.connection.commit() + affected = resolve_rowcount(cursor) if returns_rows: - return SQLResult( - statement=None, - data=rows, - rows_affected=len(rows), - operation_type=cached.operation_type, + result = self.build_cached_select_result( + sql, + params, + cached, + selected_data=rows, column_names=names, - total_count=len(rows), + data_row_count=len(rows), row_format="tuple", - metadata={"column_types": column_types}, + compiled_sql=exec_sql, ) - return SQLResult( - statement=None, data=[], rows_affected=affected, operation_type=cached.operation_type, total_count=0 - ) + result.metadata = {"column_types": column_types} + return result + return DMLResult(cached.operation_type, affected) def _release_open_work(self) -> None: for source in tuple(self._stream_sources): diff --git a/sqlspec/adapters/teradata/migrations.py b/sqlspec/adapters/teradata/migrations.py index 37f7471e1..d51d81b24 100644 --- a/sqlspec/adapters/teradata/migrations.py +++ b/sqlspec/adapters/teradata/migrations.py @@ -2,13 +2,16 @@ from typing import TYPE_CHECKING +from mypy_extensions import mypyc_attr + from sqlspec.adapters.teradata.core import TABLE_EXISTS_SQL, format_identifier, format_table_ref, split_schema_and_table -from sqlspec.builder import CreateTable, sql +from sqlspec.builder._ddl import CreateTable +from sqlspec.builder._factory import sql from sqlspec.migrations.schema import DropStatement from sqlspec.migrations.tracker import SyncMigrationTracker if TYPE_CHECKING: - from sqlspec.driver import SyncDriverAdapterBase + from sqlspec.driver._sync import SyncDriverAdapterBase __all__ = ("TeradataSyncMigrationTracker", "build_guarded_drop_table") @@ -30,6 +33,7 @@ def build_guarded_drop_table(table_name: str, statement: str | None = None) -> D ) +@mypyc_attr(allow_interpreted_subclasses=True) class TeradataSyncMigrationTracker(SyncMigrationTracker): """Track migrations using bounded Teradata metadata columns.""" diff --git a/sqlspec/adapters/teradata/pool.py b/sqlspec/adapters/teradata/pool.py index e1d7137d9..96de5b267 100644 --- a/sqlspec/adapters/teradata/pool.py +++ b/sqlspec/adapters/teradata/pool.py @@ -24,6 +24,7 @@ DatabaseConnectionError, ImproperConfigurationError, MissingDependencyError, + with_cause, ) from sqlspec.utils.module_loader import import_optional @@ -143,7 +144,7 @@ def new_connection(self) -> Any: "DatabaseError", "InterfaceError", }: - raise create_mapped_exception(exc) from exc + raise with_cause(create_mapped_exception(exc), exc) from exc raise try: connection.autocommit = self._autocommit_baseline diff --git a/sqlspec/base.py b/sqlspec/base.py index be4bef32a..9ad11b319 100644 --- a/sqlspec/base.py +++ b/sqlspec/base.py @@ -20,7 +20,7 @@ SyncConfigT, SyncDatabaseConfig, ) -from sqlspec.core import ( +from sqlspec.core.cache import ( CacheConfig, get_cache_config, get_cache_statistics, @@ -30,9 +30,10 @@ ) from sqlspec.driver._common import _CLOSED_CONNECTION from sqlspec.exceptions import ImproperConfigurationError -from sqlspec.extensions.events import AsyncEventChannel, SyncEventChannel +from sqlspec.extensions.events._channel import AsyncEventChannel, SyncEventChannel from sqlspec.loader import SQLFileLoader -from sqlspec.observability import ObservabilityConfig, ObservabilityRuntime, TelemetryDiagnostics +from sqlspec.observability._diagnostics import TelemetryDiagnostics +from sqlspec.observability._runtime import ObservabilityRuntime from sqlspec.typing import ConnectionT from sqlspec.utils.logging import get_logger from sqlspec.utils.type_guards import has_name @@ -42,10 +43,12 @@ from types import TracebackType from uuid import UUID - from sqlspec.core import SQL, ParameterDeclaration, StatementConfig + from sqlspec.core.parameters._declared import ParameterDeclaration + from sqlspec.core.statement import SQL, StatementConfig from sqlspec.driver._routing import AsyncDriverT, AsyncRoutingSession, SyncDriverT, SyncRoutingSession + from sqlspec.observability._config import ObservabilityConfig from sqlspec.typing import PoolT - from sqlspec.utils.tenant import TenantContextPayload + from sqlspec.utils.tenant._context import TenantContextPayload __all__ = ("SQLSpec", "coerce_sqlspec") @@ -218,13 +221,15 @@ def provide_connection( connection_context = config.provide_connection(*args, **kwargs) runtime = config.get_observability_runtime() + if not runtime.has_session_lifecycle: + return connection_context - if _is_async_context_manager(connection_context): - async_context = cast("AbstractAsyncContextManager[ConnectionT]", connection_context) - return _RuntimeAsyncContext[ConnectionT](async_context, runtime) + config_is_async = getattr(config, "is_async", None) + is_async = config_is_async if type(config_is_async) is bool else _is_async_context_manager(connection_context) + if is_async: + return _RuntimeAsyncContext(cast("AbstractAsyncContextManager[Any]", connection_context), runtime) - sync_context = cast("AbstractContextManager[ConnectionT]", connection_context) - return _RuntimeSyncContext[ConnectionT](sync_context, runtime) + return _RuntimeSyncContext(cast("AbstractContextManager[Any]", connection_context), runtime) @overload def provide_session( @@ -270,23 +275,31 @@ def provide_session( """ self._ensure_registered(config) - call_kwargs = dict(kwargs) - if statement_config is not None: - call_kwargs["statement_config"] = statement_config - if tenant is not None: - call_kwargs["tenant"] = tenant - if bypass_tenant: - call_kwargs["bypass_tenant"] = True - - session_context = config.provide_session(*args, **call_kwargs) + if statement_config is not None or tenant is not None or bypass_tenant: + call_kwargs = dict(kwargs) if kwargs else {} + if statement_config is not None: + call_kwargs["statement_config"] = statement_config + if tenant is not None: + call_kwargs["tenant"] = tenant + if bypass_tenant: + call_kwargs["bypass_tenant"] = True + session_context = config.provide_session(*args, **call_kwargs) + elif kwargs: + session_context = config.provide_session(*args, **kwargs) + else: + session_context = config.provide_session(*args) runtime = config.get_observability_runtime() + if not runtime.has_session_lifecycle and isinstance( + config, (SyncDatabaseConfig, AsyncDatabaseConfig, NoPoolSyncConfig, NoPoolAsyncConfig) + ): + return session_context - if _is_async_context_manager(session_context): - async_session = cast("AbstractAsyncContextManager[DriverT]", session_context) - return _RuntimeAsyncContext[DriverT](async_session, runtime, config) + config_is_async = getattr(config, "is_async", None) + is_async = config_is_async if type(config_is_async) is bool else _is_async_context_manager(session_context) + if is_async: + return _RuntimeAsyncContext(cast("AbstractAsyncContextManager[Any]", session_context), runtime, config) - sync_session = cast("AbstractContextManager[DriverT]", session_context) - return _RuntimeSyncContext[DriverT](sync_session, runtime, config) + return _RuntimeSyncContext(cast("AbstractContextManager[Any]", session_context), runtime, config) @overload def provide_replica_connection( @@ -949,7 +962,11 @@ def __enter__(self) -> ContextValueT: raw_connection: Any | None = None connection_created = False try: - prepared: Any = config._prepare_driver(value) # pyright: ignore[reportPrivateUsage] + prepared: Any = ( + value + if getattr(value, "_observability", None) is self._runtime + else cast("Any", config)._prepare_driver(value) + ) driver = prepared raw_connection = prepared.connection if raw_connection is not None and raw_connection is not _CLOSED_CONNECTION: @@ -1005,7 +1022,8 @@ def __exit__( if raw_connection is not None and raw_connection is not _CLOSED_CONNECTION: self._runtime.emit_connection_destroy_sync(raw_connection) finally: - value._mark_closed() + if value.connection is not _CLOSED_CONNECTION: + value._mark_closed() @mypyc_attr(allow_interpreted_subclasses=False) @@ -1033,7 +1051,11 @@ async def __aenter__(self) -> ContextValueT: enter_error: BaseException | None = None prepared: Any = None try: - prepared = config._prepare_driver(value) # pyright: ignore[reportPrivateUsage] + prepared = ( + value + if getattr(value, "_observability", None) is self._runtime + else cast("Any", config)._prepare_driver(value) + ) driver = prepared raw_connection = prepared.connection if raw_connection is not None and raw_connection is not _CLOSED_CONNECTION: @@ -1097,7 +1119,8 @@ async def __aexit__( await self._runtime.emit_connection_destroy_async(raw_connection) except BaseException as exc: exit_error = exc - value._mark_closed() + if value.connection is not _CLOSED_CONNECTION: + value._mark_closed() if exit_error is not None: raise exit_error return suppressed @@ -1140,7 +1163,7 @@ def __enter__(self) -> ContextValueT: if connection_created and connection is not None and connection is not _CLOSED_CONNECTION: runtime.emit_connection_destroy_sync(connection) finally: - if self._session: + if self._session and value.connection is not _CLOSED_CONNECTION: value._mark_closed() raise self._runtime = runtime @@ -1167,7 +1190,7 @@ def __exit__( if connection is not None and connection is not _CLOSED_CONNECTION: runtime.emit_connection_destroy_sync(connection) finally: - if self._session and value is not None: + if self._session and value is not None and value.connection is not _CLOSED_CONNECTION: value._mark_closed() @@ -1213,7 +1236,7 @@ async def __aenter__(self) -> ContextValueT: await runtime.emit_connection_destroy_async(connection) except BaseException as exc: enter_error = exc - if self._session: + if self._session and value.connection is not _CLOSED_CONNECTION: value._mark_closed() raise enter_error self._runtime = runtime @@ -1245,7 +1268,7 @@ async def __aexit__( await runtime.emit_connection_destroy_async(connection) except BaseException as exc: exit_error = exc - if self._session and value is not None: + if self._session and value is not None and value.connection is not _CLOSED_CONNECTION: value._mark_closed() if exit_error is not None: raise exit_error diff --git a/sqlspec/builder/_base.py b/sqlspec/builder/_base.py index 3456c7c88..c239d5989 100644 --- a/sqlspec/builder/_base.py +++ b/sqlspec/builder/_base.py @@ -40,22 +40,17 @@ _resolve_dialect, ) from sqlspec.builder._vector_distance import has_vector_distance_ancestor, register_vector_distance_generator -from sqlspec.core import ( - SQL, - ParameterStyle, - ParameterStyleConfig, - ParameterValidator, - StatementConfig, - get_cache, - get_cache_config, - hash_optimized_expression, -) +from sqlspec.core.cache import get_cache, get_cache_config from sqlspec.core.compiler import validate_placeholder_names from sqlspec.core.filters import StatementFilter -from sqlspec.core.hashing import _expression_cache_fingerprint +from sqlspec.core.hashing import _expression_cache_fingerprint, hash_optimized_expression +from sqlspec.core.parameters._types import ParameterStyle, ParameterStyleConfig +from sqlspec.core.parameters._validator import ParameterValidator from sqlspec.core.query_modifiers import is_valid_parameter_name, sanitize_parameter_name -from sqlspec.data_dictionary import DialectConfig, get_dialect_config -from sqlspec.exceptions import SQLBuilderError +from sqlspec.core.statement import SQL, StatementConfig +from sqlspec.data_dictionary._registry import get_dialect_config +from sqlspec.data_dictionary._types import DialectConfig +from sqlspec.exceptions import SQLBuilderError, with_cause from sqlspec.utils.logging import get_logger from sqlspec.utils.type_guards import has_expression_and_parameters, is_expression from sqlspec.utils.uuids import uuid4 @@ -104,6 +99,7 @@ def __hash__(self) -> int: return hash((self.sql, frozenset(self.parameters.items()), self.dialect)) +@mypyc_attr(allow_interpreted_subclasses=True) class QueryBuilder: """Abstract base class for SQL query builders. @@ -633,7 +629,7 @@ def _raise_builder_error(message: str, cause: BaseException | None = None) -> No Raises: SQLBuilderError: Always raises this exception. """ - raise SQLBuilderError(message) from cause + raise with_cause(SQLBuilderError(message), cause) from cause @staticmethod def _raise_invalid_expression_type(expression: Any) -> NoReturn: @@ -673,7 +669,7 @@ def _raise_cte_parse_error(cause: BaseException) -> NoReturn: SQLBuilderError: Always raised with chained cause """ msg = f"Failed to parse CTE query: {cause!s}" - raise SQLBuilderError(msg) from cause + raise with_cause(SQLBuilderError(msg), cause) from cause def _prepare_expression_for_build(self, dialect: DialectType = None, *, copy: bool = True) -> exp.Expr: """Validate builder state and return the expression ready for optimization and rendering.""" @@ -1395,6 +1391,7 @@ def _should_identify(self, dialect: "DialectType | str | None") -> bool: return str(dialect).lower() not in _UPPER_FOLDING_DIALECTS +@mypyc_attr(allow_interpreted_subclasses=True) class ExpressionBuilder(QueryBuilder): """Builder wrapper for a pre-parsed SQLGlot expression.""" diff --git a/sqlspec/builder/_column.py b/sqlspec/builder/_column.py index 0aa0d40f1..033edae76 100644 --- a/sqlspec/builder/_column.py +++ b/sqlspec/builder/_column.py @@ -11,7 +11,7 @@ from sqlglot import exp from sqlspec.builder._vector_distance import VectorDistance, validate_vector_elements -from sqlspec.core import NullsPlacement, ordered +from sqlspec.core._ordering import NullsPlacement, ordered __all__ = ("Column", "ColumnExpression", "ColumnNamespace", "FunctionColumn") diff --git a/sqlspec/builder/_ddl.py b/sqlspec/builder/_ddl.py index 7bb9787a5..aea30cdbd 100644 --- a/sqlspec/builder/_ddl.py +++ b/sqlspec/builder/_ddl.py @@ -29,9 +29,9 @@ _truthy_metadata_flag, parse_order_expression, ) -from sqlspec.core import SQL from sqlspec.core.splitter import contains_statement_terminator -from sqlspec.data_dictionary import ( +from sqlspec.core.statement import SQL +from sqlspec.data_dictionary._types import ( DDLResult, ForeignKeyMetadata, MetadataFidelity, @@ -39,7 +39,7 @@ MetadataSupport, ObjectIdentity, ) -from sqlspec.exceptions import SQLBuilderError +from sqlspec.exceptions import SQLBuilderError, with_cause from sqlspec.utils.arrow_helpers import sql_type_for_category, sql_type_from_arrow_type from sqlspec.utils.type_guards import has_sqlglot_expression @@ -468,6 +468,7 @@ def build_constraint_expression( return None +@mypyc_attr(allow_interpreted_subclasses=True) class DDLBuilder(QueryBuilder): """Base class for DDL builders (CREATE, DROP, ALTER, etc).""" @@ -542,6 +543,7 @@ def _resolve_select_query(self, query: object, context: str, *, require_select_t return select_expr +@mypyc_attr(allow_interpreted_subclasses=True) @trait class _IfExistsDDLMixin: __slots__ = () @@ -553,6 +555,7 @@ def if_exists(self) -> Self: return self +@mypyc_attr(allow_interpreted_subclasses=True) @trait class _CascadeRestrictDDLMixin: __slots__ = () @@ -568,6 +571,7 @@ def restrict(self) -> Self: return self +@mypyc_attr(allow_interpreted_subclasses=True) @trait class _IfNotExistsDDLMixin: __slots__ = () @@ -4463,7 +4467,7 @@ def _parse_ddl_expression(text: str, *, label: str, dialect: "DialectType | None statements = sqlglot.parse(text, read=norm_dialect) except (ParseError, TokenError) as exc: msg = f"Invalid {label}: {text!r}" - raise SQLBuilderError(msg) from exc + raise with_cause(SQLBuilderError(msg), exc) from exc if len(statements) != 1 or statements[0] is None: msg = f"Invalid {label}: {text!r}" raise SQLBuilderError(msg) @@ -4494,6 +4498,6 @@ def _coerce_int_argument(value: Any, *, name: str) -> int: return int(stripped) except ValueError as exc: msg = f"{name} must be an integer, got {value!r}" - raise SQLBuilderError(msg) from exc + raise with_cause(SQLBuilderError(msg), exc) from exc msg = f"{name} must be an integer, got {type(value).__name__}" raise SQLBuilderError(msg) diff --git a/sqlspec/builder/_delete.py b/sqlspec/builder/_delete.py index f3cef625a..2455ff9ed 100644 --- a/sqlspec/builder/_delete.py +++ b/sqlspec/builder/_delete.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any +from mypy_extensions import mypyc_attr from sqlglot import exp from sqlspec.builder._base import BuiltQuery, QueryBuilder @@ -20,6 +21,7 @@ __all__ = ("Delete",) +@mypyc_attr(allow_interpreted_subclasses=True) class Delete(QueryBuilder, WhereClauseMixin, ReturningClauseMixin, DeleteFromClauseMixin, ExplainMixin): """Builder for DELETE statements. diff --git a/sqlspec/builder/_dml.py b/sqlspec/builder/_dml.py index 5fe59ccbd..da6dba9ba 100644 --- a/sqlspec/builder/_dml.py +++ b/sqlspec/builder/_dml.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, Final, Self, cast import sqlglot -from mypy_extensions import trait +from mypy_extensions import mypyc_attr, trait from sqlglot import exp from sqlglot.errors import ParseError, TokenError @@ -19,9 +19,9 @@ unwrap_expression_source, ) from sqlspec.core.query_modifiers import is_valid_parameter_name, parse_column_for_condition -from sqlspec.exceptions import SQLBuilderError, SQLSpecError +from sqlspec.exceptions import SQLBuilderError, SQLSpecError, with_cause from sqlspec.protocols import SQLBuilderProtocol -from sqlspec.utils.serializers import schema_dump +from sqlspec.utils.serializers._schema import schema_dump from sqlspec.utils.type_guards import is_dict if TYPE_CHECKING: @@ -48,6 +48,7 @@ _RETURNING_WORD_RE: Final["re.Pattern[str]"] = re.compile(rf"{_RETURNING_WORD_PART}(?:\.{_RETURNING_WORD_PART}){{0,2}}") +@mypyc_attr(allow_interpreted_subclasses=True) @trait class DeleteFromClauseMixin: """Mixin providing FROM clause support for DELETE builders.""" @@ -74,6 +75,7 @@ def from_(self, table: str) -> Self: return self +@mypyc_attr(allow_interpreted_subclasses=True) @trait class InsertIntoClauseMixin: __slots__ = () @@ -98,6 +100,7 @@ def into(self, table: str) -> Self: return self +@mypyc_attr(allow_interpreted_subclasses=True) @trait class InsertValuesMixin: __slots__ = () @@ -245,6 +248,7 @@ def _align_mapping_to_columns(self, row: "Mapping[str, Any]", column_defs: list[ raise SQLBuilderError(msg) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class InsertFromSelectMixin: __slots__ = () @@ -310,6 +314,7 @@ def from_select(self, select_builder: SQLBuilderProtocol) -> Self: return self +@mypyc_attr(allow_interpreted_subclasses=True) @trait class UpdateTableClauseMixin: __slots__ = () @@ -331,6 +336,7 @@ def table(self, table_name: str, alias: str | None = None) -> Self: return self +@mypyc_attr(allow_interpreted_subclasses=True) @trait class UpdateSetClauseMixin: __slots__ = () @@ -396,7 +402,7 @@ def _assignment_target(self, col: Any) -> exp.Expr: try: return parse_column_for_condition(col) except SQLSpecError as exc: - raise SQLBuilderError(str(exc)) from exc + raise with_cause(SQLBuilderError(str(exc)), exc) from exc return exp.column(col, copy=False) def _process_update_value(self, val: Any, col: Any) -> exp.Expr: @@ -411,6 +417,7 @@ def _process_update_value(self, val: Any, col: Any) -> exp.Expr: return placeholder +@mypyc_attr(allow_interpreted_subclasses=True) @trait class UpdateFromClauseMixin: __slots__ = () @@ -532,6 +539,7 @@ def from_(self, table: str | exp.Expr | Any, alias: str | None = None) -> Self: return self +@mypyc_attr(allow_interpreted_subclasses=True) @trait class ReturningClauseMixin: """Mixin providing RETURNING clause support for DML builders.""" @@ -600,7 +608,7 @@ def _parse_returning_column(column: str, dialect: "DialectType | None") -> exp.E try: statements = sqlglot.parse(f"SELECT {stripped}", dialect=_normalize_dialect(dialect)) except (ParseError, TokenError) as exc: - raise SQLBuilderError(msg) from exc + raise with_cause(SQLBuilderError(msg), exc) from exc select = statements[0] if len(statements) == 1 else None if not isinstance(select, exp.Select) or len(select.expressions) != 1: raise SQLBuilderError(msg) diff --git a/sqlspec/builder/_explain.py b/sqlspec/builder/_explain.py index db131024d..4a9c863ef 100644 --- a/sqlspec/builder/_explain.py +++ b/sqlspec/builder/_explain.py @@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Any, Final, Self -from mypy_extensions import trait +from mypy_extensions import mypyc_attr, trait from sqlglot import Dialect, exp -from sqlspec.core import SQL, StatementConfig from sqlspec.core.explain import ( ORACLE_EXPLAIN_DISPLAY_CURSOR_META_KEY, ORACLE_EXPLAIN_PREFIX, @@ -18,6 +17,7 @@ ExplainFormat, ExplainOptions, ) +from sqlspec.core.statement import SQL, StatementConfig from sqlspec.exceptions import SQLBuilderError from sqlspec.utils.type_guards import has_expression_and_sql, has_parameter_builder, is_expression @@ -293,6 +293,7 @@ def build_explain_sql(statement_sql: str, options: "ExplainOptions", dialect: "D return build_generic_explain(statement_sql, options) +@mypyc_attr(allow_interpreted_subclasses=True) class Explain: """Builder for EXPLAIN statements with dialect-aware rendering. @@ -614,6 +615,7 @@ def _resolve_statement_config(self, target_dialect: "DialectType | None") -> "St return source_config +@mypyc_attr(allow_interpreted_subclasses=True) @trait class ExplainMixin: """Mixin to add .explain() method to QueryBuilder subclasses. diff --git a/sqlspec/builder/_factory.py b/sqlspec/builder/_factory.py index 0d951ee96..bf2dd7495 100644 --- a/sqlspec/builder/_factory.py +++ b/sqlspec/builder/_factory.py @@ -67,9 +67,9 @@ from sqlspec.builder._select import Case, Select, SubqueryBuilder, WindowFunctionBuilder from sqlspec.builder._update import Update from sqlspec.builder._values import Values -from sqlspec.core import SQL, StatementConfig from sqlspec.core.explain import ExplainFormat, ExplainOptions -from sqlspec.exceptions import SQLBuilderError +from sqlspec.core.statement import SQL, StatementConfig +from sqlspec.exceptions import SQLBuilderError, with_cause from sqlspec.utils.logging import get_logger if TYPE_CHECKING: @@ -197,7 +197,7 @@ def __call__(self, statement: str, dialect: DialectType = None) -> "Any": parsed_expr = sqlglot.parse_one(statement, read=resolved_dialect) except Exception as e: msg = f"Failed to parse SQL: {e}" - raise SQLBuilderError(msg) from e + raise with_cause(SQLBuilderError(msg), e) from e actual_type = type(parsed_expr).__name__.upper() actual_type_str = actual_type if actual_type_str == "SELECT" or ( @@ -929,7 +929,7 @@ def raw(sql_fragment: str, **parameters: Any) -> "exp.Expr | SQL": parsed: exp.Expr = exp.maybe_parse(sql_fragment) except Exception as e: msg = f"Failed to parse raw SQL fragment '{sql_fragment}': {e}" - raise SQLBuilderError(msg) from e + raise with_cause(SQLBuilderError(msg), e) from e return parsed return SQL(sql_fragment, parameters) @@ -1761,7 +1761,7 @@ def _coerce_numeric_argument(value: Any, *, name: str) -> int | float: parsed = float(stripped) except ValueError as exc: msg = f"{name} must be a numeric value, got {value!r}" - raise SQLBuilderError(msg) from exc + raise with_cause(SQLBuilderError(msg), exc) from exc if not math.isfinite(parsed): msg = f"{name} must be a finite numeric value, got {value!r}" raise SQLBuilderError(msg) diff --git a/sqlspec/builder/_insert.py b/sqlspec/builder/_insert.py index 717a8f418..62133a482 100644 --- a/sqlspec/builder/_insert.py +++ b/sqlspec/builder/_insert.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Final, Self +from mypy_extensions import mypyc_attr from sqlglot import exp from sqlspec.builder._base import QueryBuilder @@ -13,8 +14,8 @@ from sqlspec.builder._explain import ExplainMixin from sqlspec.builder._parsing_utils import resolve_value_expression from sqlspec.core.query_modifiers import parse_column_for_condition -from sqlspec.exceptions import SQLBuilderError, SQLSpecError -from sqlspec.utils.serializers import schema_dump, serialize_collection +from sqlspec.exceptions import SQLBuilderError, SQLSpecError, with_cause +from sqlspec.utils.serializers._schema import schema_dump, serialize_collection if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -27,6 +28,7 @@ ERR_MSG_EXPRESSION_NOT_INITIALIZED: Final[str] = "Internal error: base expression not initialized." +@mypyc_attr(allow_interpreted_subclasses=True) class Insert( QueryBuilder, ReturningClauseMixin, InsertValuesMixin, InsertFromSelectMixin, InsertIntoClauseMixin, ExplainMixin ): @@ -277,7 +279,7 @@ def _build_conflict_keys(self) -> list[exp.Expr] | None: try: keys.append(parse_column_for_condition(col)) except SQLSpecError as exc: - raise SQLBuilderError(str(exc)) from exc + raise with_cause(SQLBuilderError(str(exc)), exc) from exc else: keys.append(exp.to_identifier(col)) return keys @@ -294,7 +296,7 @@ def _build_conflict_set_expressions(builder: "Insert", kwargs: dict[str, Any]) - try: target = parse_column_for_condition(col) except SQLSpecError as exc: - raise SQLBuilderError(str(exc)) from exc + raise with_cause(SQLBuilderError(str(exc)), exc) from exc leaf_name = col.rsplit(".", maxsplit=1)[-1] value_expr = resolve_value_expression(val, builder, leaf_name) if value_expr is None: diff --git a/sqlspec/builder/_join.py b/sqlspec/builder/_join.py index 79bea72b7..995f3e0cd 100644 --- a/sqlspec/builder/_join.py +++ b/sqlspec/builder/_join.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Self, Union, cast, final -from mypy_extensions import trait +from mypy_extensions import mypyc_attr, trait from sqlglot import exp from sqlspec.builder._base import BuiltQuery, QueryBuilder, register_subquery_fragment @@ -19,8 +19,8 @@ unwrap_expression_source, ) from sqlspec.builder._temporal import create_temporal_version, register_version_generators -from sqlspec.core import SQL from sqlspec.core.query_modifiers import is_valid_parameter_name +from sqlspec.core.statement import SQL from sqlspec.exceptions import SQLBuilderError from sqlspec.utils.type_guards import has_parameter_builder @@ -47,6 +47,7 @@ def build_join_clause( return join_expr +@mypyc_attr(allow_interpreted_subclasses=True) @trait class JoinClauseMixin: """Mixin providing JOIN clause methods for SELECT builders. diff --git a/sqlspec/builder/_merge.py b/sqlspec/builder/_merge.py index d841b824d..77d1e593d 100644 --- a/sqlspec/builder/_merge.py +++ b/sqlspec/builder/_merge.py @@ -13,7 +13,7 @@ from uuid import UUID import sqlglot -from mypy_extensions import trait +from mypy_extensions import mypyc_attr, trait from sqlglot import exp from sqlglot.errors import SqlglotError @@ -28,9 +28,9 @@ unwrap_expression_source, ) from sqlspec.core.splitter import contains_statement_terminator -from sqlspec.exceptions import DialectNotSupportedError, SQLBuilderError +from sqlspec.exceptions import DialectNotSupportedError, SQLBuilderError, with_cause from sqlspec.utils.dispatch import TypeDispatcher -from sqlspec.utils.serializers import to_json +from sqlspec.utils.serializers._json import to_json if TYPE_CHECKING: from sqlglot.dialects.dialect import DialectType @@ -91,6 +91,7 @@ ) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class _MergeAssignmentMixin: """Shared assignment helpers for MERGE clause mixins.""" @@ -115,6 +116,7 @@ def _append_when(self, current_expr: exp.Merge, when_node: exp.When) -> None: whens.append("expressions", when_node) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class MergeIntoClauseMixin: """Mixin providing INTO clause for MERGE builders.""" @@ -143,6 +145,7 @@ def into(self, table: str | exp.Expr, alias: str | None = None) -> Self: return self +@mypyc_attr(allow_interpreted_subclasses=True) @trait class MergeUsingClauseMixin(_MergeAssignmentMixin): """Mixin providing USING clause for MERGE builders.""" @@ -654,6 +657,7 @@ def _create_select_union_source( return exp.paren(select_expr, copy=False) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class MergeOnClauseMixin: """Mixin providing ON clause for MERGE builders.""" @@ -671,6 +675,7 @@ def on(self, condition: str | exp.Expr) -> Self: return self +@mypyc_attr(allow_interpreted_subclasses=True) @trait class MergeMatchedClauseMixin(_MergeAssignmentMixin): """Mixin providing WHEN MATCHED THEN ... clauses for MERGE builders.""" @@ -718,6 +723,7 @@ def when_matched_then_delete(self, condition: str | exp.Expr | None = None) -> S return self +@mypyc_attr(allow_interpreted_subclasses=True) @trait class MergeNotMatchedClauseMixin(_MergeAssignmentMixin): """Mixin providing WHEN NOT MATCHED THEN ... clauses for MERGE builders.""" @@ -789,6 +795,7 @@ def when_not_matched_then_insert( return self +@mypyc_attr(allow_interpreted_subclasses=True) @trait class MergeNotMatchedBySourceClauseMixin(_MergeAssignmentMixin): """Mixin providing WHEN NOT MATCHED BY SOURCE THEN ... clauses.""" @@ -823,6 +830,7 @@ def when_not_matched_by_source_then_delete(self) -> Self: return self +@mypyc_attr(allow_interpreted_subclasses=True) class Merge( QueryBuilder, MergeUsingClauseMixin, @@ -1004,7 +1012,7 @@ def _parse_merge_condition( statements = sqlglot.parse(condition, read=dialect_name) except SqlglotError as exc: msg = f"Could not parse {condition_label}: {condition}" - raise SQLBuilderError(msg) from exc + raise with_cause(SQLBuilderError(msg), exc) from exc if len(statements) != 1 or statements[0] is None: msg = f"Could not parse {condition_label}: {condition}" raise SQLBuilderError(msg) diff --git a/sqlspec/builder/_parsing_utils.py b/sqlspec/builder/_parsing_utils.py index 6bb444b0a..f6f14d4cb 100644 --- a/sqlspec/builder/_parsing_utils.py +++ b/sqlspec/builder/_parsing_utils.py @@ -19,10 +19,13 @@ from sqlspec.builder._column import Column, ColumnExpression, FunctionColumn from sqlspec.builder._expression_wrappers import ExpressionWrapper -from sqlspec.core import SQL, ParameterStyle, ParameterValidator, default_nulls, ordered +from sqlspec.core._ordering import default_nulls, ordered +from sqlspec.core.parameters._types import ParameterStyle +from sqlspec.core.parameters._validator import ParameterValidator from sqlspec.core.query_modifiers import is_valid_parameter_name, parse_column_for_condition from sqlspec.core.splitter import contains_statement_terminator -from sqlspec.exceptions import SQLBuilderError, SQLSpecError +from sqlspec.core.statement import SQL +from sqlspec.exceptions import SQLBuilderError, SQLSpecError, with_cause from sqlspec.utils.type_guards import has_expression_attr, has_parameter_builder if TYPE_CHECKING: @@ -246,7 +249,7 @@ def parse_column_expression( parsed: exp.Expr | None = exp.maybe_parse(column_input, dialect=resolved_dialect) except SqlglotError as exc: msg = f"Invalid column expression: {column_input!r}" - raise SQLBuilderError(msg) from exc + raise with_cause(SQLBuilderError(msg), exc) from exc if parsed is None: msg = f"Invalid column expression: {column_input!r}" raise SQLBuilderError(msg) @@ -352,7 +355,7 @@ def parse_table_function( try: statements = sqlglot.parse(f"SELECT * FROM {function_input}", dialect=_table_reader(dialect)) except SqlglotError as error: - raise SQLBuilderError(msg) from error + raise with_cause(SQLBuilderError(msg), error) from error statement = statements[0] if len(statements) == 1 else None if not isinstance(statement, exp.Select) or _set_arguments(statement) != {"expressions", "from_"}: @@ -423,7 +426,7 @@ def parse_order_expression(order_input: str | exp.Expr, *, dialect: "DialectType parsed = maybe_parse(order_value, into=exp.Ordered, dialect=resolved_dialect) except SqlglotError as exc: msg = f"Invalid ORDER BY expression: {order_input!r}" - raise SQLBuilderError(msg) from exc + raise with_cause(SQLBuilderError(msg), exc) from exc if parsed: inner = parsed.this if isinstance(parsed, exp.Ordered) and isinstance(parsed.this, exp.Expr) else parsed _validate_scalar_ast_nodes(inner, raw=order_value) @@ -514,7 +517,7 @@ def parse_condition_expression(condition_input: str | exp.Expr | tuple[str, Any] parsed = exp.condition(condition_input, dialect=resolved_dialect, copy=False) except SqlglotError as exc: msg = f"Invalid condition expression: {condition_input!r}" - raise SQLBuilderError(msg) from exc + raise with_cause(SQLBuilderError(msg), exc) from exc if ( isinstance(parsed, (*_FORBIDDEN_STATEMENT_NODES, exp.Select, exp.SetOperation)) @@ -797,7 +800,7 @@ def _read_table_tokens(text: str, dialect: "DialectType | None", error: str) -> try: tokens = list(_table_reader(dialect).tokenize(text)) except SqlglotError as exc: - raise SQLBuilderError(error) from exc + raise with_cause(SQLBuilderError(error), exc) from exc if not tokens or any(token.comments for token in tokens) or _has_invisible_unquoted_text(text, tokens): raise SQLBuilderError(error) return tokens @@ -1009,7 +1012,7 @@ def _coerce_column(value: str | exp.Expr | Column | FunctionColumn | ColumnExpre try: return parse_column_for_condition(value) except SQLSpecError as exc: - raise SQLBuilderError(str(exc)) from exc + raise with_cause(SQLBuilderError(str(exc)), exc) from exc if isinstance(value, (Column, FunctionColumn, ColumnExpression)): return value.sqlglot_expression if isinstance(value, ExpressionWrapper): diff --git a/sqlspec/builder/_select.py b/sqlspec/builder/_select.py index cf0c57f56..9b4d19b97 100644 --- a/sqlspec/builder/_select.py +++ b/sqlspec/builder/_select.py @@ -38,8 +38,10 @@ to_expression, unwrap_expression_source, ) -from sqlspec.core import SQL, ParameterStyle, apply_direction, ordered +from sqlspec.core._ordering import apply_direction, ordered +from sqlspec.core.parameters._types import ParameterStyle from sqlspec.core.query_modifiers import expr_eq, expr_gt, expr_gte, expr_lt, expr_lte, expr_neq, expr_not_like +from sqlspec.core.statement import SQL from sqlspec.exceptions import SQLBuilderError from sqlspec.utils.type_guards import has_parameter_builder, is_iterable_parameters @@ -216,6 +218,7 @@ def _build_function_expression(self) -> exp.Expr: return exp.Anonymous(this=self._function_name, expressions=expressions) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class SelectClauseMixin: """Mixin providing SELECT clause methods.""" @@ -365,6 +368,7 @@ def group_by_grouping_sets(self, *column_sets: tuple[Any, ...] | list[Any]) -> S return self.group_by(_grouping_sets_expression(*column_sets)) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class OrderByClauseMixin: __slots__ = () @@ -395,6 +399,7 @@ def order_by( return cast("Self", builder) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class LimitOffsetClauseMixin: __slots__ = () @@ -421,6 +426,7 @@ def offset(self, value: int) -> Self: return cast("Self", builder) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class WhereClauseMixin: __slots__ = () @@ -992,6 +998,7 @@ def _build_not_exists_condition(self, subquery: Any) -> exp.Expr: return exp.Not(this=self._build_exists_condition(subquery)) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class HavingClauseMixin: __slots__ = () @@ -1014,6 +1021,7 @@ def having(self, condition: str | exp.Expr | exp.Condition | tuple[str, Any] | t return cast("Self", builder) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class PivotClauseMixin: __slots__ = () @@ -1064,6 +1072,7 @@ def pivot( return cast("Select", self) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class UnpivotClauseMixin: __slots__ = () @@ -1110,6 +1119,7 @@ def unpivot( return cast("Select", self) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class CommonTableExpressionMixin: """Mixin for Common Table Expression (CTE) support. @@ -1121,6 +1131,7 @@ class CommonTableExpressionMixin: __slots__ = () +@mypyc_attr(allow_interpreted_subclasses=True) @trait class SetOperationMixin: __slots__ = () @@ -1189,6 +1200,7 @@ def _combine_with_other(self, other: Any, *, operator: str, distinct: bool) -> S TABLE_HINT_PATTERN: Final[str] = r"\b{}\b(\s+AS\s+\w+)?" +@mypyc_attr(allow_interpreted_subclasses=True) class Select( QueryBuilder, WhereClauseMixin, diff --git a/sqlspec/builder/_update.py b/sqlspec/builder/_update.py index 37b15a76d..a28bd5217 100644 --- a/sqlspec/builder/_update.py +++ b/sqlspec/builder/_update.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Self, cast +from mypy_extensions import mypyc_attr from sqlglot import exp from sqlspec.builder._base import BuiltQuery, QueryBuilder @@ -29,6 +30,7 @@ __all__ = ("Update",) +@mypyc_attr(allow_interpreted_subclasses=True) class Update( QueryBuilder, WhereClauseMixin, diff --git a/sqlspec/builder/_values.py b/sqlspec/builder/_values.py index 6400ab33e..2b045765a 100644 --- a/sqlspec/builder/_values.py +++ b/sqlspec/builder/_values.py @@ -7,6 +7,7 @@ from collections.abc import Mapping, Sequence from typing import Any, Self +from mypy_extensions import mypyc_attr from sqlglot import exp from sqlspec.builder._base import QueryBuilder @@ -16,6 +17,7 @@ __all__ = ("Values",) +@mypyc_attr(allow_interpreted_subclasses=True) class Values(QueryBuilder): """Builder for SQL VALUES clauses. diff --git a/sqlspec/builder/_vector_distance.py b/sqlspec/builder/_vector_distance.py index c4e3a653e..18403aa27 100644 --- a/sqlspec/builder/_vector_distance.py +++ b/sqlspec/builder/_vector_distance.py @@ -15,7 +15,7 @@ from sqlspec.builder._generation import generator_class_for_dialect, invalidate_generator_dispatch from sqlspec.core.splitter import contains_statement_terminator -from sqlspec.exceptions import DialectNotSupportedError +from sqlspec.exceptions import DialectNotSupportedError, with_cause if TYPE_CHECKING: from sqlglot.generator import Generator @@ -547,7 +547,7 @@ def _validate_oracle_where_clause(where_clause: str) -> str: statements = sqlglot.parse(stripped, read="oracle") except SqlglotError as exc: msg = f"Invalid Oracle WHERE clause: {where_clause!r}" - raise ValueError(msg) from exc + raise with_cause(ValueError(msg), exc) from exc if ( not tokens or any(token.comments or token.token_type == TokenType.SEMICOLON for token in tokens) @@ -591,7 +591,7 @@ def _extract_array_number_literal(item: exp.Expr) -> str: numeric_val = float(raw_text) except ValueError as exc: msg = f"Vector array literal elements must be numeric literals, got {item!r}" - raise ValueError(msg) from exc + raise with_cause(ValueError(msg), exc) from exc if not math.isfinite(numeric_val): msg = f"Vector array literal elements must be finite numbers, got {item!r}" raise ValueError(msg) diff --git a/sqlspec/config.py b/sqlspec/config.py index 985ad860c..f6842e75f 100644 --- a/sqlspec/config.py +++ b/sqlspec/config.py @@ -1032,6 +1032,7 @@ class DatabaseConfigProtocol(ABC, Generic[ConnectionT, PoolT, DriverT]): _default_statement_config: "ClassVar[StatementConfig]" is_async: "ClassVar[bool]" = False supports_connection_pooling: "ClassVar[bool]" = False + supports_transactions: "ClassVar[bool]" = True supports_transactional_ddl: "ClassVar[bool]" = False supports_native_arrow_import: "ClassVar[bool]" = False supports_native_arrow_export: "ClassVar[bool]" = False diff --git a/sqlspec/core/_cursor.py b/sqlspec/core/_cursor.py index e80dc0671..869df7b3d 100644 --- a/sqlspec/core/_cursor.py +++ b/sqlspec/core/_cursor.py @@ -16,7 +16,7 @@ from typing import Any from uuid import UUID -from sqlspec.exceptions import ImproperConfigurationError, InvalidCursorError +from sqlspec.exceptions import ImproperConfigurationError, InvalidCursorError, with_cause __all__ = ( "CURSOR_TOKEN_MAX_LENGTH", @@ -351,7 +351,7 @@ def decode_cursor( payload = json.loads(raw_bytes.decode("utf-8")) except (UnicodeDecodeError, ValueError, RecursionError) as exc: msg = "Invalid pagination cursor: malformed token" - raise InvalidCursorError(msg) from exc + raise with_cause(InvalidCursorError(msg), exc) from exc if ( not isinstance(payload, dict) or type(payload.get("v")) is not int @@ -370,7 +370,7 @@ def decode_cursor( values = tuple(_decode_value(entry) for entry in payload["k"]) except (ValueError, TypeError, ArithmeticError, LookupError) as exc: msg = "Invalid pagination cursor: malformed token" - raise InvalidCursorError(msg) from exc + raise with_cause(InvalidCursorError(msg), exc) from exc return DecodedCursor(values, payload["d"] == "b") @@ -458,7 +458,7 @@ def _decode_signed_parts(parts: "list[str]", *, secret: bytes | None, encrypted: return base64.urlsafe_b64decode(body + "=" * (-len(body) % 4)) except binascii.Error as exc: msg = "Invalid pagination cursor: malformed token" - raise InvalidCursorError(msg) from exc + raise with_cause(InvalidCursorError(msg), exc) from exc def _decode_encrypted_parts(parts: "list[str]", *, secret: bytes | None, encrypted: bool) -> bytes: @@ -485,7 +485,7 @@ def _decode_encrypted_parts(parts: "list[str]", *, secret: bytes | None, encrypt sealed = base64.urlsafe_b64decode(parts[1] + "=" * (-len(parts[1]) % 4)) except binascii.Error as exc: msg = "Invalid pagination cursor: malformed token" - raise InvalidCursorError(msg) from exc + raise with_cause(InvalidCursorError(msg), exc) from exc if len(sealed) < _NONCE_LENGTH: msg = "Invalid pagination cursor: malformed token" raise InvalidCursorError(msg) @@ -502,7 +502,7 @@ def _parse_timestamp_claim(raw_value: Any) -> float: numeric_value = float(raw_value) except OverflowError as exc: msg = "Invalid pagination cursor: malformed token" - raise InvalidCursorError(msg) from exc + raise with_cause(InvalidCursorError(msg), exc) from exc if not math.isfinite(numeric_value): msg = "Invalid pagination cursor: malformed token" raise InvalidCursorError(msg) diff --git a/sqlspec/core/_pipeline.py b/sqlspec/core/_pipeline.py index ad83716fc..94adb2535 100644 --- a/sqlspec/core/_pipeline.py +++ b/sqlspec/core/_pipeline.py @@ -4,7 +4,7 @@ import hashlib import os from collections import OrderedDict -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast from mypy_extensions import mypyc_attr @@ -49,9 +49,26 @@ ) +_NEXT_PIPELINE_EPOCH: int = 1 + + +def _allocate_pipeline_epoch() -> int: + global _NEXT_PIPELINE_EPOCH + epoch = _NEXT_PIPELINE_EPOCH + _NEXT_PIPELINE_EPOCH += 1 + return epoch + + @mypyc_attr(allow_interpreted_subclasses=False) class StatementPipelineRegistry: - __slots__ = ("_cache_enabled", "_max_pipelines", "_pipeline_cache_size", "_pipeline_parse_cache_size", "_pipelines") + __slots__ = ( + "_cache_enabled", + "_epoch", + "_max_pipelines", + "_pipeline_cache_size", + "_pipeline_parse_cache_size", + "_pipelines", + ) def __init__( self, @@ -65,6 +82,7 @@ def __init__( self._pipeline_cache_size = cache_size self._pipeline_parse_cache_size = parse_cache_size self._cache_enabled = cache_enabled + self._epoch = _allocate_pipeline_epoch() def compile( self, @@ -75,11 +93,26 @@ def compile( expression: "exp.Expr | None" = None, param_fingerprint: "Any | None" = None, ) -> "CompiledSQL": + record_metrics = _RECORD_PIPELINE_METRICS + if config._cached_pipeline_epoch == self._epoch: + cached_pipeline = config._cached_pipeline + if cached_pipeline is not None and (cached_pipeline._metrics is not None) == record_metrics: + return cast( + "CompiledSQL", + cached_pipeline.compile( + sql, + parameters, + is_many, + record_metrics, + expression=expression, + param_fingerprint=param_fingerprint, + ), + ) + if not getattr(config, "_is_frozen", False): config.freeze() base_key = self._fingerprint_config(config) pipeline = self._pipelines.get(base_key) - record_metrics = _RECORD_PIPELINE_METRICS if pipeline is not None and _matches_pipeline_config(pipeline.config, config): self._pipelines.move_to_end(base_key) @@ -90,17 +123,25 @@ def compile( config, self._pipeline_cache_size, self._pipeline_parse_cache_size, self._cache_enabled, record_metrics ) if len(self._pipelines) >= self._max_pipelines: - self._pipelines.popitem(last=False) + _, evicted = self._pipelines.popitem(last=False) + evicted.config._cached_pipeline = None + evicted.config._cached_pipeline_epoch = 0 self._pipelines[base_key] = pipeline + config._cached_pipeline = pipeline + config._cached_pipeline_epoch = self._epoch + return pipeline.compile( sql, parameters, is_many, record_metrics, expression=expression, param_fingerprint=param_fingerprint ) def reset(self) -> None: for pipeline in self._pipelines.values(): + pipeline.config._cached_pipeline = None + pipeline.config._cached_pipeline_epoch = 0 pipeline.reset() self._pipelines.clear() + self._epoch = _allocate_pipeline_epoch() def configure_cache(self, cache_size: int, parse_cache_size: int, cache_enabled: bool) -> None: self._pipeline_cache_size = max(cache_size, 0) @@ -143,7 +184,9 @@ def _resolve_collision_pipeline( config, self._pipeline_cache_size, self._pipeline_parse_cache_size, self._cache_enabled, record_metrics ) if len(self._pipelines) >= self._max_pipelines: - self._pipelines.popitem(last=False) + _, evicted = self._pipelines.popitem(last=False) + evicted.config._cached_pipeline = None + evicted.config._cached_pipeline_epoch = 0 self._pipelines[slot_key] = pipeline return pipeline diff --git a/sqlspec/core/cache.py b/sqlspec/core/cache.py index 5df84741a..20365e9b0 100644 --- a/sqlspec/core/cache.py +++ b/sqlspec/core/cache.py @@ -23,7 +23,8 @@ get_statement_pipeline_metrics, reset_statement_pipeline_cache, ) -from sqlspec.core.parameters import ParameterProcessor, ParameterStyle +from sqlspec.core.parameters._processor import ParameterProcessor +from sqlspec.core.parameters._types import ParameterStyle from sqlspec.utils.logging import get_logger, log_with_context from sqlspec.utils.type_guards import has_field_name, has_filter_attributes @@ -33,7 +34,7 @@ import sqlglot.expressions as exp from sqlspec.core.compiler import OperationProfile, OperationType - from sqlspec.core.parameters import ParameterProfile + from sqlspec.core.parameters._types import ParameterProfile from sqlspec.core.statement import SQL, ProcessedState, StatementConfig @@ -77,6 +78,11 @@ ParameterStyle.NAMED_DOLLAR.value, ParameterStyle.NAMED_PYFORMAT.value, )) +_POSITIONAL_NUMBERED_STYLES: Final[frozenset[str]] = frozenset(( + ParameterStyle.NUMERIC.value, + ParameterStyle.POSITIONAL_COLON.value, +)) +_DEFAULT_PASSTHROUGH_TYPES: Final[tuple[type, ...]] = (str, int, float, bool, bytes, type(None)) CACHE_NODE_SLOTS: Final = ("key", "value", "prev", "next", "timestamp") @@ -191,18 +197,19 @@ class CacheNode: __slots__ = CACHE_NODE_SLOTS - def __init__(self, key: CacheKey, value: Any) -> None: + def __init__(self, key: CacheKey, value: Any, timestamp: float = 0.0) -> None: """Initialize cache node. Args: key: Cache key for this node value: Cached value + timestamp: Creation timestamp used when TTL eviction is configured """ self.key = key self.value = value self.prev: CacheNode | None = None self.next: CacheNode | None = None - self.timestamp = time.time() + self.timestamp = timestamp @mypyc_attr(allow_interpreted_subclasses=False) @@ -321,13 +328,14 @@ def put(self, key: CacheKey, value: Any) -> None: """ with self._lock: existing_node = self._cache.get(key) + now = time.time() if self._ttl is not None else 0.0 if existing_node is not None: existing_node.value = value - existing_node.timestamp = time.time() + existing_node.timestamp = now self._move_to_head(existing_node) return - new_node = CacheNode(key, value) + new_node = CacheNode(key, value, now) self._cache[key] = new_node self._add_to_head(new_node) @@ -470,7 +478,7 @@ def reset_stats_only() -> None: with _holders_lock: holders = list(_statement_cache_holders.values()) for holder in holders: - holder.query_cache.get_stats().reset() + holder.query_cache.reset_stats() def get_cache_statistics() -> "dict[str, CacheStats]": @@ -626,6 +634,7 @@ class CachedStatement: """ __slots__ = ( + "_source_guard", "applied_wrap_types", "compiled_sql", "expression", @@ -642,7 +651,9 @@ def __init__( input_named_parameters: "tuple[str, ...] | None" = None, parameter_profile: "ParameterProfile | None" = None, applied_wrap_types: bool = False, + _source_guard: "tuple[Any, ...] | None" = None, ) -> None: + self._source_guard = _source_guard self.applied_wrap_types = applied_wrap_types self.compiled_sql = compiled_sql self.expression = expression @@ -715,7 +726,7 @@ class NamespacedCache: predictable while avoiding stringly-typed cache access. """ - __slots__ = ("_caches", "_config") + __slots__ = ("_caches", "_config", "_enabled_flags") def __init__(self, config: "CacheConfig | None" = None, ttl_seconds: int | None = DEFAULT_TTL_SECONDS) -> None: """Initialize namespaced cache. @@ -724,8 +735,17 @@ def __init__(self, config: "CacheConfig | None" = None, ttl_seconds: int | None config: Cache configuration to apply. ttl_seconds: Time-to-live in seconds (None for no expiration). """ - self._config = config or get_cache_config() - self._caches = self._build_caches(self._config, ttl_seconds) + cfg = config or get_cache_config() + self._config = cfg + self._caches = self._build_caches(cfg, ttl_seconds) + compiled_ok = bool(cfg.compiled_cache_enabled) + self._enabled_flags: dict[str, bool] = { + "statement": compiled_ok and bool(cfg.sql_cache_enabled), + "builder": compiled_ok and bool(cfg.sql_cache_enabled), + "expression": compiled_ok and bool(cfg.fragment_cache_enabled), + "file": compiled_ok and bool(cfg.fragment_cache_enabled), + "optimized": compiled_ok and bool(cfg.optimized_cache_enabled), + } def get_statement(self, key: str, dialect: str | None = None) -> Any | None: """Get cached statement data. @@ -933,10 +953,7 @@ def _build_caches(config: "CacheConfig", ttl_seconds: int | None) -> "dict[str, return caches def _is_enabled(self, namespace: str) -> bool: - if not self._config.compiled_cache_enabled: - return False - enabled_getter = NAMESPACED_CACHE_CONFIG[namespace][0] - return bool(enabled_getter(self._config)) + return self._enabled_flags.get(namespace, False) def _get(self, namespace: str, key: str, dialect: str | None = None) -> Any | None: """Get cached value by namespace. @@ -1141,8 +1158,12 @@ class CachedQuery: param_count: int processed_state: "ProcessedState" needs_style_remap: bool + positional_passthrough: bool + simple_wire_fast_path: bool + _sql_hash: "str | None" __slots__ = ( + "_sql_hash", "applied_wrap_types", "compiled_sql", "input_named_parameters", @@ -1152,7 +1173,9 @@ class CachedQuery: "param_count", "parameter_casts", "parameter_profile", + "positional_passthrough", "processed_state", + "simple_wire_fast_path", ) def __init__( @@ -1167,6 +1190,8 @@ def __init__( param_count: int, processed_state: "ProcessedState", needs_style_remap: bool | None = None, + positional_passthrough: bool | None = None, + simple_wire_fast_path: bool = False, ) -> None: self.compiled_sql = compiled_sql self.parameter_profile = parameter_profile @@ -1177,11 +1202,27 @@ def __init__( self.operation_profile = operation_profile self.param_count = param_count self.processed_state = processed_state - self.needs_style_remap = ( + remap_needed = ( bool(_CACHED_NAMED_STYLES.intersection(parameter_profile.styles)) if needs_style_remap is None else needs_style_remap ) + self.needs_style_remap = remap_needed + if positional_passthrough is not None: + self.positional_passthrough = positional_passthrough + elif applied_wrap_types or remap_needed: + self.positional_passthrough = False + elif not input_named_parameters or ( + bool(parameter_profile.styles) + and _POSITIONAL_NUMBERED_STYLES.issuperset(parameter_profile.styles) + and len(input_named_parameters) == param_count + and input_named_parameters == tuple(str(i) for i in range(1, param_count + 1)) + ): + self.positional_passthrough = True + else: + self.positional_passthrough = False + self.simple_wire_fast_path = simple_wire_fast_path and self.positional_passthrough + self._sql_hash = None @final @@ -1189,13 +1230,15 @@ def __init__( class QueryCache: """LRU cache for compiled query metadata (L1 per-config fast-path cache).""" - __slots__ = ("_cache", "_lock", "_max_size", "_stats") + __slots__ = ("_cache", "_evictions", "_hits", "_lock", "_max_size", "_misses") def __init__(self, max_size: int = STMT_CACHE_MAX_SIZE) -> None: self._cache: OrderedDict[str, CachedQuery] = OrderedDict() self._lock = threading.Lock() self._max_size = max_size - self._stats = CacheStats() + self._hits = 0 + self._misses = 0 + self._evictions = 0 def __len__(self) -> int: with self._lock: @@ -1206,17 +1249,19 @@ def get(self, sql: str) -> "CachedQuery | None": Reads take no lock: the lookup and the recency update are each a single atomic ``OrderedDict`` operation, and an entry evicted between the two is still returned. + Below capacity, LRU recency promotion is skipped because eviction cannot trigger. """ cache = self._cache entry = cache.get(sql) if entry is None: - self._stats.record_miss() + self._misses += 1 return None - self._stats.record_hit() - try: - cache.move_to_end(sql) - except KeyError: - return entry + self._hits += 1 + if len(cache) >= self._max_size: + try: + cache.move_to_end(sql) + except KeyError: + return entry return entry def set(self, sql: str, entry: "CachedQuery") -> None: @@ -1227,18 +1272,52 @@ def set(self, sql: str, entry: "CachedQuery") -> None: self._cache.move_to_end(sql) elif len(self._cache) >= self._max_size: self._cache.popitem(last=False) - self._stats.record_eviction() + self._evictions += 1 self._cache[sql] = entry def get_stats(self) -> CacheStats: """Return cache hit/miss/eviction statistics for this L1 query cache.""" - return self._stats + stats = CacheStats() + stats.hits = self._hits + stats.misses = self._misses + stats.evictions = self._evictions + stats.total_operations = self._hits + self._misses + return stats + + def reset_stats(self) -> None: + """Reset hit, miss, and eviction counters without clearing cached entries.""" + self._hits = 0 + self._misses = 0 + self._evictions = 0 def clear(self) -> None: """Clear all cached entries and reset statistics.""" with self._lock: self._cache.clear() - self._stats.reset() + self.reset_stats() + + +def _seed_passthrough_types(type_coercion_map: "dict[type, Any] | None") -> "set[type]": + """Build the initial set of parameter types proven safe to pass without conversion.""" + if not type_coercion_map: + return set(_DEFAULT_PASSTHROUGH_TYPES) + seeded: set[type] = set() + map_keys = tuple(type_coercion_map.keys()) + for candidate in _DEFAULT_PASSTHROUGH_TYPES: + if candidate in type_coercion_map: + continue + overridden = False + for key in map_keys: + if isinstance(key, type): + try: + if issubclass(candidate, key): + overridden = True + break + except TypeError: + continue + if not overridden: + seeded.add(candidate) + return seeded @final @@ -1250,6 +1329,7 @@ class StatementCacheHolder: max_size: int query_cache: QueryCache rebind_processor: ParameterProcessor + passthrough_types: "set[type]" _statement_cache: "OrderedDict[str, SQL]" _row_count_hints: "OrderedDict[str, int]" _bulk_insert_sql: "OrderedDict[str, tuple[str, str]]" @@ -1261,6 +1341,7 @@ class StatementCacheHolder: "_row_count_hints", "_statement_cache", "max_size", + "passthrough_types", "query_cache", "rebind_processor", "statement_config", @@ -1274,8 +1355,8 @@ def __init__(self, statement_config: "StatementConfig", max_size: int = STMT_CAC converter=statement_config.parameter_converter, validator=statement_config.parameter_validator, cache_max_size=0, - validator_cache_max_size=0, ) + self.passthrough_types = _seed_passthrough_types(statement_config.parameter_config.type_coercion_map) self._statement_cache = OrderedDict() self._row_count_hints = OrderedDict() self._bulk_insert_sql = OrderedDict() @@ -1283,13 +1364,17 @@ def __init__(self, statement_config: "StatementConfig", max_size: int = STMT_CAC _register_statement_cache_holder(self) def get_statement(self, sql: str) -> "SQL | None": - """Retrieve a cached SQL object and mark it as recently used.""" - with self._lock: - entry = self._statement_cache.get(sql) - if entry is None: - return None - self._statement_cache.move_to_end(sql) - return entry + """Retrieve a cached SQL object without locking, eliding LRU promotion below capacity.""" + cache = self._statement_cache + entry = cache.get(sql) + if entry is None: + return None + if len(cache) >= self.max_size: + try: + cache.move_to_end(sql) + except KeyError: + return entry + return entry def set_statement(self, sql: str, stmt: "SQL", max_size: int | None = None) -> None: """Store a prepared SQL object in the bounded LRU cache.""" @@ -1305,13 +1390,17 @@ def set_statement(self, sql: str, stmt: "SQL", max_size: int | None = None) -> N self._statement_cache[sql] = stmt def get_row_count_hint(self, compiled_sql: str) -> int | None: - """Return the last observed row count for a compiled SQL string.""" - with self._lock: - entry = self._row_count_hints.get(compiled_sql) - if entry is None: - return None - self._row_count_hints.move_to_end(compiled_sql) - return entry + """Return the last observed row count for a compiled SQL string without locking.""" + cache = self._row_count_hints + entry = cache.get(compiled_sql) + if entry is None: + return None + if len(cache) >= self.max_size: + try: + cache.move_to_end(compiled_sql) + except KeyError: + return entry + return entry def set_row_count_hint(self, compiled_sql: str, row_count: int) -> None: """Record the observed row count for a compiled SQL string.""" @@ -1325,13 +1414,17 @@ def set_row_count_hint(self, compiled_sql: str, row_count: int) -> None: self._row_count_hints[compiled_sql] = row_count def get_bulk_insert_sql(self, compiled_sql: str) -> "tuple[str, str] | None": - """Return cached (table_sql, columns_sql) fragments for a bulk INSERT statement.""" - with self._lock: - entry = self._bulk_insert_sql.get(compiled_sql) - if entry is None: - return None - self._bulk_insert_sql.move_to_end(compiled_sql) - return entry + """Return cached (table_sql, columns_sql) fragments without locking.""" + cache = self._bulk_insert_sql + entry = cache.get(compiled_sql) + if entry is None: + return None + if len(cache) >= self.max_size: + try: + cache.move_to_end(compiled_sql) + except KeyError: + return entry + return entry def set_bulk_insert_sql(self, compiled_sql: str, entry: "tuple[str, str]") -> None: """Cache (table_sql, columns_sql) fragments for a bulk INSERT statement.""" diff --git a/sqlspec/core/compiler.py b/sqlspec/core/compiler.py index e8827533b..8921e2039 100644 --- a/sqlspec/core/compiler.py +++ b/sqlspec/core/compiler.py @@ -18,15 +18,16 @@ from sqlglot.errors import ParseError import sqlspec.exceptions -from sqlspec.core.parameters import ( +from sqlspec.core.parameters._alignment import validate_parameter_alignment +from sqlspec.core.parameters._processor import ( ParameterProcessor, - ParameterProfile, + _make_cache_key_tuple, structural_fingerprint, - validate_parameter_alignment, value_fingerprint, ) -from sqlspec.core.parameters._processor import _make_cache_key_tuple +from sqlspec.core.parameters._types import ParameterProfile from sqlspec.core.sqlcommenter import _append_comment, _comment_attributes +from sqlspec.exceptions import with_cause from sqlspec.utils.logging import get_logger, log_with_context from sqlspec.utils.type_guards import get_value_attribute @@ -1173,7 +1174,7 @@ def _compile_statement( raise except RecursionError as exc: msg = "Statement nesting exceeds the supported depth" - raise sqlspec.exceptions.SQLParsingError(msg) from exc + raise with_cause(sqlspec.exceptions.SQLParsingError(msg), exc) from exc except ParseError as exc: log_with_context(logger, logging.DEBUG, "sql.compile", error_type=type(exc).__name__, status="fallback") fallback_profile = OperationProfile(returns_rows=SQLProcessor._statement_text_returns_rows(sql)) diff --git a/sqlspec/core/config_runtime.py b/sqlspec/core/config_runtime.py index f8098282c..65829febd 100644 --- a/sqlspec/core/config_runtime.py +++ b/sqlspec/core/config_runtime.py @@ -5,13 +5,14 @@ import threading from typing import TYPE_CHECKING, Any, TypeVar -from sqlspec.core.parameters import ParameterStyle, ParameterStyleConfig +from sqlspec.core.cache import StatementCacheHolder, resolve_statement_cache_size +from sqlspec.core.parameters._types import ParameterStyle, ParameterStyleConfig from sqlspec.core.statement import StatementConfig if TYPE_CHECKING: from collections.abc import Awaitable, Callable - from sqlspec.storage import StorageCapabilities + from sqlspec.storage.pipeline import StorageCapabilities __all__ = ( @@ -93,6 +94,7 @@ def resolve_postgres_extension_state( if driver_features is not None: driver_features["active_extensions"] = active_extensions + original_statement_config = statement_config if statement_config.dialect == "postgres": if paradedb_available: statement_config = statement_config.replace(dialect="paradedb") @@ -101,6 +103,13 @@ def resolve_postgres_extension_state( elif pgvector_available: statement_config = statement_config.replace(dialect="pgvector") + if statement_config is not original_statement_config and driver_features is not None: + existing_holder = driver_features.get("_statement_cache_holder") + if isinstance(existing_holder, StatementCacheHolder): + driver_features["_statement_cache_holder"] = StatementCacheHolder( + statement_config, resolve_statement_cache_size(driver_features) + ) + return statement_config, pgvector_available, paradedb_available diff --git a/sqlspec/core/filters.py b/sqlspec/core/filters.py index 6cd359af2..fb75c64b8 100644 --- a/sqlspec/core/filters.py +++ b/sqlspec/core/filters.py @@ -38,7 +38,7 @@ sanitize_parameter_name, wrap_as_subquery, ) -from sqlspec.exceptions import ImproperConfigurationError, InvalidCursorError +from sqlspec.exceptions import ImproperConfigurationError, InvalidCursorError, with_cause from sqlspec.utils.type_guards import has_field_name from sqlspec.utils.uuids import uuid4 @@ -931,7 +931,7 @@ def _row_values(self, row: abc.Mapping[str, Any]) -> list[Any]: value = row[key.result_name] except KeyError as exc: msg = f"Cursor key column '{key.result_name}' is not present in the result row; select it or set CursorKey.result_name" - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc if value is None and key.nulls is None: msg = f"Cursor key '{key.field_name}' returned NULL; declare nulls='first' or nulls='last'" raise ImproperConfigurationError(msg) diff --git a/sqlspec/core/hashing.py b/sqlspec/core/hashing.py index 60c9059c5..91aafb351 100644 --- a/sqlspec/core/hashing.py +++ b/sqlspec/core/hashing.py @@ -11,7 +11,7 @@ from sqlglot import exp from sqlspec.core._ordering import has_default_nulls, has_explicit_nulls -from sqlspec.core.parameters import TypedParameter +from sqlspec.core.parameters._types import TypedParameter from sqlspec.utils.type_guards import is_expression if TYPE_CHECKING: diff --git a/sqlspec/core/metrics.py b/sqlspec/core/metrics.py index 1c85c9c4d..d8874bf7b 100644 --- a/sqlspec/core/metrics.py +++ b/sqlspec/core/metrics.py @@ -5,7 +5,7 @@ from mypy_extensions import mypyc_attr if TYPE_CHECKING: # pragma: no cover - from sqlspec.observability import ObservabilityRuntime + from sqlspec.observability._runtime import ObservabilityRuntime __all__ = ("StackExecutionMetrics",) diff --git a/sqlspec/core/parameters/_converter.py b/sqlspec/core/parameters/_converter.py index 41a133d66..92f394f9f 100644 --- a/sqlspec/core/parameters/_converter.py +++ b/sqlspec/core/parameters/_converter.py @@ -23,7 +23,7 @@ PositionalParameterOutput, ) from sqlspec.core.parameters._validator import ParameterValidator -from sqlspec.exceptions import SQLSpecError +from sqlspec.exceptions import SQLSpecError, with_cause __all__ = ("ParameterConverter",) @@ -659,7 +659,7 @@ def _format_literal(param_value: object | None, dialect: "str | None" = None) -> rendered = exp.Literal.string(text).sql(dialect=literal_dialect, copy=False) except ValueError as exc: msg = f"Cannot embed a string value in statically compiled SQL for dialect '{dialect}'" - raise SQLSpecError(msg) from exc + raise with_cause(SQLSpecError(msg), exc) from exc if "\x00" in rendered: msg = "Cannot embed a string containing NUL characters in statically compiled SQL" raise SQLSpecError(msg) diff --git a/sqlspec/core/parameters/_declared.py b/sqlspec/core/parameters/_declared.py index 06ee87d9e..5f31188bf 100644 --- a/sqlspec/core/parameters/_declared.py +++ b/sqlspec/core/parameters/_declared.py @@ -12,7 +12,7 @@ from typing import Final, TypeAlias from uuid import UUID -from sqlspec.utils.serializers import to_json +from sqlspec.utils.serializers._json import to_json __all__ = ( "ParamTypeMatcher", diff --git a/sqlspec/core/parameters/_registry.py b/sqlspec/core/parameters/_registry.py index 3df277e59..76e8fbeb5 100644 --- a/sqlspec/core/parameters/_registry.py +++ b/sqlspec/core/parameters/_registry.py @@ -5,7 +5,8 @@ import sqlspec.exceptions from sqlspec.core.parameters._types import DriverParameterProfile, ParameterStyleConfig -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.exceptions import with_cause +from sqlspec.utils.serializers._json import from_json, to_json if TYPE_CHECKING: from collections.abc import Mapping @@ -42,7 +43,7 @@ def get_driver_profile(adapter_key: str) -> "DriverParameterProfile": return DRIVER_PARAMETER_PROFILES[key] except KeyError as exc: msg = f"No driver parameter profile registered for adapter '{adapter_key}'." - raise sqlspec.exceptions.ImproperConfigurationError(msg) from exc + raise with_cause(sqlspec.exceptions.ImproperConfigurationError(msg), exc) from exc def register_driver_profile( diff --git a/sqlspec/core/parameters/_validator.py b/sqlspec/core/parameters/_validator.py index b99f2988c..0df708481 100644 --- a/sqlspec/core/parameters/_validator.py +++ b/sqlspec/core/parameters/_validator.py @@ -3,6 +3,7 @@ import re from bisect import bisect_left from collections import OrderedDict +from contextlib import suppress from typing import TYPE_CHECKING, Final from mypy_extensions import mypyc_attr @@ -349,14 +350,18 @@ def extract_parameters(self, sql: str, dialect: str | None = None) -> "list[Para cache_key = (sql, normalized_dialect) cached_result = self._parameter_cache.get(cache_key) if cached_result is not None: - self._parameter_cache.move_to_end(cache_key) self._cache_hits += 1 + try: + self._parameter_cache.move_to_end(cache_key) + except KeyError: + return cached_result return cached_result self._cache_misses += 1 parameters = self._extract_parameters_uncached(sql, normalized_dialect) if len(self._parameter_cache) >= self._cache_max_size: - self._parameter_cache.popitem(last=False) + with suppress(KeyError): + self._parameter_cache.popitem(last=False) self._parameter_cache[cache_key] = parameters return parameters diff --git a/sqlspec/core/query_modifiers.py b/sqlspec/core/query_modifiers.py index 6210217fe..2b2a1d96a 100644 --- a/sqlspec/core/query_modifiers.py +++ b/sqlspec/core/query_modifiers.py @@ -53,6 +53,12 @@ ) ConditionFactory = Callable[[exp.Expr, exp.Expr], exp.Expr] +_COLUMN_SEGMENT_RE: Final["re.Pattern[str]"] = re.compile(r"\A[\w$-]+\Z", re.UNICODE) + + +def _is_valid_column_segment(segment: str) -> bool: + return "--" not in segment and _COLUMN_SEGMENT_RE.fullmatch(segment) is not None + _TABLE_QUALIFIED_PARTS: Final[int] = 2 _DATABASE_QUALIFIED_PARTS: Final[int] = 3 @@ -179,23 +185,24 @@ def parse_column_for_condition(column: str | exp.Column | exp.Expr) -> exp.Expr: if isinstance(column, exp.Expr): return column - if isinstance(column, str): - if not column.strip() or "\x00" in column: - msg = "Column name must be nonempty and must not contain NUL characters" + col_name = column if isinstance(column, str) else str(column) + if not col_name.strip() or "\x00" in col_name: + msg = "Column name must be nonempty and must not contain NUL characters" + raise SQLSpecError(msg) + if "." in col_name: + parts = col_name.split(".") + if len(parts) > _CATALOG_QUALIFIED_PARTS or not all(_is_valid_column_segment(part) for part in parts): + msg = f"Invalid qualified column name: {col_name!r}" raise SQLSpecError(msg) - if "." in column: - parts = column.split(".") - if len(parts) > _CATALOG_QUALIFIED_PARTS or not all(part.strip() for part in parts): - msg = f"Invalid qualified column name: {column!r}" - raise SQLSpecError(msg) - if len(parts) == _TABLE_QUALIFIED_PARTS: - return exp.column(parts[1], table=parts[0], copy=False) - if len(parts) == _DATABASE_QUALIFIED_PARTS: - return exp.column(parts[2], table=parts[1], db=parts[0], copy=False) - return exp.column(parts[3], table=parts[2], db=parts[1], catalog=parts[0], copy=False) - return exp.column(column, copy=False) - - return exp.column(str(column), copy=False) + if len(parts) == _TABLE_QUALIFIED_PARTS: + return exp.column(parts[1], table=parts[0], copy=False) + if len(parts) == _DATABASE_QUALIFIED_PARTS: + return exp.column(parts[2], table=parts[1], db=parts[0], copy=False) + return exp.column(parts[3], table=parts[2], db=parts[1], catalog=parts[0], copy=False) + if not _is_valid_column_segment(col_name): + msg = f"Invalid column name: {col_name!r}" + raise SQLSpecError(msg) + return exp.column(col_name, copy=False) def extract_column_name(column: str | exp.Column | exp.Expr) -> str: diff --git a/sqlspec/core/result/_base.py b/sqlspec/core/result/_base.py index 92700d61d..c8ed46ed9 100644 --- a/sqlspec/core/result/_base.py +++ b/sqlspec/core/result/_base.py @@ -13,21 +13,22 @@ import contextlib from abc import abstractmethod from collections.abc import Iterable, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload +from typing import TYPE_CHECKING, Any, Final, Literal, cast, final, overload from mypy_extensions import mypyc_attr from typing_extensions import TypeVar from sqlspec.core.statement import SQL, ProcessedState -from sqlspec.exceptions import MultipleResultsFoundError -from sqlspec.storage import ( +from sqlspec.exceptions import MultipleResultsFoundError, with_cause +from sqlspec.storage.pipeline import ( AsyncStoragePipeline, StorageDestination, StorageFormat, StorageTelemetry, SyncStoragePipeline, + close_async_arrow_result_resources, + close_sync_arrow_source, ) -from sqlspec.storage.pipeline import close_async_arrow_result_resources, close_sync_arrow_source from sqlspec.utils.arrow_helpers import ( arrow_reader_to_return_format, arrow_table_column_names, @@ -55,7 +56,15 @@ from sqlspec.typing import ArrowReturnFormat, ArrowTable, PandasDataFrame, PolarsDataFrame, SchemaT -__all__ = ("ArrowResult", "DMLResult", "EmptyResult", "SQLResult", "StackResult", "StatementResult") +__all__ = ( + "ArrowResult", + "CachedSelectResult", + "DMLResult", + "EmptyResult", + "SQLResult", + "StackResult", + "StatementResult", +) T = TypeVar("T") RowFormat = Literal["dict", "tuple", "record"] @@ -290,18 +299,18 @@ class SQLResult(StatementResult): """ __slots__ = ( + "_column_names", + "_errors", + "_inserted_ids", "_materialized_dicts", "_row_format", "_schema_row_cache", "_schema_rows_cache", - "column_names", + "_statement_results", "error", - "errors", "has_more", - "inserted_ids", "operation_index", "parameters", - "statement_results", "successful_statements", "total_count", "total_statements", @@ -371,22 +380,74 @@ def __init__( self._schema_rows_cache: dict[type[Any], list[Any]] | None = None self._schema_row_cache: dict[type[Any], Any] | None = None - self.column_names = column_names or [] + self._column_names: list[str] | None = column_names self.total_count = total_count self.has_more = has_more - self.inserted_ids = inserted_ids or [] - self.statement_results = statement_results or [] - self.errors = errors or [] + self._inserted_ids: list[int | str] | None = inserted_ids + self._statement_results: list[SQLResult] | None = statement_results + self._errors: list[str] | None = errors self.total_statements = total_statements self.successful_statements = successful_statements - if not self.column_names and data and len(data) > 0: + if not self._column_names and data and len(data) > 0: first_row = data[0] if isinstance(first_row, dict): - self.column_names = list(first_row.keys()) + self._column_names = list(first_row.keys()) if self.total_count is None: self.total_count = len(data) if data is not None else 0 + @property + def column_names(self) -> "list[str]": + """Column names for the result set, initialized lazily when empty.""" + val = self._column_names + if val is None: + val = [] + self._column_names = val + return val + + @column_names.setter + def column_names(self, value: "list[str]") -> None: + self._column_names = value + + @property + def inserted_ids(self) -> "list[int | str]": + """Inserted row identifiers, allocated lazily.""" + val = self._inserted_ids + if val is None: + val = [] + self._inserted_ids = val + return val + + @inserted_ids.setter + def inserted_ids(self, value: "list[int | str]") -> None: + self._inserted_ids = value + + @property + def statement_results(self) -> "list[SQLResult]": + """Individual statement results for scripts, allocated lazily.""" + val = self._statement_results + if val is None: + val = [] + self._statement_results = val + return val + + @statement_results.setter + def statement_results(self, value: "list[SQLResult]") -> None: + self._statement_results = value + + @property + def errors(self) -> "list[str]": + """Script error messages, allocated lazily.""" + val = self._errors + if val is None: + val = [] + self._errors = val + return val + + @errors.setter + def errors(self, value: "list[str]") -> None: + self._errors = value + def __len__(self) -> int: """Get the number of rows in the result set. @@ -443,7 +504,7 @@ def num_columns(self) -> int: Returns: Number of columns. """ - return len(self.column_names) if self.column_names else 0 + return len(self._column_names) if self._column_names else 0 @classmethod def create_cached_select( @@ -460,23 +521,18 @@ def create_cached_select( last_inserted_id: int | str | None = None, ) -> "SQLResult": """Construct a fast-path SELECT result with deferred SQL statement materialization.""" - result = SQLResult( - statement=None, + return CachedSelectResult( + sql=sql, + config=config, + cached_state=cached_state, + compiled_sql=compiled_sql, + params=params, data=data, - rows_affected=rows_affected, - last_inserted_id=last_inserted_id, - metadata=_DEFAULT_DML_METADATA, - operation_type="SELECT", column_names=column_names, - total_count=rows_affected, + rows_affected=rows_affected, row_format=row_format, + last_inserted_id=last_inserted_id, ) - result._lazy_sql = sql - result._lazy_config = config - result._lazy_cached_state = cached_state - result._lazy_compiled_sql = compiled_sql - result._lazy_params = params - return result def is_success(self) -> bool: """Check if the operation was successful. @@ -486,17 +542,17 @@ def is_success(self) -> bool: """ op_type = self.operation_type - if op_type == "SCRIPT" or self.statement_results: - return not self.errors and self.total_statements == self.successful_statements + if op_type == "SCRIPT" or self._statement_results: + return not self._errors and self.total_statements == self.successful_statements if op_type == "SELECT": return self.data is not None and self.rows_affected >= 0 if op_type in {"INSERT", "UPDATE", "DELETE", "MERGE", "UPSERT", "EXECUTE"}: - return not self.errors and self.rows_affected >= 0 + return not self._errors and self.rows_affected >= 0 if op_type in {"DDL", "PRAGMA", "COPY", "COPY_FROM", "COPY_TO", "COMMAND"}: - return not self.errors and self.rows_affected >= -1 + return not self._errors and self.rows_affected >= -1 return False @@ -556,9 +612,9 @@ def get_total_rows_affected(self) -> int: Returns: Total rows affected. """ - if self.statement_results: + if self._statement_results: total = 0 - for stmt in self.statement_results: + for stmt in self._statement_results: if stmt.rows_affected and stmt.rows_affected > 0: total += stmt.rows_affected return total @@ -578,10 +634,18 @@ def get_first(self, *, schema_type: "type[SchemaT] | None" = None) -> "SchemaT | Returns: First row (optionally transformed to schema_type) or None if no data. """ - rows = self._get_rows() - if not rows: - return None - row = rows[0] + if self._materialized_dicts is None and not (self._row_format == "tuple" and not self._column_names): + raw = self.data + if not raw or not isinstance(raw, list): + return None + if __debug__: + self._assert_row_format_fidelity(raw) + row = _row_to_dict(raw[0], self._row_format, self._column_names) + else: + rows = self._get_rows() + if not rows: + return None + row = rows[0] if schema_type: return self._get_schema_row(schema_type, row) return row @@ -673,17 +737,30 @@ def one(self, *, schema_type: "type[SchemaT] | None" = None) -> "SchemaT | dict[ Raises: ValueError: If no results or more than one result """ - rows = self._get_rows() - if not rows: - msg = "No result found, exactly one row expected" - raise ValueError(msg) + if self._materialized_dicts is None and not (self._row_format == "tuple" and not self._column_names): + raw = self.data + if not raw or not isinstance(raw, list): + msg = "No result found, exactly one row expected" + raise ValueError(msg) + data_len = len(raw) + if data_len > 1: + msg = f"Multiple results found ({data_len}), exactly one row expected" + raise ValueError(msg) + if __debug__: + self._assert_row_format_fidelity(raw) + row = _row_to_dict(raw[0], self._row_format, self._column_names) + else: + rows = self._get_rows() + if not rows: + msg = "No result found, exactly one row expected" + raise ValueError(msg) - data_len = len(rows) - if data_len > 1: - msg = f"Multiple results found ({data_len}), exactly one row expected" - raise ValueError(msg) + data_len = len(rows) + if data_len > 1: + msg = f"Multiple results found ({data_len}), exactly one row expected" + raise ValueError(msg) - row = rows[0] + row = rows[0] if schema_type: return self._get_schema_row(schema_type, row) return row @@ -705,16 +782,28 @@ def one_or_none(self, *, schema_type: "type[SchemaT] | None" = None) -> "SchemaT Raises: MultipleResultsFoundError: If more than one result. """ - rows = self._get_rows() - if not rows: - return None + if self._materialized_dicts is None and not (self._row_format == "tuple" and not self._column_names): + raw = self.data + if not raw or not isinstance(raw, list): + return None + data_len = len(raw) + if data_len > 1: + msg = f"Multiple results found ({data_len}), at most one row expected" + raise MultipleResultsFoundError(msg) + if __debug__: + self._assert_row_format_fidelity(raw) + row = _row_to_dict(raw[0], self._row_format, self._column_names) + else: + rows = self._get_rows() + if not rows: + return None - data_len = len(rows) - if data_len > 1: - msg = f"Multiple results found ({data_len}), at most one row expected" - raise MultipleResultsFoundError(msg) + data_len = len(rows) + if data_len > 1: + msg = f"Multiple results found ({data_len}), at most one row expected" + raise MultipleResultsFoundError(msg) - row = rows[0] + row = rows[0] if schema_type: return self._get_schema_row(schema_type, row) return row @@ -725,6 +814,21 @@ def scalar(self) -> Any: Returns: The scalar value from first column of first row """ + if self._materialized_dicts is None and not (self._row_format == "tuple" and not self._column_names): + raw = self.data + if not raw or not isinstance(raw, list): + msg = "No result found, exactly one row expected" + raise ValueError(msg) + data_len = len(raw) + if data_len > 1: + msg = f"Multiple results found ({data_len}), exactly one row expected" + raise ValueError(msg) + if __debug__: + self._assert_row_format_fidelity(raw) + first_row = raw[0] + if self._row_format == "dict": + return next(iter(cast("dict[str, Any]", first_row).values())) + return first_row[0] row = self.one() return next(iter(row.values())) @@ -734,6 +838,20 @@ def scalar_or_none(self) -> Any: Returns: The scalar value from first column of first row, or None """ + if self._materialized_dicts is None and not (self._row_format == "tuple" and not self._column_names): + raw = self.data + if not raw or not isinstance(raw, list): + return None + data_len = len(raw) + if data_len > 1: + msg = f"Multiple results found ({data_len}), at most one row expected" + raise MultipleResultsFoundError(msg) + if __debug__: + self._assert_row_format_fidelity(raw) + first_row = raw[0] + if self._row_format == "dict": + return next(iter(cast("dict[str, Any]", first_row).values())) + return first_row[0] row = self.one_or_none() if row is None: return None @@ -842,7 +960,7 @@ def _get_rows(self) -> "list[dict[str, Any]]": if fmt == "dict": self._materialized_dicts = raw elif fmt == "tuple": - col_names = self.column_names + col_names = self._column_names if not col_names: self._materialized_dicts = [] elif len(col_names) == 1: @@ -875,7 +993,7 @@ def _assert_row_format_fidelity(self, rows: "list[Any]") -> None: keys = first_row.keys except AttributeError as exc: msg = "row_format='record' requires dict-like rows" - raise AssertionError(msg) from exc + raise with_cause(AssertionError(msg), exc) from exc assert callable(keys), "row_format='record' requires dict-like rows" def _get_schema_rows( @@ -1269,6 +1387,76 @@ def get_data(self) -> "list[Any]": return [] +def _row_to_dict(row: Any, fmt: RowFormat, col_names: "list[str] | None") -> "dict[str, Any]": + """Materialize a single raw row into a dict without allocating a row list.""" + if fmt == "dict": + return cast("dict[str, Any]", row) + if fmt == "tuple": + if not col_names: + return {} + if len(col_names) == 1: + return {col_names[0]: row[0]} + if len(col_names) == _TWO_COLUMN_THRESHOLD: + return {col_names[0]: row[0], col_names[1]: row[1]} + return dict(zip(col_names, row, strict=False)) + return dict(row) + + +@final +@mypyc_attr(allow_interpreted_subclasses=False) +class CachedSelectResult(SQLResult): + """Optimized flat-slot SELECT result constructed by cached query fast paths.""" + + __slots__ = () + + def __init__( + self, + sql: str, + config: "StatementConfig", + cached_state: "ProcessedState", + compiled_sql: str, + params: Any, + data: "list[Any]", + column_names: "list[str]", + rows_affected: int, + row_format: RowFormat = "dict", + last_inserted_id: int | str | None = None, + ) -> None: + self._statement = None + self._lazy_sql = sql + self._lazy_config = config + self._lazy_cached_state = cached_state + self._lazy_compiled_sql = compiled_sql + self._lazy_params = params + self._operation_type = "SELECT" + self.data = data + self.rows_affected = rows_affected + self.last_inserted_id = last_inserted_id + self.execution_time = None + self.metadata = _DEFAULT_DML_METADATA + + self.error = None + self.operation_index = None + self.parameters = None + self._row_format = row_format + self._materialized_dicts = None + self._schema_rows_cache = None + self._schema_row_cache = None + + self._column_names = column_names + if not column_names and data: + first_row = data[0] + if isinstance(first_row, dict): + self._column_names = list(first_row.keys()) + self.total_count = rows_affected + self.has_more = False + self._inserted_ids = None + self._statement_results = None + self._errors = None + self.total_statements = 0 + self.successful_statements = 0 + + @mypyc_attr(allow_interpreted_subclasses=False) class DMLResult(SQLResult): """Optimized result for simple DML operations (INSERT/UPDATE/DELETE). @@ -1303,12 +1491,12 @@ def __init__( self._schema_rows_cache = None self._schema_row_cache = None - self.column_names: list[str] = [] + self._column_names = None self.total_count = 0 self.has_more = False - self.inserted_ids: list[int | str] = [] - self.statement_results: list[SQLResult] = [] - self.errors: list[str] = [] + self._inserted_ids = None + self._statement_results = None + self._errors = None self.total_statements = 0 self.successful_statements = 0 diff --git a/sqlspec/core/stack.py b/sqlspec/core/stack.py index 99fe62b84..8df95fbdc 100644 --- a/sqlspec/core/stack.py +++ b/sqlspec/core/stack.py @@ -7,7 +7,7 @@ from mypy_extensions import mypyc_attr if TYPE_CHECKING: # pragma: no cover - from sqlspec.builder import QueryBuilder + from sqlspec.builder._base import QueryBuilder from sqlspec.core.filters import StatementFilter from sqlspec.core.statement import Statement, StatementConfig from sqlspec.typing import StatementParameters diff --git a/sqlspec/core/statement.py b/sqlspec/core/statement.py index e7cb5fe84..6c7aafeb3 100644 --- a/sqlspec/core/statement.py +++ b/sqlspec/core/statement.py @@ -3,7 +3,8 @@ import hashlib import re from collections.abc import Mapping, Sequence -from types import BuiltinFunctionType, FunctionType +from functools import partial +from types import BuiltinFunctionType, FunctionType, MethodType from typing import TYPE_CHECKING, Any, Final, TypeAlias import sqlglot @@ -20,17 +21,10 @@ from sqlspec.core.compiler import OperationProfile, OperationType from sqlspec.core.explain import ExplainFormat, ExplainOptions from sqlspec.core.hashing import hash_filters -from sqlspec.core.parameters import ( - ParameterConverter, - ParameterDeclaration, - ParameterProcessor, - ParameterProfile, - ParameterStyle, - ParameterStyleConfig, - ParameterValidator, - structural_fingerprint, - unescape_qmark_operator, -) +from sqlspec.core.parameters._converter import ParameterConverter +from sqlspec.core.parameters._processor import ParameterProcessor, structural_fingerprint +from sqlspec.core.parameters._types import ParameterProfile, ParameterStyle, ParameterStyleConfig +from sqlspec.core.parameters._validator import ParameterValidator, unescape_qmark_operator from sqlspec.core.query_modifiers import ( ConditionFactory, apply_column_pruning, @@ -59,6 +53,7 @@ ) from sqlspec.core.splitter import contains_statement_terminator from sqlspec.core.sqlcommenter import create_sqlcommenter_statement_transformer, format_wire_sqlcomment +from sqlspec.exceptions import with_cause from sqlspec.observability._common import resolve_db_system from sqlspec.typing import Empty, EmptyEnum from sqlspec.utils.logging import get_logger @@ -70,8 +65,9 @@ from sqlglot.dialects.dialect import DialectType - from sqlspec.builder import QueryBuilder + from sqlspec.builder._base import QueryBuilder from sqlspec.core.filters import StatementFilter + from sqlspec.core.parameters._declared import ParameterDeclaration __all__ = ( @@ -687,7 +683,7 @@ def compile(self) -> "tuple[str, Any]": raise except RecursionError as e: msg = "Statement nesting exceeds the supported depth" - raise sqlspec.exceptions.SQLParsingError(msg) from e + raise with_cause(sqlspec.exceptions.SQLParsingError(msg), e) from e except Exception as e: new_state = self._handle_compile_failure(e) @@ -1098,7 +1094,7 @@ def explain(self, analyze: bool = False, verbose: bool = False, format: "str | N Returns: New SQL instance containing the EXPLAIN statement """ - from sqlspec.builder import Explain + from sqlspec.builder._explain import Explain fmt = None if format is not None: @@ -1132,7 +1128,12 @@ def builder(self, dialect: "DialectType | None" = None) -> "QueryBuilder": msg = "QueryBuilder does not support execute_many SQL statements." raise sqlspec.exceptions.SQLBuilderError(msg) - from sqlspec.builder import Delete, ExpressionBuilder, Insert, Merge, Select, Update + from sqlspec.builder._base import ExpressionBuilder + from sqlspec.builder._delete import Delete + from sqlspec.builder._insert import Insert + from sqlspec.builder._merge import Merge + from sqlspec.builder._select import Select + from sqlspec.builder._update import Update builder_dialect = dialect or self._dialect lexical_dialect = self._normalize_dialect(builder_dialect) @@ -1164,7 +1165,7 @@ def builder(self, dialect: "DialectType | None" = None) -> "QueryBuilder": expression = sqlglot.parse_one(converted_sql, dialect=builder_dialect) except ParseError as exc: msg = f"Failed to parse SQL for builder: {exc}" - raise sqlspec.exceptions.SQLBuilderError(msg) from exc + raise with_cause(sqlspec.exceptions.SQLBuilderError(msg), exc) from exc base_expression = expression ctes: list[exp.CTE] | None = None @@ -1684,7 +1685,7 @@ def _wrapped_raw_sql_expression(self, raw_sql: str) -> exp.Expr: ) except ParseError as exc: msg = f"Failed to parse SQL statement: {exc}" - raise sqlspec.exceptions.SQLParsingError(msg) from exc + raise with_cause(sqlspec.exceptions.SQLParsingError(msg), exc) from exc def _filter_expression(self) -> exp.Expr: """Return a mutable expression copy for statement filters. @@ -1779,6 +1780,8 @@ class StatementConfig: __slots__ = ( *SQL_CONFIG_SLOTS, + "_cached_pipeline", + "_cached_pipeline_epoch", "_dialect_variants", "_has_output_transformer", "_has_transformers", @@ -1886,6 +1889,8 @@ def __init__( self._fingerprint_cache: str | None = None self._hash_cache: int | None = None self._is_frozen = False + self._cached_pipeline: Any | None = None + self._cached_pipeline_epoch: int = 0 self._dialect_variants: dict[str, StatementConfig] | None = None self._script_variant: StatementConfig | None = None self._has_transformers = bool(self._user_statement_transformers) @@ -2208,15 +2213,34 @@ def _raw_condition(condition: str, dialect: "str | None" = None) -> exp.Expr: def _is_uncacheable_callable(func: Any) -> bool: - if func is None: - return False - if isinstance(func, (FunctionType, BuiltinFunctionType)): - qualname = getattr(func, "__qualname__", "") or "" - return "" in qualname or getattr(func, "__closure__", None) is not None - func_type = type(func) - return ( - getattr(func_type, "__hash__", None) is object.__hash__ or getattr(func_type, "__eq__", None) is object.__eq__ - ) + pending: list[Any] = [func] + seen_ids: set[int] = set() + while pending: + current = pending.pop() + if current is None: + continue + obj_id = id(current) + if obj_id in seen_ids: + continue + seen_ids.add(obj_id) + if isinstance(current, (FunctionType, BuiltinFunctionType)): + qualname = getattr(current, "__qualname__", "") or "" + if "" in qualname or getattr(current, "__closure__", None) is not None: + return True + continue + if isinstance(current, MethodType): + pending.append(current.__self__) + pending.append(current.__func__) + continue + if isinstance(current, partial): + return True + current_type = type(current) + if ( + getattr(current_type, "__hash__", None) is object.__hash__ + or getattr(current_type, "__eq__", None) is object.__eq__ + ): + return True + return False def _config_has_uncacheable_transformers( diff --git a/sqlspec/core/tenant/base.py b/sqlspec/core/tenant/base.py index 7d9a0f57f..5dc8b65b4 100644 --- a/sqlspec/core/tenant/base.py +++ b/sqlspec/core/tenant/base.py @@ -8,7 +8,7 @@ from sqlspec.exceptions import ImproperConfigurationError, SQLSpecError if TYPE_CHECKING: - from sqlspec.utils.tenant import TenantContextPayload + from sqlspec.utils.tenant._context import TenantContextPayload __all__ = ( "AsyncTenantStrategy", diff --git a/sqlspec/data_dictionary/dialects/bigquery/config.py b/sqlspec/data_dictionary/dialects/bigquery/config.py index ec04989e5..81ca82394 100644 --- a/sqlspec/data_dictionary/dialects/bigquery/config.py +++ b/sqlspec/data_dictionary/dialects/bigquery/config.py @@ -2,7 +2,8 @@ import re -from sqlspec.data_dictionary import DialectConfig, FeatureFlags, FeatureVersions, register_dialect +from sqlspec.data_dictionary._registry import register_dialect +from sqlspec.data_dictionary._types import DialectConfig, FeatureFlags, FeatureVersions __all__ = ( "BIGQUERY_CONFIG", diff --git a/sqlspec/data_dictionary/dialects/cockroachdb/config.py b/sqlspec/data_dictionary/dialects/cockroachdb/config.py index 3cd98c466..fa79e1455 100644 --- a/sqlspec/data_dictionary/dialects/cockroachdb/config.py +++ b/sqlspec/data_dictionary/dialects/cockroachdb/config.py @@ -2,7 +2,8 @@ import re -from sqlspec.data_dictionary import DialectConfig, FeatureFlags, FeatureVersions, VersionInfo, register_dialect +from sqlspec.data_dictionary._registry import register_dialect +from sqlspec.data_dictionary._types import DialectConfig, FeatureFlags, FeatureVersions, VersionInfo __all__ = ( "COCKROACHDB_CONFIG", diff --git a/sqlspec/data_dictionary/dialects/db2/config.py b/sqlspec/data_dictionary/dialects/db2/config.py index 67a9a523c..d7896e8ac 100644 --- a/sqlspec/data_dictionary/dialects/db2/config.py +++ b/sqlspec/data_dictionary/dialects/db2/config.py @@ -4,7 +4,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final -from sqlspec.data_dictionary import register_dialect +from sqlspec.data_dictionary._registry import register_dialect from sqlspec.data_dictionary._types import ( DialectConfig, FeatureFlags, @@ -18,7 +18,7 @@ ) if TYPE_CHECKING: - from sqlspec.data_dictionary import TableMetadata + from sqlspec.data_dictionary._types import TableMetadata __all__ = ( "DB2_CONFIG", diff --git a/sqlspec/data_dictionary/dialects/duckdb/config.py b/sqlspec/data_dictionary/dialects/duckdb/config.py index cd6aafe92..43b1fc87d 100644 --- a/sqlspec/data_dictionary/dialects/duckdb/config.py +++ b/sqlspec/data_dictionary/dialects/duckdb/config.py @@ -2,7 +2,8 @@ import re -from sqlspec.data_dictionary import DialectConfig, FeatureFlags, FeatureVersions, VersionInfo, register_dialect +from sqlspec.data_dictionary._registry import register_dialect +from sqlspec.data_dictionary._types import DialectConfig, FeatureFlags, FeatureVersions, VersionInfo __all__ = ( "DUCKDB_CONFIG", diff --git a/sqlspec/data_dictionary/dialects/mssql/config.py b/sqlspec/data_dictionary/dialects/mssql/config.py index e8948115a..c9836128b 100644 --- a/sqlspec/data_dictionary/dialects/mssql/config.py +++ b/sqlspec/data_dictionary/dialects/mssql/config.py @@ -5,7 +5,9 @@ from typing import TYPE_CHECKING, Any, Final from sqlspec.builder._ddl import compile_table_ddl_results -from sqlspec.data_dictionary import ( +from sqlspec.data_dictionary._capabilities import system_metadata_gated_result +from sqlspec.data_dictionary._registry import register_dialect +from sqlspec.data_dictionary._types import ( DDLResult, DialectConfig, FeatureFlags, @@ -22,14 +24,12 @@ SystemMetadataRequest, SystemMetadataResult, VersionInfo, - register_dialect, - system_metadata_gated_result, ) if TYPE_CHECKING: from collections.abc import Sequence - from sqlspec.data_dictionary import TableMetadata + from sqlspec.data_dictionary._types import TableMetadata __all__ = ( "MSSQL_CONFIG", diff --git a/sqlspec/data_dictionary/dialects/mysql/config.py b/sqlspec/data_dictionary/dialects/mysql/config.py index d81835652..0fadc75f8 100644 --- a/sqlspec/data_dictionary/dialects/mysql/config.py +++ b/sqlspec/data_dictionary/dialects/mysql/config.py @@ -2,7 +2,8 @@ import re -from sqlspec.data_dictionary import ( +from sqlspec.data_dictionary._registry import register_dialect +from sqlspec.data_dictionary._types import ( DDLResult, DialectConfig, FeatureFlags, @@ -16,7 +17,6 @@ ObjectIdentity, SystemMetadataCapability, VersionInfo, - register_dialect, ) from sqlspec.utils.text import quote_backtick_identifier, split_qualified_identifier diff --git a/sqlspec/data_dictionary/dialects/oracle/config.py b/sqlspec/data_dictionary/dialects/oracle/config.py index 0df820a30..82b5489d3 100644 --- a/sqlspec/data_dictionary/dialects/oracle/config.py +++ b/sqlspec/data_dictionary/dialects/oracle/config.py @@ -4,10 +4,11 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Final -from sqlspec.data_dictionary import DialectConfig, FeatureFlags, FeatureVersions, VersionInfo, register_dialect +from sqlspec.data_dictionary._registry import register_dialect +from sqlspec.data_dictionary._types import DialectConfig, FeatureFlags, FeatureVersions, VersionInfo if TYPE_CHECKING: - from sqlspec.data_dictionary import TableMetadata + from sqlspec.data_dictionary._types import TableMetadata __all__ = ( "ORACLE_CONFIG", diff --git a/sqlspec/data_dictionary/dialects/postgres/config.py b/sqlspec/data_dictionary/dialects/postgres/config.py index 4bb9353b6..dc95b1ba5 100644 --- a/sqlspec/data_dictionary/dialects/postgres/config.py +++ b/sqlspec/data_dictionary/dialects/postgres/config.py @@ -2,7 +2,8 @@ import re -from sqlspec.data_dictionary import DialectConfig, FeatureFlags, FeatureVersions, VersionInfo, register_dialect +from sqlspec.data_dictionary._registry import register_dialect +from sqlspec.data_dictionary._types import DialectConfig, FeatureFlags, FeatureVersions, VersionInfo __all__ = ( "POSTGRES_CONFIG", diff --git a/sqlspec/data_dictionary/dialects/snowflake/config.py b/sqlspec/data_dictionary/dialects/snowflake/config.py index 96848409f..a4aa97bc0 100644 --- a/sqlspec/data_dictionary/dialects/snowflake/config.py +++ b/sqlspec/data_dictionary/dialects/snowflake/config.py @@ -4,7 +4,8 @@ from collections.abc import Mapping, Sequence from typing import Any, Final, cast -from sqlspec.data_dictionary import DialectConfig, FeatureFlags, FeatureVersions, ForeignKeyMetadata, register_dialect +from sqlspec.data_dictionary._registry import register_dialect +from sqlspec.data_dictionary._types import DialectConfig, FeatureFlags, FeatureVersions, ForeignKeyMetadata from sqlspec.exceptions import SQLSpecError from sqlspec.utils.text import quote_identifier, resolve_catalog_identifier diff --git a/sqlspec/data_dictionary/dialects/spanner/config.py b/sqlspec/data_dictionary/dialects/spanner/config.py index 0a3d26259..f375dfe6a 100644 --- a/sqlspec/data_dictionary/dialects/spanner/config.py +++ b/sqlspec/data_dictionary/dialects/spanner/config.py @@ -2,7 +2,8 @@ import re -from sqlspec.data_dictionary import DialectConfig, FeatureFlags, FeatureVersions, register_dialect +from sqlspec.data_dictionary._registry import register_dialect +from sqlspec.data_dictionary._types import DialectConfig, FeatureFlags, FeatureVersions __all__ = ( "SPANNER_CONFIG", diff --git a/sqlspec/data_dictionary/dialects/sqlanywhere/config.py b/sqlspec/data_dictionary/dialects/sqlanywhere/config.py index 4652ddc32..f8e65f1fd 100644 --- a/sqlspec/data_dictionary/dialects/sqlanywhere/config.py +++ b/sqlspec/data_dictionary/dialects/sqlanywhere/config.py @@ -4,7 +4,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final -from sqlspec.data_dictionary import register_dialect +from sqlspec.data_dictionary._registry import register_dialect from sqlspec.data_dictionary._types import ( DialectConfig, FeatureFlags, @@ -18,7 +18,7 @@ ) if TYPE_CHECKING: - from sqlspec.data_dictionary import TableMetadata + from sqlspec.data_dictionary._types import TableMetadata __all__ = ( "SQLANYWHERE_CONFIG", diff --git a/sqlspec/data_dictionary/dialects/sqlite/config.py b/sqlspec/data_dictionary/dialects/sqlite/config.py index 1b60e8f9f..ed58dbd3f 100644 --- a/sqlspec/data_dictionary/dialects/sqlite/config.py +++ b/sqlspec/data_dictionary/dialects/sqlite/config.py @@ -7,11 +7,11 @@ from sqlglot import exp from sqlglot.errors import SqlglotError -from sqlspec.data_dictionary import ( +from sqlspec.data_dictionary._dependencies import DependencyEdge, DependencyEdgeKind +from sqlspec.data_dictionary._registry import register_dialect +from sqlspec.data_dictionary._types import ( DDL_PARENT_TABLE_CONTEXT_KEY, DDLResult, - DependencyEdge, - DependencyEdgeKind, DialectConfig, FeatureFlags, FeatureVersions, @@ -20,7 +20,6 @@ MetadataSupport, ObjectIdentity, VersionInfo, - register_dialect, ) from sqlspec.exceptions import SQLSpecError from sqlspec.utils.text import quote_identifier diff --git a/sqlspec/data_dictionary/dialects/sybase/config.py b/sqlspec/data_dictionary/dialects/sybase/config.py index e4df1d757..8721c83ae 100644 --- a/sqlspec/data_dictionary/dialects/sybase/config.py +++ b/sqlspec/data_dictionary/dialects/sybase/config.py @@ -4,7 +4,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final -from sqlspec.data_dictionary import register_dialect +from sqlspec.data_dictionary._registry import register_dialect from sqlspec.data_dictionary._types import ( DialectConfig, FeatureFlags, @@ -18,7 +18,7 @@ ) if TYPE_CHECKING: - from sqlspec.data_dictionary import TableMetadata + from sqlspec.data_dictionary._types import TableMetadata __all__ = ( "SYBASE_CONFIG", diff --git a/sqlspec/data_dictionary/dialects/teradata/config.py b/sqlspec/data_dictionary/dialects/teradata/config.py index f50ea253c..ec98aeaa4 100644 --- a/sqlspec/data_dictionary/dialects/teradata/config.py +++ b/sqlspec/data_dictionary/dialects/teradata/config.py @@ -4,7 +4,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final -from sqlspec.data_dictionary import register_dialect +from sqlspec.data_dictionary._registry import register_dialect from sqlspec.data_dictionary._types import ( DialectConfig, FeatureFlags, @@ -18,7 +18,7 @@ ) if TYPE_CHECKING: - from sqlspec.data_dictionary import TableMetadata + from sqlspec.data_dictionary._types import TableMetadata __all__ = ( "TERADATA_COLUMN_TYPE_CODES", diff --git a/sqlspec/driver/__init__.py b/sqlspec/driver/__init__.py index 2a4b4a189..c4f6177c0 100644 --- a/sqlspec/driver/__init__.py +++ b/sqlspec/driver/__init__.py @@ -33,6 +33,7 @@ from sqlspec.driver._deferred import DeferredAsyncSession, raise_async_connection_required from sqlspec.driver._exception_handler import BaseAsyncExceptionHandler, BaseSyncExceptionHandler from sqlspec.driver._routing import AsyncRoutingSession, SyncRoutingSession + from sqlspec.driver._session import AsyncDriverSessionContext, SyncDriverSessionContext from sqlspec.driver._sql_helpers import convert_to_dialect from sqlspec.driver._stream import AsyncRowStream, SyncRowStream, rows_to_dicts from sqlspec.driver._sync import ( @@ -48,6 +49,7 @@ __all__ = ( "AsyncDataDictionaryBase", "AsyncDriverAdapterBase", + "AsyncDriverSessionContext", "AsyncExceptionHandler", "AsyncPoolConnectionContext", "AsyncPoolSessionFactory", @@ -66,6 +68,7 @@ "StackExecutionObserver", "SyncDataDictionaryBase", "SyncDriverAdapterBase", + "SyncDriverSessionContext", "SyncExceptionHandler", "SyncPoolConnectionContext", "SyncPoolSessionFactory", @@ -86,6 +89,7 @@ _EXPORTS: dict[str, tuple[str, str]] = { "AsyncDataDictionaryBase": ("sqlspec.driver._async", "AsyncDataDictionaryBase"), "AsyncDriverAdapterBase": ("sqlspec.driver._async", "AsyncDriverAdapterBase"), + "AsyncDriverSessionContext": ("sqlspec.driver._session", "AsyncDriverSessionContext"), "AsyncExceptionHandler": ("sqlspec.driver._common", "AsyncExceptionHandler"), "AsyncPoolConnectionContext": ("sqlspec.driver._async", "AsyncPoolConnectionContext"), "AsyncPoolSessionFactory": ("sqlspec.driver._async", "AsyncPoolSessionFactory"), @@ -103,6 +107,7 @@ "StackExecutionObserver": ("sqlspec.driver._common", "StackExecutionObserver"), "SyncDataDictionaryBase": ("sqlspec.driver._sync", "SyncDataDictionaryBase"), "SyncDriverAdapterBase": ("sqlspec.driver._sync", "SyncDriverAdapterBase"), + "SyncDriverSessionContext": ("sqlspec.driver._session", "SyncDriverSessionContext"), "SyncExceptionHandler": ("sqlspec.driver._common", "SyncExceptionHandler"), "SyncPoolConnectionContext": ("sqlspec.driver._sync", "SyncPoolConnectionContext"), "SyncPoolSessionFactory": ("sqlspec.driver._sync", "SyncPoolSessionFactory"), diff --git a/sqlspec/driver/_async.py b/sqlspec/driver/_async.py index 7e181ca9c..fe675d1cd 100644 --- a/sqlspec/driver/_async.py +++ b/sqlspec/driver/_async.py @@ -16,23 +16,26 @@ from mypy_extensions import mypyc_attr -from sqlspec.core import SQL, CachedQuery, StackResult, create_arrow_result -from sqlspec.core.result import DMLResult +from sqlspec.core.cache import CachedQuery +from sqlspec.core.result._base import DMLResult, SQLResult, StackResult, create_arrow_result from sqlspec.core.stack import StackOperation, StatementStack -from sqlspec.data_dictionary import ( +from sqlspec.core.statement import SQL +from sqlspec.data_dictionary._capabilities import ( DEFAULT_SYSTEM_METADATA_DOMAINS, + ensure_system_metadata_request, + system_metadata_capabilities_from_domains, + system_metadata_gated_result, + unsupported_system_metadata_capability, +) +from sqlspec.data_dictionary._index_ddl import AsyncWithoutIndexesContext +from sqlspec.data_dictionary._types import ( ColumnProfileMetadata, DDLResult, MetadataCapabilityProfile, MetadataResult, MetadataSupport, ObjectIdentity, - ensure_system_metadata_request, - system_metadata_capabilities_from_domains, - system_metadata_gated_result, - unsupported_system_metadata_capability, ) -from sqlspec.data_dictionary._index_ddl import AsyncWithoutIndexesContext from sqlspec.driver._common import ( _CLOSED_CONNECTION, AsyncExceptionHandler, @@ -79,9 +82,12 @@ from sqlglot.dialects.dialect import DialectType - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, OperationType, SQLResult, Statement, StatementConfig, StatementFilter - from sqlspec.data_dictionary import ( + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.compiler import OperationType + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult + from sqlspec.core.statement import Statement, StatementConfig + from sqlspec.data_dictionary._types import ( ColumnMetadata, ForeignKeyMetadata, IndexMetadata, @@ -123,11 +129,11 @@ def __init__(self, config: Any) -> None: async def __aenter__(self) -> Any: if self._tenant_lease is not None: return await self._tenant_lease.acquire() - if hasattr(self._config, "provide_pool"): - pool = await self._config.provide_pool() - else: - pool = self._config.connection_instance - if pool is None: + pool = self._config.connection_instance + if pool is None: + if hasattr(self._config, "provide_pool"): + pool = await self._config.provide_pool() + else: pool = await self._config.create_pool() self._config.connection_instance = pool self._pool = pool @@ -168,11 +174,11 @@ def __init__(self, config: Any) -> None: async def acquire_connection(self) -> Any: if self._tenant_lease is not None: return await self._tenant_lease.acquire() - if hasattr(self._config, "provide_pool"): - pool = await self._config.provide_pool() - else: - pool = self._config.connection_instance - if pool is None: + pool = self._config.connection_instance + if pool is None: + if hasattr(self._config, "provide_pool"): + pool = await self._config.provide_pool() + else: pool = await self._config.create_pool() self._config.connection_instance = pool self._pool = pool @@ -560,6 +566,25 @@ async def execute( **kwargs: Any, ) -> "SQLResult": """Execute a statement with parameter handling.""" + if ( + self._stmt_cache_enabled + and (statement_config is None or statement_config is self.statement_config) + and isinstance(statement, str) + ): + fast_params = _resolve_fast_params(parameters, kwargs) + if fast_params is not None: + if self._auto_begin_on_write: + await self._ensure_transaction_for_cached_sql(statement, fast_params) + outcome = run_cached_execution(self, statement, fast_params) + if outcome is not None: + if isinstance(outcome, SQLResult): + return outcome + fast_result = await outcome + if fast_result is not None: + return fast_result + cached = self._stmt_cache.get(statement) + if cached is not None: + return await self._execute_cache_hit(statement, fast_params, cached) return await self._execute(statement, parameters, statement_config, kwargs) async def execute_many( @@ -1727,6 +1752,7 @@ async def _execute_with_observability_async( is_many: bool, is_script: bool, runner: "Callable[..., Awaitable[SQLResult]]", + precomputed_sql_hash: "str | None" = None, **runner_kwargs: Any, ) -> "SQLResult": """Execute an async runner while emitting spans, lifecycle hooks, and statement events.""" @@ -1751,9 +1777,10 @@ async def _execute_with_observability_async( if db_system is not None: query_context["db_system"] = db_system await runtime.emit_query_start_async(**query_context) - sql_hash = None - if runtime.span_manager.is_enabled or ( - runtime.has_statement_observers and runtime.config.logging and runtime.config.logging.include_sql_hash + sql_hash = precomputed_sql_hash if not is_script else None + if sql_hash is None and ( + runtime.span_manager.is_enabled + or (runtime.has_statement_observers and runtime.config.logging and runtime.config.logging.include_sql_hash) ): sql_hash = observability_runtime.compute_sql_hash(observed_sql) span = runtime.start_query_span(observed_sql, operation, driver_name, sql_hash=sql_hash, db_system=db_system) @@ -1875,13 +1902,21 @@ async def _execute_cache_hit_observed( ) -> "SQLResult": """Execute an async fast-path cache hit wrapped in observability instrumentation.""" obs_params = list(params) if isinstance(params, tuple) else params + wire_sql = self._format_wire_sql(cached.compiled_sql) + sql_hash: str | None = None + if not self.statement_config.enable_sqlcommenter: + sql_hash = cached._sql_hash + if sql_hash is None: + sql_hash = observability_runtime.compute_sql_hash(wire_sql) + cached._sql_hash = sql_hash return await self._execute_with_observability_async( - compiled_sql=self._format_wire_sql(cached.compiled_sql), + compiled_sql=wire_sql, execution_parameters=obs_params, operation=cached.operation_type, is_many=False, is_script=False, runner=self._execute_cache_hit, + precomputed_sql_hash=sql_hash, sql=sql, params=params, cached=cached, @@ -1934,20 +1969,6 @@ async def _execute( statement_config: "StatementConfig | None", kwargs: "dict[str, Any]", ) -> "SQLResult": - fast_params = _resolve_fast_params(parameters, kwargs) - if ( - self._stmt_cache_enabled - and (statement_config is None or statement_config is self.statement_config) - and isinstance(statement, str) - and fast_params is not None - ): - if self._auto_begin_on_write: - await self._ensure_transaction_for_cached_sql(statement, fast_params) - pending_result = run_cached_execution(self, statement, fast_params) - if pending_result is not None: - fast_result = await cast("Awaitable[SQLResult | None]", pending_result) - if fast_result is not None: - return fast_result sql_statement = self.prepare_statement( statement, parameters, statement_config=statement_config or self.statement_config, kwargs=kwargs ) @@ -2105,6 +2126,8 @@ async def _execute_cache_hit_with_cursor( fetched_data = fetchall() if isawaitable(fetched_data): fetched_data = await fetched_data + if isinstance(fetched_data, tuple): + fetched_data = list(fetched_data) data, column_names, row_count = self.collect_rows(cursor, fetched_data) result = self.build_cached_select_result( sql, diff --git a/sqlspec/driver/_common.py b/sqlspec/driver/_common.py index 4df0df2fa..38a95ed06 100644 --- a/sqlspec/driver/_common.py +++ b/sqlspec/driver/_common.py @@ -38,30 +38,29 @@ from sqlglot.errors import ParseError, SqlglotError from sqlglot.tokenizer_core import TokenType -from sqlspec.builder import QueryBuilder +from sqlspec.builder._base import QueryBuilder from sqlspec.builder._ddl import compile_table_ddl_results from sqlspec.builder._parsing_utils import _normalize_dialect -from sqlspec.core import ( - SQL, +from sqlspec.core._pool import ObjectPool, get_processed_state_pool, get_sql_pool +from sqlspec.core.cache import ( + _POSITIONAL_NUMBERED_STYLES, + CachedQuery, CachedStatement, - OperationProfile, - ParameterDeclaration, - ParameterStyle, - SQLResult, - Statement, - StatementConfig, - TypedParameter, + QueryCache, + StatementCacheHolder, get_cache, get_cache_config, - matches_param_type, - split_sql_script, + resolve_statement_cache_size, +) +from sqlspec.core.compiler import ( + OperationProfile, + _split_leading_keyword, # pyright: ignore[reportPrivateUsage] ) -from sqlspec.core._pool import get_processed_state_pool, get_sql_pool -from sqlspec.core.cache import CachedQuery, QueryCache, StatementCacheHolder, resolve_statement_cache_size -from sqlspec.core.compiler import _split_leading_keyword # pyright: ignore[reportPrivateUsage] from sqlspec.core.filters import find_filter as _find_filter_impl from sqlspec.core.metrics import StackExecutionMetrics -from sqlspec.core.parameters import ( +from sqlspec.core.parameters._converter import ambiguous_index_message +from sqlspec.core.parameters._declared import ParameterDeclaration, matches_param_type +from sqlspec.core.parameters._processor import ( ParameterProcessor, apply_type_coercion, structural_fingerprint, @@ -69,17 +68,25 @@ type_coercion_fallbacks, value_fingerprint, ) -from sqlspec.core.parameters._converter import ambiguous_index_message -from sqlspec.core.result._base import RowFormat +from sqlspec.core.parameters._types import ParameterStyle, TypedParameter +from sqlspec.core.result._base import RowFormat, SQLResult +from sqlspec.core.splitter import split_sql_script from sqlspec.core.sqlcommenter import format_wire_sqlcomment, strip_sqlcomment -from sqlspec.core.statement import ProcessedState, _is_uncacheable_callable -from sqlspec.data_dictionary import ( - DDL_PARENT_TABLE_CONTEXT_KEY, - ColumnProfileMetadata, - DDLResult, +from sqlspec.core.statement import SQL, ProcessedState, Statement, StatementConfig, _is_uncacheable_callable +from sqlspec.data_dictionary._dependencies import ( DependencyEdge, DependencyEdgeKind, DependencyStrength, + dependency_edges_from_foreign_keys, + sort_ddl_results, + sort_dependencies, +) +from sqlspec.data_dictionary._loader import get_data_dictionary_loader +from sqlspec.data_dictionary._registry import get_dialect_config +from sqlspec.data_dictionary._types import ( + DDL_PARENT_TABLE_CONTEXT_KEY, + ColumnProfileMetadata, + DDLResult, ForeignKeyMetadata, MetadataCapability, MetadataFidelity, @@ -89,12 +96,7 @@ ObjectIdentity, VersionCacheResult, VersionInfo, - dependency_edges_from_foreign_keys, - get_data_dictionary_loader, - sort_ddl_results, - sort_dependencies, ) -from sqlspec.data_dictionary._registry import get_dialect_config from sqlspec.driver._stream import _CLOSED_CONNECTION_MESSAGE, SyncRowStream from sqlspec.exceptions import ( ImproperConfigurationError, @@ -104,8 +106,8 @@ StorageCapabilityError, TransactionError, ) -from sqlspec.observability import ObservabilityRuntime, get_trace_context, resolve_db_system -from sqlspec.observability._common import redact_sql_literals +from sqlspec.observability._common import get_trace_context, redact_sql_literals, resolve_db_system +from sqlspec.observability._runtime import ObservabilityRuntime from sqlspec.protocols import StatementProtocol from sqlspec.storage._ingest import ( execute_load_from_storage_async, @@ -133,7 +135,7 @@ ) from sqlspec.utils.logging import get_logger, log_with_context from sqlspec.utils.schema import to_schema as _to_schema_impl -from sqlspec.utils.serializers import from_json as decode_json +from sqlspec.utils.serializers._json import from_json from sqlspec.utils.text import normalize_identifier, quote_identifier, split_qualified_identifier from sqlspec.utils.type_guards import ( has_asdict_method, @@ -149,11 +151,13 @@ from collections.abc import Awaitable from types import TracebackType - from sqlspec.core import ArrowResult, FilterTypeT, OperationType, StatementFilter + from sqlspec.core.compiler import OperationType + from sqlspec.core.filters import FilterTypeT, StatementFilter from sqlspec.core.parameters._types import ConvertedParameters + from sqlspec.core.result._base import ArrowResult from sqlspec.core.stack import StatementStack - from sqlspec.data_dictionary import DialectConfig - from sqlspec.storage import ( + from sqlspec.data_dictionary._types import DialectConfig + from sqlspec.storage.pipeline import ( AsyncStoragePipeline, StorageBridgeJob, StorageCapabilities, @@ -182,6 +186,7 @@ "parameter_value_needs_processing", "parameter_values_need_processing", "type_coercion_fallbacks", + "validate_load_from_storage_kwargs", "validate_savepoint_name", ) @@ -312,7 +317,9 @@ def handle_single_row_error(error: ValueError) -> "NoReturn": message = str(error) if message.startswith("No result found"): msg = "No rows found" - raise NotFoundError(msg) from error + not_found = NotFoundError(msg) + not_found.__cause__ = error + raise not_found from error raise error @@ -1830,7 +1837,9 @@ class CommonDriverAttributesMixin: "_cache_hit_executor", "_cache_hit_state", "_cache_holder", + "_has_active_observability", "_observability", + "_observability_generation", "_open_streams", "_pending_statement_profile", "_processed_state_pool", @@ -1896,8 +1905,10 @@ def __init__( | None ) = None self._cache_hit_state: Any = None - self._statement_pool = get_sql_pool() - self._processed_state_pool = get_processed_state_pool() + self._has_active_observability = False + self._observability_generation = -1 + self._statement_pool: ObjectPool[SQL] | None = None + self._processed_state_pool: ObjectPool[ProcessedState] | None = None self._refresh_statement_cache_state() @property @@ -2354,6 +2365,9 @@ def _statement_cache_size(self) -> int: return resolve_statement_cache_size(self.driver_features) def _refresh_statement_cache_state(self) -> None: + runtime = self._observability + self._has_active_observability = runtime is not None and not runtime.is_idle + self._observability_generation = runtime.hook_generation if runtime is not None else -1 self._stmt_cache_enabled = bool( self.connection is not _CLOSED_CONNECTION and self._stmt_cache_max_size > 0 @@ -2391,9 +2405,23 @@ def _raise_storage_not_implemented(self, capability: str) -> NoReturn: remediation = "Override storage methods on the adapter to enable this capability." raise StorageCapabilityError(msg, capability=capability, remediation=remediation) + def _get_statement_pool(self) -> "ObjectPool[SQL]": + pool = self._statement_pool + if pool is None: + pool = get_sql_pool() + self._statement_pool = pool + return pool + + def _get_processed_state_pool(self) -> "ObjectPool[ProcessedState]": + pool = self._processed_state_pool + if pool is None: + pool = get_processed_state_pool() + self._processed_state_pool = pool + return pool + def _release_pooled_statement(self, statement: "SQL") -> None: if statement._pooled: - self._statement_pool.release(statement) + self._get_statement_pool().release(statement) def _cached_statement( self, @@ -2525,7 +2553,7 @@ def _bound_script_statements(self, statement: "SQL") -> "list[SQL] | None": return statements def _sub_statement(self, sql: str, execution_parameters: "ConvertedParameters") -> "SQL": - direct_state = self._processed_state_pool.acquire() + direct_state = self._get_processed_state_pool().acquire() ProcessedState.__init__(direct_state, compiled_sql=sql, execution_parameters=execution_parameters) return SQL._create_cached_direct(sql, self.statement_config, direct_state) @@ -2650,6 +2678,23 @@ def _cache_statement(self, statement: "SQL") -> None: else processed.compiled_sql ) cached_state = _clone_processed_state(processed, operation_profile, compiled_sql=clean_sql) + needs_style_remap = bool(_CACHED_NAMED_STYLES.intersection(param_profile.styles)) + stmt_config = self.statement_config + positional_passthrough = (not processed.applied_wrap_types and not needs_style_remap) and ( + not processed.input_named_parameters + or ( + bool(param_profile.styles) + and _POSITIONAL_NUMBERED_STYLES.issuperset(param_profile.styles) + and len(processed.input_named_parameters) == param_profile.total_count + and processed.input_named_parameters == tuple(str(i) for i in range(1, param_profile.total_count + 1)) + ) + ) + simple_wire_fast_path = ( + positional_passthrough + and not stmt_config._has_output_transformer + and not stmt_config.enable_sqlcommenter + and stmt_config.parameter_config.ast_transformer is None + ) cached = CachedQuery( compiled_sql=clean_sql, parameter_profile=param_profile, @@ -2660,7 +2705,9 @@ def _cache_statement(self, statement: "SQL") -> None: operation_profile=operation_profile, param_count=param_profile.total_count, processed_state=cached_state, - needs_style_remap=bool(_CACHED_NAMED_STYLES.intersection(param_profile.styles)), + needs_style_remap=needs_style_remap, + positional_passthrough=positional_passthrough, + simple_wire_fast_path=simple_wire_fast_path, ) self._stmt_cache.set(statement.raw_sql, cached) @@ -3065,6 +3112,16 @@ def _compiled_statement( ) if not isinstance(cached_result, CachedStatement): cached_result = None + if cached_result is not None: + expected_guard = ( + statement.raw_sql, + statement.is_many, + statement.is_script, + flatten_single_parameters, + statement_config, + ) + if cached_result._source_guard is not None and cached_result._source_guard != expected_guard: + cached_result = None if cached_result is not None: parameter_profile = cached_result.parameter_profile if parameter_profile is not None: @@ -3132,17 +3189,21 @@ def _compiled_statement( ) if cache_key is not None and cache is not None: - stored_statement = ( - CachedStatement( - compiled_sql=stored_sql, - parameters=cached_statement.parameters, - expression=cached_statement.expression, - input_named_parameters=cached_statement.input_named_parameters, - parameter_profile=cached_statement.parameter_profile, - applied_wrap_types=cached_statement.applied_wrap_types, - ) - if stored_sql != compiled_sql - else cached_statement + source_guard = ( + statement.raw_sql, + statement.is_many, + statement.is_script, + flatten_single_parameters, + statement_config, + ) + stored_statement = CachedStatement( + compiled_sql=stored_sql, + parameters=cached_statement.parameters, + expression=cached_statement.expression, + input_named_parameters=cached_statement.input_named_parameters, + parameter_profile=cached_statement.parameter_profile, + applied_wrap_types=cached_statement.applied_wrap_types, + _source_guard=source_guard, ) cache.put_statement(cache_key, stored_statement, dialect_key) @@ -3666,18 +3727,17 @@ def cache_hit_hooks_unchanged(driver_type: type, owner: type) -> bool: return all(getattr(driver_type, name, None) is getattr(owner, name, None) for name in _CACHE_HIT_HOOKS) -async def _await_cache_hit( - driver: "CommonDriverAttributesMixin", - pending: "Awaitable[SQLResult | None]", - statement: str, - params: "tuple[Any, ...] | list[Any] | dict[str, Any]", - cached: "CachedQuery", -) -> "SQLResult | None": - """Await an async cache-hit executor and run the driver's cache-hit method when it declines.""" - outcome = await pending - if outcome is not None: - return outcome - return await cast("Awaitable[SQLResult]", driver._execute_cache_hit(statement, params, cached)) +def _has_active_observability(driver: "CommonDriverAttributesMixin") -> bool: + """Return whether the driver's runtime has hooks, observers, or spans to notify. + + The driver caches the answer and recomputes it when a lifecycle hook is registered on the runtime. + """ + runtime = driver._observability + if runtime is None: + return False + if runtime.hook_generation != driver._observability_generation: + driver._refresh_statement_cache_state() + return driver._has_active_observability def run_cached_execution( @@ -3706,18 +3766,38 @@ def run_cached_execution( config = driver.statement_config param_config = config.parameter_config - if param_config.ast_transformer is not None: + if cached.simple_wire_fast_path: + is_mapping = _is_mapping_parameters(params) + if not is_mapping: + needs_rebind = False + if params: + holder = getattr(driver, "_cache_holder", None) + passthrough = holder.passthrough_types if holder is not None else None + param_values = _cache_param_values(params) + needs_rebind = parameter_values_need_processing( + param_values, param_config.type_coercion_map, passthrough_types=passthrough + ) + if not needs_rebind: + if _has_active_observability(driver): + return driver._execute_cache_hit_observed(statement, params, cached) + executor = driver._cache_hit_executor + if executor is None: + return driver._execute_cache_hit(statement, params, cached) + outcome = executor(driver.connection, statement, params, cached, config, driver._cache_hit_state) + if outcome is None: + return driver._execute_cache_hit(statement, params, cached) + return outcome + elif param_config.ast_transformer is not None: return None - needs_rebind = bool( - cached.input_named_parameters - or cached.applied_wrap_types - or cached.needs_style_remap - or _is_mapping_parameters(params) - ) + needs_rebind = (not cached.positional_passthrough) or _is_mapping_parameters(params) if not needs_rebind and params: + holder = getattr(driver, "_cache_holder", None) + passthrough = holder.passthrough_types if holder is not None else None param_values = _cache_param_values(params) - needs_rebind = parameter_values_need_processing(param_values, param_config.type_coercion_map) + needs_rebind = parameter_values_need_processing( + param_values, param_config.type_coercion_map, passthrough_types=passthrough + ) if not needs_rebind and not config._has_output_transformer: effective_cached = cached @@ -3735,9 +3815,10 @@ def run_cached_execution( param_count=cached.param_count, processed_state=cached.processed_state, needs_style_remap=cached.needs_style_remap, + positional_passthrough=cached.positional_passthrough, + simple_wire_fast_path=False, ) - runtime = driver._observability - if runtime is not None and not runtime.is_idle: + if _has_active_observability(driver): return driver._execute_cache_hit_observed(statement, params, effective_cached) executor = driver._cache_hit_executor if executor is None: @@ -3745,9 +3826,7 @@ def run_cached_execution( outcome = executor(driver.connection, statement, params, effective_cached, config, driver._cache_hit_state) if outcome is None: return driver._execute_cache_hit(statement, params, effective_cached) - if isinstance(outcome, SQLResult): - return outcome - return _await_cache_hit(driver, outcome, statement, params, effective_cached) + return outcome if needs_rebind: rebound_params = driver.stmt_cache_rebind(params, cached) @@ -3765,8 +3844,7 @@ def run_cached_execution( prepared = driver._cached_statement( statement, params, cached, rebound_params, params_are_simple=params_are_simple, compiled_sql=compiled_sql ) - runtime = driver._observability - if runtime is not None and not runtime.is_idle: + if _has_active_observability(driver): return driver._execute_cached_statement_observed(prepared, compiled_sql, rebound_params, cached.operation_type) return driver._execute_cached_statement(prepared) @@ -3775,6 +3853,7 @@ def parameter_values_need_processing( values: "tuple[Any, ...] | list[Any]", type_coercion_map: "dict[type, Any] | None", fallback_items: "tuple[tuple[type, Any], ...] | None" = None, + passthrough_types: "set[type] | None" = None, ) -> bool: if not values: return False @@ -3791,17 +3870,17 @@ def parameter_values_need_processing( if dispatcher.resolve_type(value_type) is not None: return True return False - passthrough_types = _passthrough_types(type_coercion_map) + active_passthrough = passthrough_types if passthrough_types is not None else _passthrough_types(type_coercion_map) for value in values: value_type = type(value) - if value_type in passthrough_types: + if value_type in active_passthrough: continue if value_type is TypedParameter or value_type in type_coercion_map: return True if type_coercion_dispatcher(type_coercion_fallbacks(type_coercion_map)).resolve_type(value_type) is not None: return True - if len(passthrough_types) < _PASSTHROUGH_TYPES_MAX_SIZE: - passthrough_types.add(value_type) + if len(active_passthrough) < _PASSTHROUGH_TYPES_MAX_SIZE: + active_passthrough.add(value_type) return False @@ -3871,6 +3950,7 @@ def _raise_database_exception( pending_exception = exc_handler.pending_exception if pending_exception is not None: if exc is not None: + pending_exception.__cause__ = exc raise pending_exception from exc if pending_exception.__cause__ is not None: raise pending_exception @@ -4640,6 +4720,34 @@ def _is_valid_profile_datetime(value: Any) -> bool: return True +_LOAD_FROM_STORAGE_KWARGS: Final[frozenset[str]] = frozenset({ + "arrow_schema", + "checkpoint_phase", + "checkpoint_table", + "concurrent_indexes", + "conflict_keys", + "drop_indexes", + "drop_stage_table", + "exclude_update_columns", + "executor", + "max_workers", + "on_conflict", + "stage_table", + "storage_options", + "type_casts", +}) + + +def validate_load_from_storage_kwargs(kwargs: "dict[str, Any]") -> None: + """Validate optional keyword arguments forwarded to ``load_from_storage``.""" + if not kwargs: + return + unknown = sorted(set(kwargs) - _LOAD_FROM_STORAGE_KWARGS) + if unknown: + msg = f"load_from_storage() got unexpected keyword argument '{unknown[0]}'" + raise TypeError(msg) + + def _is_valid_profile_json(value: Any) -> bool: if isinstance(value, (Mapping, list)): return True @@ -4651,7 +4759,7 @@ def _is_valid_profile_json(value: Any) -> bool: ): return False try: - parsed = decode_json(stripped) + parsed = from_json(stripped) except ValueError: return False return isinstance(parsed, (Mapping, list)) diff --git a/sqlspec/driver/_deferred.py b/sqlspec/driver/_deferred.py index 662f796c2..b1444c138 100644 --- a/sqlspec/driver/_deferred.py +++ b/sqlspec/driver/_deferred.py @@ -11,7 +11,7 @@ from typing import Any, NoReturn from sqlspec.driver._stream import _CLOSED_CONNECTION_MESSAGE -from sqlspec.exceptions import ImproperConfigurationError +from sqlspec.exceptions import ImproperConfigurationError, with_cause __all__ = ("DeferredAsyncSession", "raise_async_connection_required") @@ -29,6 +29,8 @@ class DeferredAsyncSession: ``async with session.transaction():`` all work. - Plain class attributes such as ``dialect``, static helpers, and the configuration's ``statement_config`` and ``driver_features`` are served without opening a connection. + When the driver stores ``dialect`` per instance, it is read from the configuration's + statement config. - Attributes that need a live connection, such as ``connection``, raise :class:`~sqlspec.exceptions.ImproperConfigurationError`. @@ -89,7 +91,7 @@ def __getattr__(self, name: str) -> Any: static = inspect.getattr_static(driver_type, name) except AttributeError: msg = f"'{driver_type.__name__}' object has no attribute '{name}'" - raise AttributeError(msg) from None + raise with_cause(AttributeError(msg), None) from None if name in _CONFIG_ATTRIBUTES: value = getattr(self._config, name, None) if value is not None: @@ -99,6 +101,10 @@ def __getattr__(self, name: str) -> Any: return getattr(driver_type, name) if callable(static): return _MethodProxy(self, name) + if name == "dialect" and not isinstance(static, str): + dialect = getattr(getattr(self._config, "statement_config", None), "dialect", None) + if dialect is not None: + return dialect if hasattr(type(static), "__get__"): self._raise_not_open(name) return static diff --git a/sqlspec/driver/_routing.py b/sqlspec/driver/_routing.py index 566e6bc47..d3f766df4 100644 --- a/sqlspec/driver/_routing.py +++ b/sqlspec/driver/_routing.py @@ -1,23 +1,18 @@ """Lazy sync and async session routing with explicit driver method contracts.""" import sys -from collections.abc import AsyncIterator, Iterator, Mapping, Sequence -from contextlib import ( - AbstractAsyncContextManager, - AbstractContextManager, - AsyncExitStack, - ExitStack, - asynccontextmanager, - contextmanager, -) +from collections.abc import Generator, Mapping, Sequence +from contextlib import AbstractAsyncContextManager, AbstractContextManager, AsyncExitStack, ExitStack, contextmanager from types import TracebackType from typing import TYPE_CHECKING, Any, Final, Generic, Literal, Self, cast, overload from uuid import UUID +from mypy_extensions import mypyc_attr from typing_extensions import TypeVar -from sqlspec.core import SQL, ArrowResult, SQLResult, StackResult, Statement, StatementConfig, StatementFilter +from sqlspec.core.result._base import ArrowResult, SQLResult, StackResult from sqlspec.core.stack import StatementStack +from sqlspec.core.statement import SQL, Statement, StatementConfig from sqlspec.driver._async import AsyncDriverAdapterBase from sqlspec.driver._common import CommonDriverAttributesMixin from sqlspec.driver._sql_helpers import DEFAULT_PRETTY @@ -26,14 +21,15 @@ from sqlspec.driver._sync import SyncDataDictionaryBase, SyncDriverAdapterBase from sqlspec.exceptions import ImproperConfigurationError, RoutingContextError, SQLSpecError, TransactionError from sqlspec.utils.routing import RoutingContext -from sqlspec.utils.tenant import TenantContextPayload +from sqlspec.utils.tenant._context import TenantContextPayload if TYPE_CHECKING: from sqlglot.dialects.dialect import DialectType - from sqlspec.builder import QueryBuilder + from sqlspec.builder._base import QueryBuilder from sqlspec.config import AsyncDatabaseConfig, NoPoolAsyncConfig, NoPoolSyncConfig, SyncDatabaseConfig - from sqlspec.data_dictionary import ( + from sqlspec.core.filters import StatementFilter + from sqlspec.data_dictionary._types import ( ColumnMetadata, ColumnProfileMetadata, DDLResult, @@ -47,9 +43,14 @@ TableMetadata, VersionInfo, ) - from sqlspec.observability import ObservabilityRuntime - from sqlspec.storage import StorageCapabilities - from sqlspec.storage.pipeline import StorageBridgeJob, StorageDestination, StorageFormat, StorageTelemetry + from sqlspec.observability._runtime import ObservabilityRuntime + from sqlspec.storage.pipeline import ( + StorageBridgeJob, + StorageCapabilities, + StorageDestination, + StorageFormat, + StorageTelemetry, + ) from sqlspec.typing import ArrowReturnFormat, SchemaT, StatementParameters from sqlspec.utils.schema import ValueT @@ -60,6 +61,7 @@ _ARROW_STACK_OPTIONS: Final[tuple[str, ...]] = ("return_format", "native_only", "batch_size", "arrow_schema") +@mypyc_attr(allow_interpreted_subclasses=True) class _RoutingSessionState: __slots__ = ( "_active_replica_group", @@ -288,6 +290,7 @@ def _can_reuse_primary_session(self, *, strict_replica: bool) -> bool: ) +@mypyc_attr(allow_interpreted_subclasses=True) class SyncRoutingSession(_RoutingSessionState, Generic[SyncDriverT]): """Lazy synchronous session selecting replicas for provably safe reads.""" @@ -402,7 +405,7 @@ def rollback_to_savepoint(self, name: str) -> None: self._primary_driver.rollback_to_savepoint(name) @contextmanager - def transaction(self) -> Iterator[Self]: + def transaction(self) -> Generator[Self, None, None]: """Own a primary transaction while preserving nested savepoint behavior. Yields: @@ -414,7 +417,9 @@ def transaction(self) -> Iterator[Self]: yield self @contextmanager - def without_indexes(self, table: str, *, schema: str | None = None, concurrent: bool = False) -> Iterator[Any]: + def without_indexes( + self, table: str, *, schema: str | None = None, concurrent: bool = False + ) -> Generator[Any, None, None]: """Temporarily remove indexes using the primary's native context. Yields: @@ -887,14 +892,326 @@ def load_from_records( type_casts=type_casts, ) - fetch = select - fetch_one = select_one - fetch_one_or_none = select_one_or_none - fetch_value = select_value - fetch_value_or_none = select_value_or_none - fetch_with_total = select_with_total - fetch_to_arrow = select_to_arrow - fetch_stream = select_stream + @overload + def fetch( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT]", + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "list[SchemaT]": ... + + @overload + def fetch( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: None = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "list[dict[str, Any]]": ... + + def fetch( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT] | None" = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "list[SchemaT] | list[dict[str, Any]]": + """Execute a select statement and return all rows.""" + if schema_type is not None: + return self.select( + statement, *parameters, schema_type=schema_type, statement_config=statement_config, **kwargs + ) + return self.select(statement, *parameters, schema_type=None, statement_config=statement_config, **kwargs) + + @overload + def fetch_one( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT]", + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "SchemaT": ... + + @overload + def fetch_one( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: None = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "dict[str, Any]": ... + + def fetch_one( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT] | None" = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "SchemaT | dict[str, Any]": + """Execute a select statement and return exactly one row.""" + if schema_type is not None: + return self.select_one( + statement, *parameters, schema_type=schema_type, statement_config=statement_config, **kwargs + ) + return self.select_one(statement, *parameters, schema_type=None, statement_config=statement_config, **kwargs) + + @overload + def fetch_one_or_none( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT]", + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "SchemaT | None": ... + + @overload + def fetch_one_or_none( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: None = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "dict[str, Any] | None": ... + + def fetch_one_or_none( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT] | None" = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "SchemaT | dict[str, Any] | None": + """Execute a select statement and return at most one row.""" + if schema_type is not None: + return self.select_one_or_none( + statement, *parameters, schema_type=schema_type, statement_config=statement_config, **kwargs + ) + return self.select_one_or_none( + statement, *parameters, schema_type=None, statement_config=statement_config, **kwargs + ) + + @overload + def fetch_value( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + value_type: "type[ValueT]", + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "ValueT": ... + + @overload + def fetch_value( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + value_type: None = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> Any: ... + + def fetch_value( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + value_type: "type[ValueT] | None" = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "ValueT | Any": + """Execute a select statement and return a single scalar value.""" + if value_type is not None: + return self.select_value( + statement, *parameters, value_type=value_type, statement_config=statement_config, **kwargs + ) + return self.select_value(statement, *parameters, statement_config=statement_config, **kwargs) + + @overload + def fetch_value_or_none( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + value_type: "type[ValueT]", + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "ValueT | None": ... + + @overload + def fetch_value_or_none( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + value_type: None = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> Any: ... + + def fetch_value_or_none( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + value_type: "type[ValueT] | None" = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "ValueT | Any | None": + """Execute a select statement and return a single scalar value or None.""" + if value_type is not None: + return self.select_value_or_none( + statement, *parameters, value_type=value_type, statement_config=statement_config, **kwargs + ) + return self.select_value_or_none(statement, *parameters, statement_config=statement_config, **kwargs) + + @overload + def fetch_with_total( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT]", + statement_config: "StatementConfig | None" = None, + count_with_window: bool = False, + **kwargs: Any, + ) -> "tuple[list[SchemaT], int]": ... + + @overload + def fetch_with_total( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: None = None, + statement_config: "StatementConfig | None" = None, + count_with_window: bool = False, + **kwargs: Any, + ) -> "tuple[list[dict[str, Any]], int]": ... + + def fetch_with_total( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT] | None" = None, + statement_config: "StatementConfig | None" = None, + count_with_window: bool = False, + **kwargs: Any, + ) -> "tuple[list[SchemaT] | list[dict[str, Any]], int]": + """Execute a select statement and return both the data and total count.""" + if schema_type is not None: + return self.select_with_total( + statement, + *parameters, + schema_type=schema_type, + statement_config=statement_config, + count_with_window=count_with_window, + **kwargs, + ) + return self.select_with_total( + statement, *parameters, statement_config=statement_config, count_with_window=count_with_window, **kwargs + ) + + def fetch_to_arrow( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + statement_config: "StatementConfig | None" = None, + return_format: "ArrowReturnFormat" = "table", + native_only: bool = False, + batch_size: int | None = None, + arrow_schema: Any = None, + **kwargs: Any, + ) -> "ArrowResult": + """Execute query and return results as Apache Arrow format.""" + return self.select_to_arrow( + statement, + *parameters, + statement_config=statement_config, + return_format=return_format, + native_only=native_only, + batch_size=batch_size, + arrow_schema=arrow_schema, + **kwargs, + ) + + @overload + def fetch_stream( + self, + statement: "SQL | Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT]", + statement_config: "StatementConfig | None" = None, + chunk_size: int = 1000, + native_only: bool = False, + **kwargs: Any, + ) -> "SyncRowStream[SchemaT]": ... + + @overload + def fetch_stream( + self, + statement: "SQL | Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: None = None, + statement_config: "StatementConfig | None" = None, + chunk_size: int = 1000, + native_only: bool = False, + **kwargs: Any, + ) -> "SyncRowStream[dict[str, Any]]": ... + + def fetch_stream( + self, + statement: "SQL | Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT] | None" = None, + statement_config: "StatementConfig | None" = None, + chunk_size: int = 1000, + native_only: bool = False, + **kwargs: Any, + ) -> "SyncRowStream[SchemaT] | SyncRowStream[dict[str, Any]]": + """Execute a query and stream rows in chunks.""" + if schema_type is not None: + return self.select_stream( + statement, + *parameters, + schema_type=schema_type, + statement_config=statement_config, + chunk_size=chunk_size, + native_only=native_only, + **kwargs, + ) + return self.select_stream( + statement, + *parameters, + statement_config=statement_config, + chunk_size=chunk_size, + native_only=native_only, + **kwargs, + ) def _release_replica(self) -> None: if self._replica_stack is not None or self._replica_driver is not None: @@ -1022,6 +1339,7 @@ def _resolve_driver( return driver, sql +@mypyc_attr(allow_interpreted_subclasses=True) class AsyncRoutingSession(_RoutingSessionState, Generic[AsyncDriverT]): """Lazy asynchronous session selecting replicas for provably safe reads.""" @@ -1134,31 +1452,25 @@ async def rollback_to_savepoint(self, name: str) -> None: raise TransactionError(msg) await self._primary_driver.rollback_to_savepoint(name) - @asynccontextmanager - async def transaction(self) -> AsyncIterator[Self]: + def transaction(self) -> AbstractAsyncContextManager[Self]: """Own a primary transaction while preserving nested savepoint behavior. - Yields: - This routing facade with its primary transaction active. + Returns: + Async context manager yielding this routing facade with its primary transaction active. """ - self._primary_only() - driver = await self._acquire_primary() - async with driver.transaction(): - yield self + return cast("AbstractAsyncContextManager[Self]", _AsyncRoutingTransactionContext(self)) - @asynccontextmanager - async def without_indexes( + def without_indexes( self, table: str, *, schema: str | None = None, concurrent: bool = False - ) -> AsyncIterator[Any]: + ) -> AbstractAsyncContextManager[Any]: """Temporarily remove indexes using the primary's native context. - Yields: - The primary driver's index restoration context. + Returns: + Async context manager wrapping the primary driver's index restoration context. """ - self._primary_only() - driver = await self._acquire_primary() - async with driver.without_indexes(table, schema=schema, concurrent=concurrent) as context: - yield context + return cast( + "AbstractAsyncContextManager[Any]", _AsyncRoutingWithoutIndexesContext(self, table, schema, concurrent) + ) async def execute_stack(self, stack: StatementStack, *, continue_on_error: bool = False) -> tuple[StackResult, ...]: """Execute a stack on one connection, pinning any mixed or write stack.""" @@ -1621,14 +1933,333 @@ async def load_from_records( type_casts=type_casts, ) - fetch = select - fetch_one = select_one - fetch_one_or_none = select_one_or_none - fetch_value = select_value - fetch_value_or_none = select_value_or_none - fetch_with_total = select_with_total - fetch_to_arrow = select_to_arrow - fetch_stream = select_stream + @overload + async def fetch( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT]", + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "list[SchemaT]": ... + + @overload + async def fetch( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: None = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "list[dict[str, Any]]": ... + + async def fetch( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT] | None" = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "list[SchemaT] | list[dict[str, Any]]": + """Execute a select statement and return all rows.""" + if schema_type is not None: + return await self.select( + statement, *parameters, schema_type=schema_type, statement_config=statement_config, **kwargs + ) + return await self.select(statement, *parameters, schema_type=None, statement_config=statement_config, **kwargs) + + @overload + async def fetch_one( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT]", + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "SchemaT": ... + + @overload + async def fetch_one( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: None = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "dict[str, Any]": ... + + async def fetch_one( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT] | None" = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "SchemaT | dict[str, Any]": + """Execute a select statement and return exactly one row.""" + if schema_type is not None: + return await self.select_one( + statement, *parameters, schema_type=schema_type, statement_config=statement_config, **kwargs + ) + return await self.select_one( + statement, *parameters, schema_type=None, statement_config=statement_config, **kwargs + ) + + @overload + async def fetch_one_or_none( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT]", + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "SchemaT | None": ... + + @overload + async def fetch_one_or_none( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: None = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "dict[str, Any] | None": ... + + async def fetch_one_or_none( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT] | None" = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "SchemaT | dict[str, Any] | None": + """Execute a select statement and return at most one row.""" + if schema_type is not None: + return await self.select_one_or_none( + statement, *parameters, schema_type=schema_type, statement_config=statement_config, **kwargs + ) + return await self.select_one_or_none( + statement, *parameters, schema_type=None, statement_config=statement_config, **kwargs + ) + + @overload + async def fetch_value( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + value_type: "type[ValueT]", + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "ValueT": ... + + @overload + async def fetch_value( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + value_type: None = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> Any: ... + + async def fetch_value( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + value_type: "type[ValueT] | None" = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "ValueT | Any": + """Execute a select statement and return a single scalar value.""" + if value_type is not None: + return await self.select_value( + statement, *parameters, value_type=value_type, statement_config=statement_config, **kwargs + ) + return await self.select_value(statement, *parameters, statement_config=statement_config, **kwargs) + + @overload + async def fetch_value_or_none( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + value_type: "type[ValueT]", + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "ValueT | None": ... + + @overload + async def fetch_value_or_none( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + value_type: None = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> Any: ... + + async def fetch_value_or_none( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + value_type: "type[ValueT] | None" = None, + statement_config: "StatementConfig | None" = None, + **kwargs: Any, + ) -> "ValueT | Any | None": + """Execute a select statement and return a single scalar value or None.""" + if value_type is not None: + return await self.select_value_or_none( + statement, *parameters, value_type=value_type, statement_config=statement_config, **kwargs + ) + return await self.select_value_or_none(statement, *parameters, statement_config=statement_config, **kwargs) + + @overload + async def fetch_with_total( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT]", + statement_config: "StatementConfig | None" = None, + count_with_window: bool = False, + **kwargs: Any, + ) -> "tuple[list[SchemaT], int]": ... + + @overload + async def fetch_with_total( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: None = None, + statement_config: "StatementConfig | None" = None, + count_with_window: bool = False, + **kwargs: Any, + ) -> "tuple[list[dict[str, Any]], int]": ... + + async def fetch_with_total( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT] | None" = None, + statement_config: "StatementConfig | None" = None, + count_with_window: bool = False, + **kwargs: Any, + ) -> "tuple[list[SchemaT] | list[dict[str, Any]], int]": + """Execute a select statement and return both the data and total count.""" + if schema_type is not None: + return await self.select_with_total( + statement, + *parameters, + schema_type=schema_type, + statement_config=statement_config, + count_with_window=count_with_window, + **kwargs, + ) + return await self.select_with_total( + statement, + *parameters, + schema_type=None, + statement_config=statement_config, + count_with_window=count_with_window, + **kwargs, + ) + + async def fetch_to_arrow( + self, + statement: "Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + statement_config: "StatementConfig | None" = None, + return_format: "ArrowReturnFormat" = "table", + native_only: bool = False, + batch_size: int | None = None, + arrow_schema: Any = None, + **kwargs: Any, + ) -> "ArrowResult": + """Execute query and return results as Apache Arrow format (async).""" + return await self.select_to_arrow( + statement, + *parameters, + statement_config=statement_config, + return_format=return_format, + native_only=native_only, + batch_size=batch_size, + arrow_schema=arrow_schema, + **kwargs, + ) + + @overload + def fetch_stream( + self, + statement: "SQL | Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT]", + statement_config: "StatementConfig | None" = None, + chunk_size: int = 1000, + native_only: bool = False, + **kwargs: Any, + ) -> "AsyncRowStream[SchemaT]": ... + + @overload + def fetch_stream( + self, + statement: "SQL | Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: None = None, + statement_config: "StatementConfig | None" = None, + chunk_size: int = 1000, + native_only: bool = False, + **kwargs: Any, + ) -> "AsyncRowStream[dict[str, Any]]": ... + + def fetch_stream( + self, + statement: "SQL | Statement | QueryBuilder", + /, + *parameters: "StatementParameters | StatementFilter", + schema_type: "type[SchemaT] | None" = None, + statement_config: "StatementConfig | None" = None, + chunk_size: int = 1000, + native_only: bool = False, + **kwargs: Any, + ) -> "AsyncRowStream[SchemaT] | AsyncRowStream[dict[str, Any]]": + """Execute a query and stream rows in chunks.""" + if schema_type is not None: + return self.select_stream( + statement, + *parameters, + schema_type=schema_type, + statement_config=statement_config, + chunk_size=chunk_size, + native_only=native_only, + **kwargs, + ) + return self.select_stream( + statement, + *parameters, + statement_config=statement_config, + chunk_size=chunk_size, + native_only=native_only, + **kwargs, + ) async def _release_replica(self) -> None: if self._replica_stack is not None or self._replica_driver is not None: @@ -1719,15 +2350,24 @@ async def _acquire_replica(self) -> AsyncDriverT: context = _RuntimeReadAsyncContext(context, session=True) lease_stack = AsyncExitStack() + enter_exc_info: tuple[type[BaseException] | None, BaseException | None, TracebackType | None] | None = None + driver = None try: driver = await lease_stack.enter_async_context(context) except BaseException: - try: - if entered_context is not None: - await entered_context._close_on_error(*sys.exc_info()) - finally: - await lease_stack.aclose() - raise + enter_exc_info = sys.exc_info() + if enter_exc_info is not None: + close_error: BaseException | None = None + if entered_context is not None: + try: + await entered_context._close_on_error(*enter_exc_info) + except BaseException as exc: + close_error = exc + await lease_stack.aclose() + if close_error is not None: + raise close_error + if enter_exc_info[1] is not None: + raise enter_exc_info[1] await self._stack.enter_async_context(lease_stack) self._replica_driver = driver self._replica_stack = lease_stack @@ -2192,3 +2832,60 @@ async def _close_on_error( if not self._closed: self._closed = True await self._context.__aexit__(exc_type, exc_val, exc_tb) + + +class _AsyncRoutingTransactionContext: + """Async context manager owning a primary transaction for AsyncRoutingSession.""" + + __slots__ = ("_inner", "_session") + + def __init__(self, session: AsyncRoutingSession[Any]) -> None: + self._session = session + self._inner: AbstractAsyncContextManager[Any] | None = None + + async def __aenter__(self) -> AsyncRoutingSession[Any]: + self._session._primary_only() + driver = await self._session._acquire_primary() + inner = driver.transaction() + await inner.__aenter__() + self._inner = inner + return self._session + + async def __aexit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> bool | None: + inner = self._inner + self._inner = None + if inner is None: + return None + return await inner.__aexit__(exc_type, exc_val, exc_tb) + + +class _AsyncRoutingWithoutIndexesContext: + """Async context manager delegating index teardown to the primary driver.""" + + __slots__ = ("_concurrent", "_inner", "_schema", "_session", "_table") + + def __init__(self, session: AsyncRoutingSession[Any], table: str, schema: str | None, concurrent: bool) -> None: + self._session = session + self._table = table + self._schema = schema + self._concurrent = concurrent + self._inner: AbstractAsyncContextManager[Any] | None = None + + async def __aenter__(self) -> Any: + self._session._primary_only() + driver = await self._session._acquire_primary() + inner = driver.without_indexes(self._table, schema=self._schema, concurrent=self._concurrent) + result = await inner.__aenter__() + self._inner = inner + return result + + async def __aexit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> bool | None: + inner = self._inner + self._inner = None + if inner is None: + return None + return await inner.__aexit__(exc_type, exc_val, exc_tb) diff --git a/sqlspec/driver/_session.py b/sqlspec/driver/_session.py new file mode 100644 index 000000000..d95ecbb35 --- /dev/null +++ b/sqlspec/driver/_session.py @@ -0,0 +1,180 @@ +"""Compiled base session context managers coordinating driver lifecycle.""" + +import contextlib +from typing import TYPE_CHECKING, Any + +from mypy_extensions import mypyc_attr + +if TYPE_CHECKING: + from collections.abc import Callable + from types import TracebackType + + from sqlspec.core.statement import StatementConfig + +__all__ = ("AsyncDriverSessionContext", "SyncDriverSessionContext") + + +@mypyc_attr(allow_interpreted_subclasses=True) +class SyncDriverSessionContext: + """Base sync context manager coordinating connection acquisition, driver creation, and cleanup.""" + + __slots__ = ( + "_acquire_connection", + "_connection", + "_driver", + "_driver_class", + "_driver_features", + "_prepare_driver", + "_release_connection", + "_statement_config", + ) + + def __init__( + self, + acquire_connection: "Callable[[], Any]", + release_connection: "Callable[..., Any]", + statement_config: "StatementConfig | Callable[[], StatementConfig]", + driver_features: "dict[str, Any]", + prepare_driver: "Callable[[Any], Any]", + driver_class: "type[Any] | None" = None, + ) -> None: + self._acquire_connection = acquire_connection + self._release_connection = release_connection + self._statement_config = statement_config + self._driver_features = driver_features + self._prepare_driver = prepare_driver + self._driver_class = driver_class + self._connection: Any = None + self._driver: Any = None + + def _resolve_driver_class(self) -> "type[Any]": + msg = f"{type(self).__name__} must define _resolve_driver_class() or pass driver_class" + raise NotImplementedError(msg) + + def _cleanup_driver(self, driver: Any, *, error: bool) -> None: + driver._close_open_streams(error=error) + + def __enter__(self) -> Any: + driver_cls = self._driver_class + if driver_cls is None: + driver_cls = self._resolve_driver_class() + self._connection = self._acquire_connection() + try: + statement_config = self._statement_config() if callable(self._statement_config) else self._statement_config + self._driver = driver_cls( + connection=self._connection, statement_config=statement_config, driver_features=self._driver_features + ) + return self._prepare_driver(self._driver) + except BaseException as exc: + with contextlib.suppress(Exception): + self.__exit__(type(exc), exc, exc.__traceback__) + raise + + def __exit__( + self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None" + ) -> "bool | None": + driver = self._driver + try: + if driver is not None: + self._cleanup_driver(driver, error=exc_type is not None) + except BaseException as close_exc: + if exc_type is None: + exc_type, exc_val, exc_tb = type(close_exc), close_exc, close_exc.__traceback__ + raise + finally: + try: + if self._connection is not None: + self._release_connection(self._connection, exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) + self._connection = None + finally: + self._driver = None + if driver is not None: + driver._mark_closed() + return None + + +@mypyc_attr(allow_interpreted_subclasses=True) +class AsyncDriverSessionContext: + """Base async context manager coordinating connection acquisition, driver creation, and cleanup.""" + + __slots__ = ( + "_acquire_connection", + "_connection", + "_driver", + "_driver_class", + "_driver_features", + "_prepare_driver", + "_release_connection", + "_statement_config", + ) + + def __init__( + self, + acquire_connection: "Callable[[], Any]", + release_connection: "Callable[..., Any]", + statement_config: "StatementConfig | Callable[[], StatementConfig]", + driver_features: "dict[str, Any]", + prepare_driver: "Callable[[Any], Any]", + driver_class: "type[Any] | None" = None, + ) -> None: + self._acquire_connection = acquire_connection + self._release_connection = release_connection + self._statement_config = statement_config + self._driver_features = driver_features + self._prepare_driver = prepare_driver + self._driver_class = driver_class + self._connection: Any = None + self._driver: Any = None + + def _resolve_driver_class(self) -> "type[Any]": + msg = f"{type(self).__name__} must define _resolve_driver_class() or pass driver_class" + raise NotImplementedError(msg) + + async def __aenter__(self) -> Any: + driver_cls = self._driver_class + if driver_cls is None: + driver_cls = self._resolve_driver_class() + self._connection = await self._acquire_connection() + enter_error: BaseException | None = None + try: + statement_config = self._statement_config() if callable(self._statement_config) else self._statement_config + self._driver = driver_cls( + connection=self._connection, statement_config=statement_config, driver_features=self._driver_features + ) + return self._prepare_driver(self._driver) + except BaseException as exc: + enter_error = exc + with contextlib.suppress(Exception): + await self.__aexit__(type(enter_error), enter_error, enter_error.__traceback__) + raise enter_error + + async def __aexit__( + self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None" + ) -> "bool | None": + driver = self._driver + close_error: BaseException | None = None + release_error: BaseException | None = None + if driver is not None: + try: + await driver._close_open_streams(error=exc_type is not None) + except BaseException as close_exc: + close_error = close_exc + if exc_type is None: + exc_type = type(close_exc) + exc_val = close_exc + exc_tb = close_exc.__traceback__ + if self._connection is not None: + connection = self._connection + self._connection = None + try: + await self._release_connection(connection, exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb) + except BaseException as rel_exc: + release_error = rel_exc + self._driver = None + if driver is not None: + driver._mark_closed() + if close_error is not None: + raise close_error + if release_error is not None: + raise release_error + return None diff --git a/sqlspec/driver/_sql_helpers.py b/sqlspec/driver/_sql_helpers.py index 29db96bbf..efba95c92 100644 --- a/sqlspec/driver/_sql_helpers.py +++ b/sqlspec/driver/_sql_helpers.py @@ -8,13 +8,13 @@ from sqlglot import exp, parse_one -from sqlspec.core import SQL +from sqlspec.core.statement import SQL from sqlspec.exceptions import SQLConversionError if TYPE_CHECKING: from sqlglot.dialects.dialect import DialectType - from sqlspec.core import Statement + from sqlspec.core.statement import Statement __all__ = ( @@ -119,7 +119,9 @@ def raise_parse_error(e: Exception) -> "NoReturn": SQLConversionError: Always raised. """ error_msg = f"Failed to parse SQL statement: {e!s}" - raise SQLConversionError(error_msg) from e + error = SQLConversionError(error_msg) + error.__cause__ = e + raise error from e def raise_conversion_error(dialect: "DialectType | None", e: Exception) -> "NoReturn": @@ -133,4 +135,6 @@ def raise_conversion_error(dialect: "DialectType | None", e: Exception) -> "NoRe SQLConversionError: Always raised. """ error_msg = f"Failed to convert SQL expression to {dialect}: {e!s}" - raise SQLConversionError(error_msg) from e + error = SQLConversionError(error_msg) + error.__cause__ = e + raise error from e diff --git a/sqlspec/driver/_stream.py b/sqlspec/driver/_stream.py index 095571ce8..2222c5774 100644 --- a/sqlspec/driver/_stream.py +++ b/sqlspec/driver/_stream.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from types import TracebackType - from sqlspec.core import SQL + from sqlspec.core.statement import SQL __all__ = ( "AsyncRowSource", diff --git a/sqlspec/driver/_sync.py b/sqlspec/driver/_sync.py index 7c85abfef..e43ff5ed4 100644 --- a/sqlspec/driver/_sync.py +++ b/sqlspec/driver/_sync.py @@ -15,23 +15,26 @@ from mypy_extensions import mypyc_attr -from sqlspec.core import SQL, CachedQuery, StackResult, create_arrow_result -from sqlspec.core.result import DMLResult +from sqlspec.core.cache import CachedQuery +from sqlspec.core.result._base import DMLResult, StackResult, create_arrow_result from sqlspec.core.stack import StackOperation, StatementStack -from sqlspec.data_dictionary import ( +from sqlspec.core.statement import SQL +from sqlspec.data_dictionary._capabilities import ( DEFAULT_SYSTEM_METADATA_DOMAINS, + ensure_system_metadata_request, + system_metadata_capabilities_from_domains, + system_metadata_gated_result, + unsupported_system_metadata_capability, +) +from sqlspec.data_dictionary._index_ddl import SyncWithoutIndexesContext +from sqlspec.data_dictionary._types import ( ColumnProfileMetadata, DDLResult, MetadataCapabilityProfile, MetadataResult, MetadataSupport, ObjectIdentity, - ensure_system_metadata_request, - system_metadata_capabilities_from_domains, - system_metadata_gated_result, - unsupported_system_metadata_capability, ) -from sqlspec.data_dictionary._index_ddl import SyncWithoutIndexesContext from sqlspec.driver._common import ( _CLOSED_CONNECTION, CommonDriverAttributesMixin, @@ -77,9 +80,12 @@ from sqlglot.dialects.dialect import DialectType - from sqlspec.builder import QueryBuilder - from sqlspec.core import ArrowResult, OperationType, SQLResult, Statement, StatementConfig, StatementFilter - from sqlspec.data_dictionary import ( + from sqlspec.builder._base import QueryBuilder + from sqlspec.core.compiler import OperationType + from sqlspec.core.filters import StatementFilter + from sqlspec.core.result._base import ArrowResult, SQLResult + from sqlspec.core.statement import Statement, StatementConfig + from sqlspec.data_dictionary._types import ( ColumnMetadata, ForeignKeyMetadata, IndexMetadata, @@ -530,18 +536,18 @@ def execute( **kwargs: Any, ) -> "SQLResult": """Execute a statement with parameter handling.""" - fast_params = _resolve_fast_params(parameters, kwargs) if ( self._stmt_cache_enabled and (statement_config is None or statement_config is self.statement_config) and isinstance(statement, str) - and fast_params is not None ): - if self._auto_begin_on_write: - self._ensure_transaction_for_cached_sql(statement, fast_params) - fast_result = run_cached_execution(self, statement, fast_params) - if fast_result is not None: - return cast("SQLResult", fast_result) + fast_params = _resolve_fast_params(parameters, kwargs) + if fast_params is not None: + if self._auto_begin_on_write: + self._ensure_transaction_for_cached_sql(statement, fast_params) + fast_result = run_cached_execution(self, statement, fast_params) + if fast_result is not None: + return cast("SQLResult", fast_result) sql_statement = self.prepare_statement( statement, parameters, statement_config=statement_config or self.statement_config, kwargs=kwargs ) @@ -1428,6 +1434,7 @@ def execute_stack(self, stack: "StatementStack", *, continue_on_error: bool = Fa results.append(StackResult.from_error(stack_error)) continue + stack_error.__cause__ = exc raise stack_error from exc results.append(StackResult(result=result)) @@ -1707,6 +1714,7 @@ def _execute_with_observability_sync( is_many: bool, is_script: bool, runner: "Callable[..., SQLResult]", + precomputed_sql_hash: "str | None" = None, **runner_kwargs: Any, ) -> "SQLResult": """Execute a sync runner while emitting spans, lifecycle hooks, and statement events.""" @@ -1731,9 +1739,10 @@ def _execute_with_observability_sync( if db_system is not None: query_context["db_system"] = db_system runtime.emit_query_start_sync(**query_context) - sql_hash = None - if runtime.span_manager.is_enabled or ( - runtime.has_statement_observers and runtime.config.logging and runtime.config.logging.include_sql_hash + sql_hash = precomputed_sql_hash if not is_script else None + if sql_hash is None and ( + runtime.span_manager.is_enabled + or (runtime.has_statement_observers and runtime.config.logging and runtime.config.logging.include_sql_hash) ): sql_hash = observability_runtime.compute_sql_hash(observed_sql) span = runtime.start_query_span(observed_sql, operation, driver_name, sql_hash=sql_hash, db_system=db_system) @@ -1849,13 +1858,21 @@ def _execute_cache_hit_observed( ) -> "SQLResult": """Execute a fast-path cache hit wrapped in observability instrumentation.""" obs_params = list(params) if isinstance(params, tuple) else params + wire_sql = self._format_wire_sql(cached.compiled_sql) + sql_hash: str | None = None + if not self.statement_config.enable_sqlcommenter: + sql_hash = cached._sql_hash + if sql_hash is None: + sql_hash = observability_runtime.compute_sql_hash(wire_sql) + cached._sql_hash = sql_hash return self._execute_with_observability_sync( - compiled_sql=self._format_wire_sql(cached.compiled_sql), + compiled_sql=wire_sql, execution_parameters=obs_params, operation=cached.operation_type, is_many=False, is_script=False, runner=self._execute_cache_hit, + precomputed_sql_hash=sql_hash, sql=sql, params=params, cached=cached, @@ -1965,6 +1982,8 @@ def _execute_cache_hit_with_cursor( if returns_rows: assert fetchall is not None fetched_data = fetchall() + if isinstance(fetched_data, tuple): + fetched_data = list(fetched_data) data, column_names, row_count = self.collect_rows(cursor, fetched_data) result = self.build_cached_select_result( sql, diff --git a/sqlspec/exceptions.py b/sqlspec/exceptions.py index 200bc504e..e4d79b0cc 100644 --- a/sqlspec/exceptions.py +++ b/sqlspec/exceptions.py @@ -1,6 +1,6 @@ """Exception hierarchy and SQLSTATE error mapping for SQLSpec.""" -from typing import Any, Final +from typing import Any, Final, TypeVar __all__ = ( "SQLSTATE_EXCEPTION_MAP", @@ -53,8 +53,11 @@ "TransactionError", "UniqueViolationError", "map_sqlstate_to_exception", + "with_cause", ) +ExceptionT = TypeVar("ExceptionT", bound=BaseException) + STACK_SQL_PREVIEW_LIMIT: Final[int] = 120 @@ -509,6 +512,24 @@ class SquashValidationError(MigrationError): } +def with_cause(error: ExceptionT, cause: "BaseException | None") -> ExceptionT: + """Attach ``cause`` to ``error`` as its direct cause and return ``error``. + + ``raise with_cause(error, cause) from cause`` sets ``__cause__`` on the exception itself + instead of relying on the ``raise`` statement to apply its ``from`` clause. Passing + ``None`` suppresses the implicit context, as ``from None`` does. + + Args: + error: Exception about to be raised. + cause: Exception that caused ``error``, or ``None`` to suppress the implicit context. + + Returns: + ``error``, with ``__cause__`` set to ``cause``. + """ + error.__cause__ = cause + return error + + def map_sqlstate_to_exception(sqlstate: str | None) -> type[SQLSpecError] | None: """Map a SQLSTATE code to a SQLSpec exception class. diff --git a/sqlspec/extensions/adk/service.py b/sqlspec/extensions/adk/service.py index 67b314511..99034e717 100644 --- a/sqlspec/extensions/adk/service.py +++ b/sqlspec/extensions/adk/service.py @@ -3,7 +3,7 @@ import asyncio import inspect import logging -from collections.abc import AsyncIterator +from collections.abc import AsyncGenerator from contextlib import asynccontextmanager, nullcontext from datetime import UTC, datetime from typing import TYPE_CHECKING, Any, Self, cast @@ -417,7 +417,7 @@ async def append_event(self, session: "Session", event: "Event") -> "Event": return trimmed_event @asynccontextmanager - async def _with_session_lock(self, *, app_name: str, user_id: str, session_id: str) -> AsyncIterator[None]: + async def _with_session_lock(self, *, app_name: str, user_id: str, session_id: str) -> AsyncGenerator[None, None]: """Serialize in-process concurrent mutations for the same session.""" lock_key = (app_name, user_id, session_id) lock = self._session_locks.get(lock_key) diff --git a/sqlspec/extensions/adk/store.py b/sqlspec/extensions/adk/store.py index fb682288e..76a2b27f7 100644 --- a/sqlspec/extensions/adk/store.py +++ b/sqlspec/extensions/adk/store.py @@ -3,7 +3,7 @@ import inspect import logging from abc import ABC, abstractmethod -from collections.abc import Iterator +from collections.abc import Generator from contextlib import AbstractAsyncContextManager, AbstractContextManager, contextmanager from contextvars import ContextVar from datetime import UTC, datetime, timedelta @@ -48,7 +48,7 @@ @contextmanager -def adk_primary_read_scope() -> Iterator[None]: +def adk_primary_read_scope() -> Generator[None, None, None]: """Keep reads inside a mutating ADK operation on the primary. Yields: diff --git a/sqlspec/extensions/events/_channel.py b/sqlspec/extensions/events/_channel.py index 43199b8b5..1ec2a1eaa 100644 --- a/sqlspec/extensions/events/_channel.py +++ b/sqlspec/extensions/events/_channel.py @@ -24,10 +24,12 @@ from sqlspec.utils.uuids import uuid4 if TYPE_CHECKING: + from collections.abc import Generator + from sqlspec.config import AsyncDatabaseConfig, SyncDatabaseConfig from sqlspec.extensions.events._hints import EventRuntimeHints from sqlspec.extensions.events._protocols import AsyncEventHandler, SyncEventHandler - from sqlspec.observability import ObservabilityRuntime + from sqlspec.observability._runtime import ObservabilityRuntime __all__ = ( "AsyncEventChannel", @@ -963,7 +965,7 @@ def _event_span( *, mode: str = "sync", result: str, -) -> "Iterator[Any]": +) -> "Generator[Any, None, None]": """Manage an observability span around one event operation. Starts a span, ends it with ``error`` when the wrapped operation raises, diff --git a/sqlspec/extensions/events/_payload.py b/sqlspec/extensions/events/_payload.py index fee224c42..141769fcb 100644 --- a/sqlspec/extensions/events/_payload.py +++ b/sqlspec/extensions/events/_payload.py @@ -4,9 +4,9 @@ from datetime import UTC, datetime from typing import Any -from sqlspec.exceptions import EventChannelError +from sqlspec.exceptions import EventChannelError, with_cause from sqlspec.extensions.events._models import EventMessage -from sqlspec.utils.serializers import from_json, to_json +from sqlspec.utils.serializers._json import from_json, to_json from sqlspec.utils.uuids import uuid4 __all__ = ( @@ -80,7 +80,7 @@ def decode_notify_payload(channel: str, payload: str) -> "EventMessage": raw = from_json(payload) except (TypeError, ValueError) as exc: msg = "Malformed event notification payload" - raise EventChannelError(msg) from exc + raise with_cause(EventChannelError(msg), exc) from exc data = raw if isinstance(raw, dict) else {"payload": raw} if "event_id" in data: raw_event_id = data["event_id"] diff --git a/sqlspec/extensions/events/_queue.py b/sqlspec/extensions/events/_queue.py index a50f2900d..4649542c5 100644 --- a/sqlspec/extensions/events/_queue.py +++ b/sqlspec/extensions/events/_queue.py @@ -9,15 +9,15 @@ from functools import partial from typing import TYPE_CHECKING, Any, ClassVar, cast, final -from sqlspec.core import SQL, StatementConfig -from sqlspec.exceptions import ImproperConfigurationError +from sqlspec.core.statement import SQL, StatementConfig +from sqlspec.exceptions import EventChannelError, ImproperConfigurationError, with_cause from sqlspec.extensions.events._hints import EventRuntimeHints, get_runtime_hints, resolve_adapter_name from sqlspec.extensions.events._models import EventMessage from sqlspec.extensions.events._names import normalize_queue_table_name from sqlspec.extensions.events._payload import coerce_dict, coerce_optional_dict, parse_event_timestamp from sqlspec.extensions.events.primitives import claim_verified, lock_clause, row_limit_clause, select_limit_prefix from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json +from sqlspec.utils.serializers._json import from_json from sqlspec.utils.sync_tools import exit_async_context from sqlspec.utils.uuids import uuid4 @@ -27,7 +27,8 @@ from threading import Event from sqlspec.config import DatabaseConfigProtocol - from sqlspec.driver import AsyncDriverAdapterBase, SyncDriverAdapterBase + from sqlspec.driver._async import AsyncDriverAdapterBase + from sqlspec.driver._sync import SyncDriverAdapterBase __all__ = ("AsyncTableEventQueue", "SyncTableEventQueue", "build_queue_backend") @@ -507,7 +508,7 @@ def dequeue( claimed = self._claim_verified(self._fetch_by_event_id(row["event_id"]), leased_until) if claimed: self._reset_empty_poll_delay(channel) - return self._claimed_event(row, leased_until) + return self._hydrate_claimed_event(row, leased_until) return None def dequeue_by_event_id(self, event_id: str, channel: str) -> "EventMessage | None": @@ -521,11 +522,24 @@ def dequeue_by_event_id(self, event_id: str, channel: str) -> "EventMessage | No if not claimed: claimed = self._claim_verified(self._fetch_by_event_id(row["event_id"]), leased_until) if claimed: - event = self._claimed_event(row, leased_until) + event = self._hydrate_claimed_event(row, leased_until) self._reset_empty_poll_delay(event.channel) return event return None + def _hydrate_claimed_event(self, row: "dict[str, Any]", leased_until: "datetime") -> EventMessage: + try: + return self._claimed_event(row, leased_until) + except (EventChannelError, ValueError, TypeError, LookupError) as exc: + event_id = row.get("event_id") + channel = row.get("channel") + if isinstance(event_id, str) and isinstance(channel, str): + self._record_lease(event_id, channel, leased_until) + with suppress(Exception): + self.nack(event_id) + msg = f"Failed to decode claimed event {event_id!r}" + raise with_cause(EventChannelError(msg), exc) from exc + def ack(self, event_id: str) -> bool: lease = self._take_lease(event_id) row = self._owned_row(self._fetch_by_event_id(event_id), lease) if lease is not None else None @@ -635,6 +649,8 @@ def _execute(self, sql: str, parameters: "dict[str, Any]") -> int: def _claim_locked_candidate(self, channel: str) -> "EventMessage | None": current_time = self._utcnow() + claimed_row: dict[str, Any] | None = None + claimed_leased_until: datetime | None = None with cast( "AbstractContextManager[SyncDriverAdapterBase]", self._config.provide_session(transaction=True) ) as driver: @@ -662,11 +678,15 @@ def _claim_locked_candidate(self, channel: str) -> "EventMessage | None": driver.rollback() return None driver.commit() - return self._claimed_event(row, leased_until) + claimed_row = row + claimed_leased_until = leased_until except Exception: with suppress(Exception): driver.rollback() raise + if claimed_row is None or claimed_leased_until is None: + return None + return self._hydrate_claimed_event(claimed_row, claimed_leased_until) def _fetch_candidate_with_driver( self, driver: "SyncDriverAdapterBase", channel: str, current_time: "datetime" @@ -802,7 +822,7 @@ async def dequeue(self, channel: str, poll_interval: float | None = None) -> "Ev claimed = self._claim_verified(await self._fetch_by_event_id(row["event_id"]), leased_until) if claimed: self._reset_empty_poll_delay(channel) - return self._claimed_event(row, leased_until) + return await self._hydrate_claimed_event(row, leased_until) return None async def dequeue_by_event_id(self, event_id: str, channel: str) -> "EventMessage | None": @@ -816,11 +836,26 @@ async def dequeue_by_event_id(self, event_id: str, channel: str) -> "EventMessag if not claimed: claimed = self._claim_verified(await self._fetch_by_event_id(row["event_id"]), leased_until) if claimed: - event = self._claimed_event(row, leased_until) + event = await self._hydrate_claimed_event(row, leased_until) self._reset_empty_poll_delay(event.channel) return event return None + async def _hydrate_claimed_event(self, row: "dict[str, Any]", leased_until: "datetime") -> EventMessage: + decode_error: Exception | None = None + try: + return self._claimed_event(row, leased_until) + except (EventChannelError, ValueError, TypeError, LookupError) as exc: + decode_error = exc + event_id = row.get("event_id") + channel = row.get("channel") + if isinstance(event_id, str) and isinstance(channel, str): + self._record_lease(event_id, channel, leased_until) + with suppress(Exception): + await self.nack(event_id) + msg = f"Failed to decode claimed event {event_id!r}" + raise with_cause(EventChannelError(msg), decode_error) from decode_error + async def ack(self, event_id: str) -> bool: lease = self._take_lease(event_id) row = self._owned_row(await self._fetch_by_event_id(event_id), lease) if lease is not None else None @@ -954,20 +989,22 @@ async def _claim_locked_candidate(self, channel: str) -> "EventMessage | None": ) driver = await session.__aenter__() claim_error: BaseException | None = None - event: EventMessage | None = None + claimed_pair: tuple[dict[str, Any], datetime] | None = None try: - event = await self._claim_candidate_with_driver(driver, channel, current_time) + claimed_pair = await self._claim_candidate_with_driver(driver, channel, current_time) except BaseException as exc: claim_error = exc if isinstance(claim_error, Exception): with suppress(Exception): await driver.rollback() await exit_async_context(session, claim_error) - return event + if claimed_pair is None: + return None + return await self._hydrate_claimed_event(claimed_pair[0], claimed_pair[1]) async def _claim_candidate_with_driver( self, driver: "AsyncDriverAdapterBase", channel: str, current_time: "datetime" - ) -> "EventMessage | None": + ) -> "tuple[dict[str, Any], datetime] | None": """Select, claim and commit one candidate on the given session, or roll back when none is claimed.""" row = await self._fetch_candidate_with_driver(driver, channel, current_time) if row is None: @@ -990,7 +1027,7 @@ async def _claim_candidate_with_driver( await driver.rollback() return None await driver.commit() - return self._claimed_event(row, leased_until) + return row, leased_until async def _fetch_candidate_with_driver( self, driver: "AsyncDriverAdapterBase", channel: str, current_time: "datetime" diff --git a/sqlspec/extensions/fastapi/extension.py b/sqlspec/extensions/fastapi/extension.py index 64c44f320..0c4082105 100644 --- a/sqlspec/extensions/fastapi/extension.py +++ b/sqlspec/extensions/fastapi/extension.py @@ -9,6 +9,7 @@ from sqlspec.extensions.fastapi.providers import provide_filters as _provide_filters from sqlspec.extensions.starlette.extension import DEFAULT_COMMIT_MODE from sqlspec.extensions.starlette.extension import SQLSpecPlugin as _StarlettePlugin +from sqlspec.utils.config_tools import resolve_extension_commit_mode if TYPE_CHECKING: from collections.abc import Awaitable, Callable @@ -342,7 +343,7 @@ def _extract_extension_settings(self, config: Any) -> "dict[str, Any]": "routing_header": framework_config.get("routing_header"), "primary_pin_cookie": framework_config.get("primary_pin_cookie"), "primary_pin_secret": framework_config.get("primary_pin_secret"), - "commit_mode": framework_config.get("commit_mode", DEFAULT_COMMIT_MODE), + "commit_mode": resolve_extension_commit_mode(config, framework_config, DEFAULT_COMMIT_MODE), "extra_commit_statuses": framework_config.get("extra_commit_statuses"), "extra_rollback_statuses": framework_config.get("extra_rollback_statuses"), "disable_di": framework_config.get("disable_di", False), diff --git a/sqlspec/extensions/fastapi/providers.py b/sqlspec/extensions/fastapi/providers.py index 818fe2b5a..cc6b3216a 100644 --- a/sqlspec/extensions/fastapi/providers.py +++ b/sqlspec/extensions/fastapi/providers.py @@ -22,7 +22,8 @@ from fastapi.exceptions import RequestValidationError from typing_extensions import TypedDict -from sqlspec.core import ( +from sqlspec.core._cursor import resolve_env_pagination_cursor_secret +from sqlspec.core.filters import ( BeforeAfterFilter, BooleanFilter, ChoicesFilter, @@ -39,9 +40,8 @@ SearchFilter, normalize_cursor_keys, ) -from sqlspec.core._cursor import resolve_env_pagination_cursor_secret -from sqlspec.exceptions import ImproperConfigurationError, InvalidCursorError -from sqlspec.utils.tenant import resolve_cursor_tenant_identity +from sqlspec.exceptions import ImproperConfigurationError, InvalidCursorError, with_cause +from sqlspec.utils.tenant._cursor import resolve_cursor_tenant_identity from sqlspec.utils.text import camelize if TYPE_CHECKING: @@ -753,8 +753,11 @@ async def __call__( keys, resolved_page_size, cursor, derived_secret, ttl=resolved_ttl, encrypted=resolved_encrypted ) except InvalidCursorError as exc: - raise RequestValidationError( - errors=[{"loc": ("query", "cursor"), "msg": "Invalid pagination cursor", "type": "value_error"}] + raise with_cause( + RequestValidationError( + errors=[{"loc": ("query", "cursor"), "msg": "Invalid pagination cursor", "type": "value_error"}] + ), + exc, ) from exc def __deepcopy__(self, memo: dict[int, Any]) -> "_CursorFilterProvider": diff --git a/sqlspec/extensions/flask/_portal.py b/sqlspec/extensions/flask/_portal.py index fa19b3ab2..e81d82515 100644 --- a/sqlspec/extensions/flask/_portal.py +++ b/sqlspec/extensions/flask/_portal.py @@ -10,10 +10,10 @@ from contextvars import copy_context from typing import TYPE_CHECKING, Any, Literal -from sqlspec.driver import AsyncDriverAdapterBase +from sqlspec.driver._async import AsyncDriverAdapterBase from sqlspec.driver._routing import AsyncRoutingSession from sqlspec.utils.routing import RoutingContext, RoutingContextPayload -from sqlspec.utils.tenant import TenantContext, TenantContextPayload, TenantRequestBinding +from sqlspec.utils.tenant._context import TenantContext, TenantContextPayload, TenantRequestBinding if TYPE_CHECKING: from sqlspec.utils.portal import Portal diff --git a/sqlspec/extensions/flask/_state.py b/sqlspec/extensions/flask/_state.py index 1363006fc..027434e5b 100644 --- a/sqlspec/extensions/flask/_state.py +++ b/sqlspec/extensions/flask/_state.py @@ -6,9 +6,9 @@ from typing import Any, Final, Literal from sqlspec.config import DatabaseConfigProtocol -from sqlspec.exceptions import ImproperConfigurationError +from sqlspec.exceptions import ImproperConfigurationError, with_cause from sqlspec.utils.routing import validate_routing_middleware_settings -from sqlspec.utils.tenant import TenantExtractorValue +from sqlspec.utils.tenant._context import TenantExtractorValue __all__ = ("FlaskConfigState",) @@ -36,8 +36,8 @@ class FlaskConfigState: connection_key: str session_key: str commit_mode: Literal["manual", "autocommit", "autocommit_include_redirect"] - extra_commit_statuses: "set[int] | None" - extra_rollback_statuses: "set[int] | None" + extra_commit_statuses: set[int] | None + extra_rollback_statuses: set[int] | None is_async: bool disable_di: bool tenant_extractor: Callable[..., TenantExtractorValue] | None = None @@ -45,7 +45,7 @@ class FlaskConfigState: extractor_takes_request: bool = field(default=False, init=False) enable_correlation_middleware: bool = True correlation_header: str = "x-request-id" - correlation_headers: "tuple[str, ...] | None" = None + correlation_headers: tuple[str, ...] | None = None auto_trace_headers: bool = True enable_sqlcommenter_middleware: bool = True replica_session_key: str = "db_replica_session" @@ -96,7 +96,7 @@ def __post_init__(self) -> None: extractor_signature.bind(object()) except TypeError as exc: msg = "Flask tenant_extractor must accept zero arguments or one request" - raise ImproperConfigurationError(msg) from exc + raise with_cause(ImproperConfigurationError(msg), exc) from exc self.extractor_takes_request = True def should_commit(self, status_code: int) -> bool: diff --git a/sqlspec/extensions/flask/_utils.py b/sqlspec/extensions/flask/_utils.py index 56d72d24d..66425d90c 100644 --- a/sqlspec/extensions/flask/_utils.py +++ b/sqlspec/extensions/flask/_utils.py @@ -8,7 +8,7 @@ from sqlspec.extensions.flask._state import tenant_receipt_key from sqlspec.utils.adapters import resolve_adapter_name from sqlspec.utils.deprecation import deprecated -from sqlspec.utils.tenant import TenantContextPayload +from sqlspec.utils.tenant._context import TenantContextPayload if TYPE_CHECKING: from sqlspec.extensions.flask._state import FlaskConfigState diff --git a/sqlspec/extensions/flask/extension.py b/sqlspec/extensions/flask/extension.py index c3463f43a..f80e5aa45 100644 --- a/sqlspec/extensions/flask/extension.py +++ b/sqlspec/extensions/flask/extension.py @@ -22,10 +22,17 @@ set_context_value, set_request_tenant, ) +from sqlspec.utils.config_tools import normalize_extension_status_codes, resolve_extension_commit_mode from sqlspec.utils.correlation import CorrelationContext from sqlspec.utils.logging import get_logger, log_with_context from sqlspec.utils.portal import PortalProvider -from sqlspec.utils.routing import RoutingContext, RoutingContextPayload, RoutingEntry, apply_request_routing_hints +from sqlspec.utils.routing import ( + RoutingContext, + RoutingContextPayload, + RoutingEntry, + apply_request_routing_hints, + validate_routing_middleware_settings, +) from sqlspec.utils.tenant import ( TenantContext, TenantContextPayload, @@ -413,23 +420,29 @@ def _config_state(self, config: Any) -> FlaskConfigState: session_key = flask_config.get("session_key", DEFAULT_SESSION_KEY) connection_key = flask_config.get("connection_key", f"sqlspec_connection_{session_key}") - commit_mode = flask_config.get("commit_mode", DEFAULT_COMMIT_MODE) - extra_commit_statuses = flask_config.get("extra_commit_statuses") - extra_rollback_statuses = flask_config.get("extra_rollback_statuses") - disable_di = flask_config.get("disable_di", False) + commit_mode = resolve_extension_commit_mode(config, flask_config, DEFAULT_COMMIT_MODE) + extra_commit_statuses = normalize_extension_status_codes(flask_config.get("extra_commit_statuses")) + extra_rollback_statuses = normalize_extension_status_codes(flask_config.get("extra_rollback_statuses")) + disable_di = bool(flask_config.get("disable_di", False)) - enable_correlation = flask_config.get("enable_correlation_middleware", True) + enable_correlation = bool(flask_config.get("enable_correlation_middleware", True)) correlation_header = flask_config.get("correlation_header", "x-request-id") correlation_headers = flask_config.get("correlation_headers") if correlation_headers is not None: correlation_headers = tuple(correlation_headers) - auto_trace_headers = flask_config.get("auto_trace_headers", True) - enable_sqlcommenter = flask_config.get("enable_sqlcommenter_middleware", True) + auto_trace_headers = bool(flask_config.get("auto_trace_headers", True)) + enable_sqlcommenter = bool(flask_config.get("enable_sqlcommenter_middleware", True)) is_async = isinstance(config, (AsyncDatabaseConfig, NoPoolAsyncConfig)) if flask_config.get("tenant_extractor") is not None and not _has_tenant_strategy(config): msg = "tenant_extractor requires driver_features['tenant_strategy'] on the same configuration" raise ImproperConfigurationError(msg) + enable_routing, routing_header, primary_pin_cookie, primary_pin_secret = validate_routing_middleware_settings( + enable_routing_middleware=flask_config.get("enable_routing_middleware", True), + routing_header=flask_config.get("routing_header"), + primary_pin_cookie=flask_config.get("primary_pin_cookie"), + primary_pin_secret=flask_config.get("primary_pin_secret"), + ) return FlaskConfigState( config=config, @@ -458,10 +471,10 @@ def _config_state(self, config: Any) -> FlaskConfigState: ), routing_session_key=flask_config.get("routing_session_key"), default_routing_mode=flask_config.get("default_routing_mode", "auto"), - enable_routing_middleware=flask_config.get("enable_routing_middleware", True), - routing_header=flask_config.get("routing_header"), - primary_pin_cookie=flask_config.get("primary_pin_cookie"), - primary_pin_secret=flask_config.get("primary_pin_secret"), + enable_routing_middleware=enable_routing, + routing_header=routing_header, + primary_pin_cookie=primary_pin_cookie, + primary_pin_secret=primary_pin_secret, ) def _ensure_unique_keys(self) -> None: @@ -999,12 +1012,15 @@ def _execute_commit(self, session: Any, config_state: FlaskConfigState, connecti ): return try: + target: Any = None if session is not None and hasattr(session, "commit"): target = session elif connection is not None: target = connection else: target = self.get_connection(config_state.session_key) + if target is None or not hasattr(target, "commit"): + return if config_state.is_async: self._call_portal(config_state, target.commit) else: @@ -1038,12 +1054,15 @@ def _execute_rollback(self, session: Any, config_state: FlaskConfigState, connec ): return try: + target: Any = None if session is not None and hasattr(session, "rollback"): target = session elif connection is not None: target = connection else: target = self.get_connection(config_state.session_key) + if target is None or not hasattr(target, "rollback"): + return if config_state.is_async: self._call_portal(config_state, target.rollback) else: diff --git a/sqlspec/extensions/litestar/_utils.py b/sqlspec/extensions/litestar/_utils.py index e5728fd40..0172a2c7a 100644 --- a/sqlspec/extensions/litestar/_utils.py +++ b/sqlspec/extensions/litestar/_utils.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Final, NamedTuple if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import Generator from litestar.types import Scope @@ -167,7 +167,7 @@ def _current_task() -> "asyncio.Task[Any] | None": @contextmanager -def request_session_scope() -> "Iterator[_RequestSessionHolder]": +def request_session_scope() -> "Generator[_RequestSessionHolder, None, None]": """Install a request session holder owned by the current task for the enclosed request. Yields: diff --git a/sqlspec/extensions/litestar/plugin.py b/sqlspec/extensions/litestar/plugin.py index e30146597..2c3763cb6 100644 --- a/sqlspec/extensions/litestar/plugin.py +++ b/sqlspec/extensions/litestar/plugin.py @@ -73,6 +73,7 @@ tenant_receipt_key, ) from sqlspec.typing import NUMPY_INSTALLED, ConnectionT, PoolT, SchemaT, import_optional_attr +from sqlspec.utils.config_tools import resolve_extension_commit_mode from sqlspec.utils.correlation import CorrelationContext from sqlspec.utils.deprecation import deprecated from sqlspec.utils.logging import get_logger, log_with_context @@ -1511,7 +1512,7 @@ def _extract_extension_settings(self, config: "AnyDatabaseConfig") -> "dict[str, connection_key = litestar_config.get("connection_key", DEFAULT_CONNECTION_KEY) pool_key = litestar_config.get("pool_key", DEFAULT_POOL_KEY) session_key = litestar_config.get("session_key", DEFAULT_SESSION_KEY) - commit_mode = litestar_config.get("commit_mode", DEFAULT_COMMIT_MODE) + commit_mode = resolve_extension_commit_mode(config, litestar_config, DEFAULT_COMMIT_MODE) if not config.supports_connection_pooling and pool_key == DEFAULT_POOL_KEY: pool_key = f"_{DEFAULT_POOL_KEY}_{id(config)}" @@ -1807,11 +1808,17 @@ def _get_plugin_state( }: return state + matches: list[PluginConfigState] = [] for state in self._plugin_configs: if state.annotation is None: self._raise_plugin_not_registered() if key in {state.config, state.annotation}: - return state + matches.append(state) + + if len(matches) > 1 and isinstance(key, type): + self._raise_ambiguous_config_type(key, matches) + if matches: + return matches[0] self._raise_config_not_found(key) msg = "unreachable" @@ -1886,7 +1893,7 @@ def _raise_ambiguous_config_type(self, config_type: type, candidates: "list[Plug bind_keys = [state.config.bind_key or "" for state in candidates] msg = ( f"Multiple database configurations use the type '{config_type.__name__}'. " - f"Look one up by its bind key instead. Candidate bind keys: {bind_keys}" + f"Pass the configuration instance instead. Candidate bind keys: {bind_keys}" ) raise KeyError(msg) diff --git a/sqlspec/extensions/litestar/providers.py b/sqlspec/extensions/litestar/providers.py index fe7d75122..8e0887461 100644 --- a/sqlspec/extensions/litestar/providers.py +++ b/sqlspec/extensions/litestar/providers.py @@ -24,7 +24,8 @@ from litestar.params import QueryParameter, SkipValidation from litestar.utils.signature import ParsedSignature -from sqlspec.core import ( +from sqlspec.core._cursor import resolve_env_pagination_cursor_secret +from sqlspec.core.filters import ( BeforeAfterFilter, BooleanFilter, ChoicesFilter, @@ -41,9 +42,8 @@ SearchFilter, normalize_cursor_keys, ) -from sqlspec.core._cursor import resolve_env_pagination_cursor_secret -from sqlspec.exceptions import ImproperConfigurationError, InvalidCursorError -from sqlspec.utils.tenant import resolve_cursor_tenant_identity +from sqlspec.exceptions import ImproperConfigurationError, InvalidCursorError, with_cause +from sqlspec.utils.tenant._cursor import resolve_cursor_tenant_identity from sqlspec.utils.text import camelize if TYPE_CHECKING: @@ -940,7 +940,7 @@ async def _provide_cursor_filter( keys, resolved_page_size, cursor, derived_secret, ttl=resolved_ttl, encrypted=resolved_encrypted ) except InvalidCursorError as exc: - raise ValidationException(detail="Invalid pagination cursor") from exc + raise with_cause(ValidationException(detail="Invalid pagination cursor"), exc) from exc class _SearchFilterProvider: diff --git a/sqlspec/extensions/litestar/store.py b/sqlspec/extensions/litestar/store.py index 8b11a48f1..85e38fbfb 100644 --- a/sqlspec/extensions/litestar/store.py +++ b/sqlspec/extensions/litestar/store.py @@ -26,7 +26,7 @@ from sqlspec.utils.type_guards import has_extension_config if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Iterator + from collections.abc import AsyncGenerator, Callable, Generator __all__ = ("BaseSQLSpecStore", "CompanionIndex") @@ -123,7 +123,7 @@ def __init__(self, config: ConfigT) -> None: self._validate_table_name(self._table_name) @asynccontextmanager - async def _store_connection(self) -> "AsyncIterator[_StoreConnection]": + async def _store_connection(self) -> "AsyncGenerator[_StoreConnection, None]": """Yield the connection a store operation runs on for the current task. Yields: @@ -142,7 +142,7 @@ async def _store_connection(self) -> "AsyncIterator[_StoreConnection]": yield _StoreConnection(connection, True) @contextmanager - def _store_connection_sync(self) -> "Iterator[_StoreConnection]": + def _store_connection_sync(self) -> "Generator[_StoreConnection, None, None]": """Yield the connection a store operation runs on for the current task. Yields: @@ -161,7 +161,7 @@ def _store_connection_sync(self) -> "Iterator[_StoreConnection]": yield _StoreConnection(connection, True) @asynccontextmanager - async def _store_session(self) -> "AsyncIterator[_StoreSession]": + async def _store_session(self) -> "AsyncGenerator[_StoreSession, None]": """Yield the database session a store operation runs on for the current task. Yields: @@ -180,7 +180,7 @@ async def _store_session(self) -> "AsyncIterator[_StoreSession]": yield _StoreSession(owned_session, True) @contextmanager - def _store_session_sync(self) -> "Iterator[_StoreSession]": + def _store_session_sync(self) -> "Generator[_StoreSession, None, None]": """Yield the database session a store operation runs on for the current task. Yields: @@ -199,7 +199,7 @@ def _store_session_sync(self) -> "Iterator[_StoreSession]": yield _StoreSession(owned_session, True) @asynccontextmanager - async def _discarding_session_async(self) -> "AsyncIterator[Any]": + async def _discarding_session_async(self) -> "AsyncGenerator[Any, None]": """Yield an owned session inside a transaction that is always rolled back. Yields: @@ -213,7 +213,7 @@ async def _discarding_session_async(self) -> "AsyncIterator[Any]": await session.rollback() @contextmanager - def _discarding_session_sync(self) -> "Iterator[Any]": + def _discarding_session_sync(self) -> "Generator[Any, None, None]": """Yield an owned session inside a transaction that is always rolled back. Yields: @@ -475,7 +475,7 @@ def _calculate_expires_at(self, expires_in: "int | timedelta | None") -> "dateti expires_in_seconds = expires_in.total_seconds() if isinstance(expires_in, timedelta) else float(expires_in) if expires_in_seconds <= 0: - return datetime.now(UTC) - timedelta(microseconds=1) + return datetime.now(UTC) - timedelta(seconds=1) return datetime.now(UTC) + timedelta(seconds=expires_in_seconds) @@ -562,7 +562,7 @@ def _current_lease(config: Any) -> StoreLease: @asynccontextmanager -async def _borrowed_session_async(lease: StoreLease) -> "AsyncIterator[Any]": +async def _borrowed_session_async(lease: StoreLease) -> "AsyncGenerator[Any, None]": """Yield the borrowed request session inside a transaction. The store joins an open request transaction. When none is open it begins one; in @@ -588,7 +588,7 @@ async def _borrowed_session_async(lease: StoreLease) -> "AsyncIterator[Any]": @contextmanager -def _borrowed_session_sync(lease: StoreLease) -> "Iterator[Any]": +def _borrowed_session_sync(lease: StoreLease) -> "Generator[Any, None, None]": """Yield the borrowed request session inside a transaction. The store joins an open request transaction. When none is open it begins one; in diff --git a/sqlspec/extensions/prometheus/_observer.py b/sqlspec/extensions/prometheus/_observer.py index 6072dec8b..50126c7fc 100644 --- a/sqlspec/extensions/prometheus/_observer.py +++ b/sqlspec/extensions/prometheus/_observer.py @@ -3,7 +3,9 @@ from collections.abc import Iterable from typing import Any -from sqlspec.observability import ObservabilityConfig, StatementEvent, StatementObserver, resolve_db_system +from sqlspec.observability._common import resolve_db_system +from sqlspec.observability._config import ObservabilityConfig, StatementObserver +from sqlspec.observability._observer import StatementEvent from sqlspec.typing import Counter, Histogram from sqlspec.utils.module_loader import ensure_prometheus @@ -23,12 +25,13 @@ "storage_backend", "transaction_state", }) +_MAX_CACHED_LABEL_SETS = 1024 class PrometheusStatementObserver: """Statement observer that records Prometheus metrics.""" - __slots__ = ("_counters", "_duration", "_label_names", "_rows") + __slots__ = ("_children", "_counters", "_duration", "_label_names", "_row_children", "_rows") def __init__( self, @@ -46,6 +49,8 @@ def __init__( msg = f"High-cardinality Prometheus label names are not allowed: {disallowed_names}" raise ValueError(msg) self._label_names = resolved_labels + self._children: dict[tuple[str, ...], tuple[Any, Any]] = {} + self._row_children: dict[tuple[str, ...], Any] = {} self._counters = Counter( "query_total", "Total SQL statements executed", @@ -78,10 +83,20 @@ def __init__( def __call__(self, event: StatementEvent) -> None: label_values = self._label_values(event) - self._counters.labels(*label_values).inc() - self._duration.labels(*label_values).observe(max(event.duration_s, 0.0)) + children = self._children.get(label_values) + if children is None: + children = (self._counters.labels(*label_values), self._duration.labels(*label_values)) + if len(self._children) < _MAX_CACHED_LABEL_SETS: + self._children[label_values] = children + children[0].inc() + children[1].observe(max(event.duration_s, 0.0)) if event.rows_affected is not None: - self._rows.labels(*label_values).observe(float(event.rows_affected)) + rows = self._row_children.get(label_values) + if rows is None: + rows = self._rows.labels(*label_values) + if len(self._row_children) < _MAX_CACHED_LABEL_SETS: + self._row_children[label_values] = rows + rows.observe(float(event.rows_affected)) def _label_values(self, event: StatementEvent) -> tuple[str, ...]: values: list[str] = [] diff --git a/sqlspec/extensions/sanic/_state.py b/sqlspec/extensions/sanic/_state.py index fe43c63da..c5ce90920 100644 --- a/sqlspec/extensions/sanic/_state.py +++ b/sqlspec/extensions/sanic/_state.py @@ -37,14 +37,14 @@ class SanicConfigState: pool_key: str session_key: str commit_mode: CommitMode - extra_commit_statuses: "set[int] | None" - extra_rollback_statuses: "set[int] | None" + extra_commit_statuses: set[int] | None + extra_rollback_statuses: set[int] | None disable_di: bool tenant_extractor: Callable[[Any], Any] | None = None tenant_receipt_keys: tuple[str, ...] = field(default=(), init=False, repr=False, compare=False) enable_correlation_middleware: bool = True correlation_header: str = "x-request-id" - correlation_headers: "tuple[str, ...] | None" = None + correlation_headers: tuple[str, ...] | None = None auto_trace_headers: bool = True enable_sqlcommenter_middleware: bool = True sqlcommenter_framework: str = "sanic" diff --git a/sqlspec/extensions/sanic/_utils.py b/sqlspec/extensions/sanic/_utils.py index dc97f103f..e53132bd1 100644 --- a/sqlspec/extensions/sanic/_utils.py +++ b/sqlspec/extensions/sanic/_utils.py @@ -8,7 +8,7 @@ from sqlspec.extensions.sanic._state import tenant_receipt_key from sqlspec.utils.adapters import resolve_adapter_name from sqlspec.utils.deprecation import deprecated -from sqlspec.utils.tenant import TenantContextPayload +from sqlspec.utils.tenant._context import TenantContextPayload if TYPE_CHECKING: from sqlspec.extensions.sanic._state import SanicConfigState @@ -118,7 +118,9 @@ def get_connection_from_request(request: Any, config_state: "SanicConfigState") f"Sanic request context does not contain SQLSpec connection '{config_state.connection_key}'. " "Ensure SQLSpecPlugin is initialized and its request middleware runs before accessing sessions." ) - raise ImproperConfigurationError(msg) from exc + error = ImproperConfigurationError(msg) + error.__cause__ = exc + raise error from exc def get_or_create_db_session(request: Any, config_state: "SanicConfigState") -> Any: diff --git a/sqlspec/extensions/sanic/extension.py b/sqlspec/extensions/sanic/extension.py index db8ced2b2..d3d6d0848 100644 --- a/sqlspec/extensions/sanic/extension.py +++ b/sqlspec/extensions/sanic/extension.py @@ -14,7 +14,7 @@ from sqlspec.driver import DeferredAsyncSession, raise_async_connection_required from sqlspec.driver._routing import AsyncRoutingSession from sqlspec.exceptions import ImproperConfigurationError -from sqlspec.extensions.sanic._state import SanicConfigState +from sqlspec.extensions.sanic._state import CommitMode, SanicConfigState from sqlspec.extensions.sanic._utils import ( get_context_value, get_or_create_db_session, @@ -25,9 +25,16 @@ set_context_value, set_request_tenant, ) +from sqlspec.utils.config_tools import normalize_extension_status_codes, resolve_extension_commit_mode from sqlspec.utils.correlation import CorrelationContext from sqlspec.utils.logging import get_logger, log_with_context -from sqlspec.utils.routing import RoutingContext, RoutingContextPayload, RoutingEntry, apply_request_routing_hints +from sqlspec.utils.routing import ( + RoutingContext, + RoutingContextPayload, + RoutingEntry, + apply_request_routing_hints, + validate_routing_middleware_settings, +) from sqlspec.utils.sync_tools import ensure_async_, with_ensure_async_ from sqlspec.utils.tenant import TenantContext, TenantContextPayload, TenantRequestBinding, is_tenant_bypass_route from sqlspec.utils.type_guards import has_name @@ -59,7 +66,7 @@ _REQUEST_EXCEPTION_KEY: Final = "sqlspec_request_exception" _ACTIVE_REQUEST_KEY: Final = "_sqlspec_active_request" _CURRENT_REQUEST: ContextVar[Any | None] = ContextVar("_sqlspec_sanic_active_request", default=None) -DEFAULT_COMMIT_MODE = "autocommit" +DEFAULT_COMMIT_MODE: CommitMode = "autocommit" DEFAULT_CONNECTION_KEY = "db_connection" DEFAULT_POOL_KEY = "db_pool" DEFAULT_SESSION_KEY = "db_session" @@ -516,7 +523,7 @@ def _extract_extension_settings(self, config: Any) -> "dict[str, Any]": "routing_header": framework_config.get("routing_header"), "primary_pin_cookie": framework_config.get("primary_pin_cookie"), "primary_pin_secret": framework_config.get("primary_pin_secret"), - "commit_mode": framework_config.get("commit_mode", DEFAULT_COMMIT_MODE), + "commit_mode": resolve_extension_commit_mode(config, framework_config, DEFAULT_COMMIT_MODE), "extra_commit_statuses": framework_config.get("extra_commit_statuses"), "extra_rollback_statuses": framework_config.get("extra_rollback_statuses"), "disable_di": framework_config.get("disable_di", False), @@ -545,31 +552,37 @@ def _config_state(self, config: Any, settings: "dict[str, Any]") -> SanicConfigS if settings["tenant_extractor"] is not None and not _has_tenant_strategy(config): msg = "tenant_extractor requires driver_features['tenant_strategy'] on the same configuration" raise ImproperConfigurationError(msg) + enable_routing, routing_header, primary_pin_cookie, primary_pin_secret = validate_routing_middleware_settings( + enable_routing_middleware=settings["enable_routing_middleware"], + routing_header=settings["routing_header"], + primary_pin_cookie=settings["primary_pin_cookie"], + primary_pin_secret=settings["primary_pin_secret"], + ) return SanicConfigState( config=config, connection_key=settings["connection_key"], pool_key=settings["pool_key"], session_key=settings["session_key"], commit_mode=settings["commit_mode"], - extra_commit_statuses=settings["extra_commit_statuses"], - extra_rollback_statuses=settings["extra_rollback_statuses"], - disable_di=settings["disable_di"], + extra_commit_statuses=normalize_extension_status_codes(settings["extra_commit_statuses"]), + extra_rollback_statuses=normalize_extension_status_codes(settings["extra_rollback_statuses"]), + disable_di=bool(settings["disable_di"]), tenant_extractor=settings["tenant_extractor"], - enable_correlation_middleware=settings["enable_correlation_middleware"], + enable_correlation_middleware=bool(settings["enable_correlation_middleware"]), correlation_header=settings["correlation_header"], correlation_headers=settings["correlation_headers"], - auto_trace_headers=settings["auto_trace_headers"], - enable_sqlcommenter_middleware=settings["enable_sqlcommenter_middleware"], + auto_trace_headers=bool(settings["auto_trace_headers"]), + enable_sqlcommenter_middleware=bool(settings["enable_sqlcommenter_middleware"]), sqlcommenter_framework=settings["sqlcommenter_framework"], replica_session_key=settings["replica_session_key"], replica_connection_key=settings["replica_connection_key"], replica_pool_key=settings["replica_pool_key"], routing_session_key=settings["routing_session_key"], default_routing_mode=settings["default_routing_mode"], - enable_routing_middleware=settings["enable_routing_middleware"], - routing_header=settings["routing_header"], - primary_pin_cookie=settings["primary_pin_cookie"], - primary_pin_secret=settings["primary_pin_secret"], + enable_routing_middleware=enable_routing, + routing_header=routing_header, + primary_pin_cookie=primary_pin_cookie, + primary_pin_secret=primary_pin_secret, ) def _tenant_state_for(self, key: Any) -> Any: diff --git a/sqlspec/extensions/starlette/_state.py b/sqlspec/extensions/starlette/_state.py index 951b9c470..7f4915a02 100644 --- a/sqlspec/extensions/starlette/_state.py +++ b/sqlspec/extensions/starlette/_state.py @@ -36,12 +36,12 @@ class SQLSpecConfigState: pool_key: str session_key: str commit_mode: CommitMode - extra_commit_statuses: "set[int] | None" - extra_rollback_statuses: "set[int] | None" + extra_commit_statuses: set[int] | None + extra_rollback_statuses: set[int] | None disable_di: bool enable_correlation_middleware: bool = True correlation_header: str = "x-request-id" - correlation_headers: "tuple[str, ...] | None" = None + correlation_headers: tuple[str, ...] | None = None auto_trace_headers: bool = True enable_sqlcommenter_middleware: bool = True sqlcommenter_framework: str = "starlette" diff --git a/sqlspec/extensions/starlette/_utils.py b/sqlspec/extensions/starlette/_utils.py index c522bf062..6b17a2836 100644 --- a/sqlspec/extensions/starlette/_utils.py +++ b/sqlspec/extensions/starlette/_utils.py @@ -7,7 +7,7 @@ from sqlspec.extensions.starlette._state import tenant_receipt_key from sqlspec.utils.adapters import resolve_adapter_name from sqlspec.utils.deprecation import deprecated -from sqlspec.utils.tenant import TenantContextPayload +from sqlspec.utils.tenant._context import TenantContextPayload if TYPE_CHECKING: from starlette.requests import Request @@ -52,7 +52,9 @@ def get_state_value(state: Any, key: str, default: Any = _MISSING) -> Any: return data[key] except KeyError as exc: msg = f"'{state.__class__.__name__}' object has no attribute '{key}'" - raise AttributeError(msg) from exc + error = AttributeError(msg) + error.__cause__ = exc + raise error from exc return data.get(key, default) diff --git a/sqlspec/extensions/starlette/extension.py b/sqlspec/extensions/starlette/extension.py index 41d61558e..3f28990a4 100644 --- a/sqlspec/extensions/starlette/extension.py +++ b/sqlspec/extensions/starlette/extension.py @@ -10,7 +10,7 @@ from sqlspec.base import SQLSpec, coerce_sqlspec from sqlspec.core._cursor import resolve_plugin_cursor_settings from sqlspec.exceptions import ImproperConfigurationError -from sqlspec.extensions.starlette._state import SQLSpecConfigState +from sqlspec.extensions.starlette._state import CommitMode, SQLSpecConfigState from sqlspec.extensions.starlette._utils import ( get_or_create_db_session, get_or_create_db_session_async, @@ -28,7 +28,9 @@ get_request_resource_async, has_tenant_strategy, ) +from sqlspec.utils.config_tools import normalize_extension_status_codes, resolve_extension_commit_mode from sqlspec.utils.logging import get_logger, log_with_context +from sqlspec.utils.routing import validate_routing_middleware_settings from sqlspec.utils.sync_tools import ensure_async_ if TYPE_CHECKING: @@ -52,7 +54,7 @@ logger = get_logger("sqlspec.extensions.starlette") -DEFAULT_COMMIT_MODE = "autocommit" +DEFAULT_COMMIT_MODE: CommitMode = "autocommit" DEFAULT_CONNECTION_KEY = "db_connection" DEFAULT_POOL_KEY = "db_pool" DEFAULT_SESSION_KEY = "db_session" @@ -487,7 +489,7 @@ def _extract_extension_settings(self, config: Any) -> "dict[str, Any]": "routing_header": framework_config.get("routing_header"), "primary_pin_cookie": framework_config.get("primary_pin_cookie"), "primary_pin_secret": framework_config.get("primary_pin_secret"), - "commit_mode": framework_config.get("commit_mode", DEFAULT_COMMIT_MODE), + "commit_mode": resolve_extension_commit_mode(config, framework_config, DEFAULT_COMMIT_MODE), "extra_commit_statuses": framework_config.get("extra_commit_statuses"), "extra_rollback_statuses": framework_config.get("extra_rollback_statuses"), "disable_di": framework_config.get("disable_di", False), @@ -519,20 +521,26 @@ def _config_state(self, config: Any, settings: "dict[str, Any]") -> SQLSpecConfi if settings.get("tenant_extractor") is not None and not has_tenant_strategy(config): msg = "tenant_extractor requires driver_features['tenant_strategy'] on the same configuration" raise ImproperConfigurationError(msg) + enable_routing, routing_header, primary_pin_cookie, primary_pin_secret = validate_routing_middleware_settings( + enable_routing_middleware=settings.get("enable_routing_middleware", True), + routing_header=settings.get("routing_header"), + primary_pin_cookie=settings.get("primary_pin_cookie"), + primary_pin_secret=settings.get("primary_pin_secret"), + ) return SQLSpecConfigState( config=config, connection_key=settings["connection_key"], pool_key=settings["pool_key"], session_key=settings["session_key"], commit_mode=settings["commit_mode"], - extra_commit_statuses=settings["extra_commit_statuses"], - extra_rollback_statuses=settings["extra_rollback_statuses"], - disable_di=settings["disable_di"], - enable_correlation_middleware=settings["enable_correlation_middleware"], + extra_commit_statuses=normalize_extension_status_codes(settings["extra_commit_statuses"]), + extra_rollback_statuses=normalize_extension_status_codes(settings["extra_rollback_statuses"]), + disable_di=bool(settings["disable_di"]), + enable_correlation_middleware=bool(settings["enable_correlation_middleware"]), correlation_header=settings["correlation_header"], correlation_headers=settings["correlation_headers"], - auto_trace_headers=settings["auto_trace_headers"], - enable_sqlcommenter_middleware=settings["enable_sqlcommenter_middleware"], + auto_trace_headers=bool(settings["auto_trace_headers"]), + enable_sqlcommenter_middleware=bool(settings["enable_sqlcommenter_middleware"]), sqlcommenter_framework=settings["sqlcommenter_framework"], tenant_extractor=settings.get("tenant_extractor"), replica_connection_key=settings.get( @@ -553,10 +561,10 @@ def _config_state(self, config: Any, settings: "dict[str, Any]") -> SQLSpecConfi ), routing_session_key=settings.get("routing_session_key"), default_routing_mode=settings.get("default_routing_mode", "auto"), - enable_routing_middleware=settings.get("enable_routing_middleware", True), - routing_header=settings.get("routing_header"), - primary_pin_cookie=settings.get("primary_pin_cookie"), - primary_pin_secret=settings.get("primary_pin_secret"), + enable_routing_middleware=enable_routing, + routing_header=routing_header, + primary_pin_cookie=primary_pin_cookie, + primary_pin_secret=primary_pin_secret, ) def _ensure_unique_keys(self) -> None: diff --git a/sqlspec/loader.py b/sqlspec/loader.py index 338734aff..f4544acf1 100644 --- a/sqlspec/loader.py +++ b/sqlspec/loader.py @@ -20,7 +20,10 @@ from sqlglot import exp -from sqlspec.core import SQL, ParameterDeclaration, ParameterValidator, get_cache, get_cache_config +from sqlspec.core.cache import get_cache, get_cache_config +from sqlspec.core.parameters._declared import ParameterDeclaration +from sqlspec.core.parameters._validator import ParameterValidator +from sqlspec.core.statement import SQL from sqlspec.exceptions import ( FileNotFoundInStorageError, SQLFileNotFoundError, @@ -29,6 +32,7 @@ SQLSlotError, SQLStatementNotFoundError, StorageOperationFailedError, + with_cause, ) from sqlspec.storage.registry import storage_registry as default_storage_registry from sqlspec.utils.correlation import CorrelationContext @@ -38,7 +42,7 @@ if TYPE_CHECKING: from collections.abc import Sequence - from sqlspec.observability import ObservabilityRuntime + from sqlspec.observability._runtime import ObservabilityRuntime from sqlspec.storage.registry import StorageRegistry __all__ = ( @@ -629,7 +633,7 @@ def get_sql(self, name: str, **slots: Any) -> "SQL": try: sql.compile() except Exception as exc: - raise SQLFileParseError(name=name, path="", original_error=exc) from exc + raise with_cause(SQLFileParseError(name=name, path="", original_error=exc), exc) from exc if slot_parameters: return SQL(sql, slot_parameters, dialect=sqlglot_dialect, declared_parameters=parsed_statement.parameters) if not slots: @@ -691,7 +695,7 @@ def _calculate_file_checksum(self, path: str | Path) -> str: try: return self._compute_checksum(self._read_file_content(path)) except Exception as e: - raise SQLFileParseError(str(path), str(path), e) from e + raise with_cause(SQLFileParseError(str(path), str(path), e), e) from e def _is_file_unchanged(self, path: str | Path, cached_file: SQLFile) -> bool: """Check if file has changed since caching. @@ -810,15 +814,15 @@ def _read_file_content(self, path: str | Path) -> str: return backend.read_text_sync(path_str, encoding=self.encoding) except KeyError as e: - raise SQLFileNotFoundError(path_str) from e + raise with_cause(SQLFileNotFoundError(path_str), e) from e except FileNotFoundInStorageError as e: - raise SQLFileNotFoundError(path_str) from e + raise with_cause(SQLFileNotFoundError(path_str), e) from e except FileNotFoundError as e: - raise SQLFileNotFoundError(path_str) from e + raise with_cause(SQLFileNotFoundError(path_str), e) from e except StorageOperationFailedError as e: - raise SQLFileParseError(path_str, path_str, e) from e + raise with_cause(SQLFileParseError(path_str, path_str, e), e) from e except Exception as e: - raise SQLFileParseError(path_str, path_str, e) from e + raise with_cause(SQLFileParseError(path_str, path_str, e), e) from e @staticmethod def _strip_leading_comments(sql_text: str) -> str: diff --git a/sqlspec/migrations/__init__.py b/sqlspec/migrations/__init__.py index 8ae33fbf9..e778bfc10 100644 --- a/sqlspec/migrations/__init__.py +++ b/sqlspec/migrations/__init__.py @@ -100,5 +100,5 @@ def __dir__() -> list[str]: if _COMPILED: for _name in __all__: - if _name not in globals(): + if _name not in globals() and _EXPORTS[_name][0] != "sqlspec.migrations.commands": __getattr__(_name) diff --git a/sqlspec/migrations/base.py b/sqlspec/migrations/base.py index c6d227592..851e7eb90 100644 --- a/sqlspec/migrations/base.py +++ b/sqlspec/migrations/base.py @@ -19,7 +19,7 @@ from sqlspec.builder._insert import Insert from sqlspec.builder._select import Select from sqlspec.builder._update import Update -from sqlspec.exceptions import MigrationError, SQLSpecError, SquashValidationError +from sqlspec.exceptions import MigrationError, SQLSpecError, SquashValidationError, with_cause from sqlspec.migrations.runner import BaseMigrationRunner, extract_replaces_versions from sqlspec.migrations.schema import SchemaTarget from sqlspec.migrations.templates import MigrationTemplateSettings, build_template_settings @@ -30,7 +30,7 @@ resolve_tracker_schema, ) from sqlspec.migrations.version import parse_version -from sqlspec.observability import resolve_db_system +from sqlspec.observability._common import resolve_db_system from sqlspec.utils.logging import get_logger, log_with_context from sqlspec.utils.module_loader import module_to_os_path from sqlspec.utils.text import split_qualified_identifier @@ -40,7 +40,7 @@ from collections.abc import Set as AbstractSet from sqlspec.config import DatabaseConfigProtocol - from sqlspec.observability import ObservabilityRuntime + from sqlspec.observability._runtime import ObservabilityRuntime __all__ = ( "AppliedMigrationRecord", @@ -58,6 +58,7 @@ _LOCK_NAME_HASH_BYTES: Final[int] = 6 _LOCK_NAME_PREFIX_LENGTH: Final[int] = _MAX_LOCK_NAME_LENGTH - 1 - (_LOCK_NAME_HASH_BYTES * 2) _BRIDGE_DRIVER_CLASS_NAMES: Final[frozenset[str]] = frozenset({"AdbcDriver", "ArrowOdbcDriver"}) +_DRIVER_BASE_CLASS_NAMES: Final[frozenset[str]] = frozenset({"SyncDriverAdapterBase", "AsyncDriverAdapterBase"}) _LEASE_LOCK_DIALECTS: Final[frozenset[str]] = frozenset({ "oracle", "duckdb", @@ -76,8 +77,13 @@ "SELECT " "c.relkind::text AS relkind, " "(COUNT(t.tgrelid) > 0) AS has_user_triggers, " - "(pg_catalog.pg_has_role(c.relowner, 'USAGE') " - "OR COALESCE((SELECT r.rolsuper FROM pg_catalog.pg_roles r WHERE r.rolname = current_user), false)) " + "EXISTS (SELECT 1 FROM pg_catalog.pg_rewrite rw WHERE rw.ev_class = c.oid AND rw.rulename <> '_RETURN') " + "AS has_rules, " + "pg_catalog.row_security_active(c.oid) AS has_row_security, " + "(c.relowner = (SELECT r.oid FROM pg_catalog.pg_roles r WHERE r.rolname = current_user) " + "OR EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.oid = c.relowner AND r.rolsuper) " + "OR (NOT COALESCE((SELECT r.rolsuper FROM pg_catalog.pg_roles r WHERE r.rolname = current_user), false) " + "AND pg_catalog.pg_has_role(current_user, c.relowner, 'MEMBER'))) " "AS owned_by_current_role " "FROM pg_catalog.pg_class c " "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace " @@ -98,9 +104,16 @@ _PG_SCHEMA_OMISSIONS_SQL: Final[str] = ( "SELECT 'grant' AS object_kind, n.nspname::text AS schema_name, c.relname::text AS object_name " "FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace " - "WHERE c.relacl IS NOT NULL AND c.relkind IN ('r', 'p') " + "WHERE (c.relacl IS NOT NULL " + "OR EXISTS (SELECT 1 FROM pg_catalog.pg_attribute a " + "WHERE a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped AND a.attacl IS NOT NULL)) " + "AND c.relkind IN ('r', 'p', 'v', 'm') " "AND n.nspname = COALESCE(CAST(:schema_name AS text), current_schema()) " "UNION ALL " + "SELECT 'materialized_view', n.nspname::text, c.relname::text " + "FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace " + "WHERE c.relkind = 'm' AND n.nspname = COALESCE(CAST(:schema_name AS text), current_schema()) " + "UNION ALL " "SELECT 'function', n.nspname::text, p.proname::text " "FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace " "WHERE n.nspname = COALESCE(CAST(:schema_name AS text), current_schema()) " @@ -787,14 +800,22 @@ def _migration_lock_key(self) -> "tuple[str, int]": def _resolve_lock_backend(cls, driver: Any) -> str: """Classify ``driver`` into its migration lock backend family. + The dialect is the driver class value, or the instance value for driver adapters whose + dialect is stored per instance. + Returns: ``"postgres"``, ``"mysql"``, ``"mssql"``, ``"sqlite"``, ``"lease"`` for backends locking through a lease row, ``"unsupported"`` for backends that cannot be serialized, or an empty string for an unknown dialect. """ driver_type = type(driver) - is_bridge_driver = any(base.__name__ in _BRIDGE_DRIVER_CLASS_NAMES for base in driver_type.__mro__) - raw_dialect = getattr(driver, "dialect", None) if is_bridge_driver else getattr(driver_type, "dialect", None) + base_names = {base.__name__ for base in driver_type.__mro__} + is_bridge_driver = not base_names.isdisjoint(_BRIDGE_DRIVER_CLASS_NAMES) + raw_dialect = getattr(driver_type, "dialect", None) + if is_bridge_driver or ( + not isinstance(raw_dialect, str) and not base_names.isdisjoint(_DRIVER_BASE_CLASS_NAMES) + ): + raw_dialect = getattr(driver, "dialect", None) dialect = raw_dialect.lower() if isinstance(raw_dialect, str) else "" if _is_cockroach_driver(driver): return "lease" @@ -1151,7 +1172,7 @@ def _resolve_extension_migrations_dir(self, ext_name: str, ext_options: "Mapping f"Set extension_config['{ext_name}']['migrations_path'] to register " "reconcilers shipped by another package." ) - raise MigrationError(msg) from exc + raise with_cause(MigrationError(msg), exc) from exc logger.warning( "Extension '%s' is configured but '%s' is not importable; no migrations were " "registered for it. Set extension_config['%s']['migrations_path'] to register " @@ -1631,7 +1652,7 @@ def _ensure_schema_extraction_safe_sync( refused = _extract_catalog_rows(driver.execute(refusal_sql, parameters)) omitted = _extract_catalog_rows(driver.execute(omissions_sql, parameters)) if omissions_sql else [] except SQLSpecError as exc: - raise _extraction_check_error(label, exc) from exc + raise with_cause(_extraction_check_error(label, exc), exc) from exc _validate_schema_extraction_rows( refused, omitted, backend_label=label, exclude_tables=exclude_tables, use_squash_error=use_squash_error ) @@ -1661,7 +1682,7 @@ async def _ensure_schema_extraction_safe_async( refused = _extract_catalog_rows(await driver.execute(refusal_sql, parameters)) omitted = _extract_catalog_rows(await driver.execute(omissions_sql, parameters)) if omissions_sql else [] except SQLSpecError as exc: - raise _extraction_check_error(label, exc) from exc + raise with_cause(_extraction_check_error(label, exc), exc) from exc _validate_schema_extraction_rows( refused, omitted, backend_label=label, exclude_tables=exclude_tables, use_squash_error=use_squash_error ) diff --git a/sqlspec/migrations/commands.py b/sqlspec/migrations/commands.py index 6b9e2eba2..41c3e3e43 100644 --- a/sqlspec/migrations/commands.py +++ b/sqlspec/migrations/commands.py @@ -50,7 +50,7 @@ from sqlspec.utils.serializers import from_json if TYPE_CHECKING: - from collections.abc import AsyncIterator, Iterator + from collections.abc import AsyncGenerator, Generator from collections.abc import Set as AbstractSet from sqlspec.config import AsyncConfigT, DatabaseConfigProtocol, SyncConfigT @@ -2197,7 +2197,7 @@ def _run_reconciler_downgrades( self.tracker.remove_migration(driver, version_num) @contextmanager - def _hold_migration_lock(self, driver: Any, *, dry_run: bool = False) -> "Iterator[None]": + def _hold_migration_lock(self, driver: Any, *, dry_run: bool = False) -> "Generator[None, None, None]": """Validate the configured schema, then hold the tracker lock for the command. Outer SQLite transactions are preserved across step commits. @@ -3502,7 +3502,7 @@ async def _run_reconciler_downgrades( await self.tracker.remove_migration(driver, version_num) @asynccontextmanager - async def _hold_migration_lock(self, driver: Any, *, dry_run: bool = False) -> "AsyncIterator[None]": + async def _hold_migration_lock(self, driver: Any, *, dry_run: bool = False) -> "AsyncGenerator[None, None]": """Validate the configured schema, then hold the async tracker lock for the command. Outer SQLite transactions are preserved across step commits. diff --git a/sqlspec/migrations/context.py b/sqlspec/migrations/context.py index 597c4e8a3..33025b601 100644 --- a/sqlspec/migrations/context.py +++ b/sqlspec/migrations/context.py @@ -12,7 +12,8 @@ from sqlspec.utils.type_guards import has_statement_config_factory if TYPE_CHECKING: - from sqlspec.driver import AsyncDriverAdapterBase, SyncDriverAdapterBase + from sqlspec.driver._async import AsyncDriverAdapterBase + from sqlspec.driver._sync import SyncDriverAdapterBase __all__ = ("MigrationContext",) diff --git a/sqlspec/migrations/loaders.py b/sqlspec/migrations/loaders.py index d310fe83b..337c29ce2 100644 --- a/sqlspec/migrations/loaders.py +++ b/sqlspec/migrations/loaders.py @@ -8,17 +8,19 @@ import builtins import inspect import os +import stat import sys import types -from collections.abc import Awaitable, Callable, Iterable, Iterator +from collections.abc import Awaitable, Callable, Generator, Iterable from contextlib import contextmanager, suppress from pathlib import Path from typing import Any, Final, cast -from sqlspec.exceptions import MigrationError +from sqlspec.exceptions import MigrationError, with_cause from sqlspec.loader import SQLFileLoader as CoreSQLFileLoader from sqlspec.migrations.context import MigrationContext from sqlspec.migrations.version import extract_migration_version +from sqlspec.utils.logging import get_logger from sqlspec.utils.sync_tools import await_ from sqlspec.utils.uuids import uuid4 @@ -35,6 +37,8 @@ "temporary_project_path", ) +logger = get_logger("sqlspec.migrations.loaders") + PROJECT_ROOT_MARKERS: Final[list[str]] = ["pyproject.toml", ".git", "setup.cfg", "setup.py"] ReconcileCallable = Callable[[MigrationContext], Awaitable[None] | None] @@ -43,6 +47,15 @@ def find_project_root(start_path: Path) -> Path: """Search upward from start_path for a project marker file within trusted ownership bounds. + On POSIX the search stops at the first directory owned by another non-root user, and a + marker counts only when the current user or root owns it and, for a symbolic link, its + target. Inside a shared world-writable directory (sticky, or owned by a different user) a + marker must also be singly linked unless it is a directory, and must not be world-writable; + such a directory is never used as the fallback root. A sticky world-writable directory such + as ``/tmp`` is never used as the project root, even when it holds a trusted marker. A + refused marker or root is logged at warning level, and the directory that ends the search + at debug level. Platforms without ``os.getuid`` skip the ownership and permission tests. + Args: start_path: Directory to start searching from. @@ -50,38 +63,137 @@ def find_project_root(start_path: Path) -> Path: Path to project root or parent directory. """ resolved_start = start_path.resolve() - allowed_uids = {os.getuid(), 0} if hasattr(os, "getuid") else None + current_uid = os.getuid() if hasattr(os, "getuid") else None current_path = resolved_start + untrusted_path: Path | None = None while current_path != current_path.parent: - if allowed_uids is not None: - try: - owner_uid: int | None = current_path.stat().st_uid - except FileNotFoundError: - owner_uid = None - except OSError: - break - if owner_uid is not None and owner_uid not in allowed_uids: - break - for marker in PROJECT_ROOT_MARKERS: - if (current_path / marker).exists(): + if not _is_trusted_root_dir(current_path, current_uid): + untrusted_path = current_path + break + marker_status = _find_trusted_marker(current_path, current_uid) + if marker_status is False: + return resolved_start + if marker_status is True: + if current_uid is None or not _is_sticky_world_writable(current_path): return current_path + logger.warning( + "Ignoring the project marker in %s: a sticky world-writable directory is not used as the project root", + current_path, + ) + break current_path = current_path.parent fallback = resolved_start.parent - if allowed_uids is not None: - try: - if fallback.stat().st_uid not in allowed_uids: - return resolved_start - except FileNotFoundError: - pass - except OSError: - return resolved_start + if fallback == untrusted_path or not _is_trusted_root_dir(fallback, current_uid, allow_missing=True): + return resolved_start + if current_uid is not None and _is_shared_directory(fallback, current_uid): + logger.warning( + "Using %s as the project root: parent directory %s is a shared world-writable directory", + resolved_start, + fallback, + ) + return resolved_start return fallback +def _is_trusted_root_dir(path: Path, current_uid: "int | None", *, allow_missing: bool = False) -> bool: + """Return True when ``path`` is a non-symlinked directory that the current user or root owns.""" + try: + path_stat = path.stat(follow_symlinks=False) + except FileNotFoundError: + return allow_missing + except OSError as exc: + logger.debug("Stopping project root discovery at %s: directory cannot be inspected (%s)", path, exc) + return False + if stat.S_ISLNK(getattr(path_stat, "st_mode", 0)): + logger.debug("Stopping project root discovery at %s: directory is a symbolic link", path) + return False + if current_uid is not None and path_stat.st_uid not in {current_uid, 0}: + logger.debug( + "Stopping project root discovery at %s: directory is owned by another user (uid %s)", path, path_stat.st_uid + ) + return False + return True + + +def _is_shared_directory(path: Path, current_uid: int) -> bool: + """Return True when ``path`` is world-writable and either sticky or owned by a different user.""" + try: + path_stat = path.stat(follow_symlinks=False) + except FileNotFoundError: + return False + except OSError: + return True + mode = getattr(path_stat, "st_mode", 0) + if not mode & stat.S_IWOTH: + return False + return bool(mode & stat.S_ISVTX) or path_stat.st_uid != current_uid + + +def _is_sticky_world_writable(path: Path) -> bool: + """Return True when ``path`` is a world-writable directory with the sticky bit set.""" + try: + mode = getattr(path.stat(follow_symlinks=False), "st_mode", 0) + except OSError: + return True + return bool(mode & stat.S_IWOTH) and bool(mode & stat.S_ISVTX) + + +def _find_trusted_marker(directory: Path, current_uid: "int | None") -> "bool | None": + """Return True for a trusted project marker in ``directory``, False for an untrusted one, None for none.""" + for marker in PROJECT_ROOT_MARKERS: + marker_status = _is_trusted_marker(directory / marker, current_uid) + if marker_status is not None: + return marker_status + return None + + +def _is_trusted_marker(path: Path, current_uid: "int | None") -> "bool | None": + """Return True if a project marker exists and is trusted, False if untrusted, None if absent. + + On POSIX the current user or root must own ``path`` and, for a symbolic link, its target. + Inside a shared world-writable directory ``path`` must also be singly linked unless it is + a directory, and its target must not be world-writable. + """ + try: + link_stat = path.stat(follow_symlinks=False) + except FileNotFoundError: + return None + except OSError as exc: + logger.warning("Refusing project marker %s: it cannot be inspected (%s)", path, exc) + return False + link_mode = getattr(link_stat, "st_mode", 0) + target_stat = link_stat + if stat.S_ISLNK(link_mode): + try: + target_stat = path.stat() + except OSError as exc: + logger.warning("Refusing project marker %s: symbolic link target cannot be inspected (%s)", path, exc) + return False + if current_uid is None: + return True + for owner_uid in (link_stat.st_uid, target_stat.st_uid): + if owner_uid not in {current_uid, 0}: + logger.warning("Refusing project marker %s: it is owned by another user (uid %s)", path, owner_uid) + return False + if not _is_shared_directory(path.parent, current_uid): + return True + if not stat.S_ISDIR(link_mode) and getattr(link_stat, "st_nlink", 1) > 1: + logger.warning( + "Refusing project marker %s: it has several hard links inside a shared world-writable directory", path + ) + return False + if getattr(target_stat, "st_mode", 0) & stat.S_IWOTH: + logger.warning( + "Refusing project marker %s: it is world-writable inside a shared world-writable directory", path + ) + return False + return True + + @contextmanager -def temporary_project_path(project_root: Path) -> Iterator[None]: +def temporary_project_path(project_root: Path) -> Generator[None, None, None]: """Temporarily add project_root to sys.path for imports. Args: @@ -426,7 +538,7 @@ def validate_migration_file(self, path: Path) -> None: tree = _parse_python_migration_ast(path) except (OSError, SyntaxError) as exc: msg = f"Failed to parse migration file {path}: {exc}" - raise MigrationLoadError(msg) from exc + raise with_cause(MigrationLoadError(msg), exc) from exc if _resolve_ast_callable_binding(tree, ("up", "migrate_up")) is None: msg = f"Migration {path} missing required upgrade function. Expected 'up()' or 'migrate_up()'" @@ -458,7 +570,7 @@ def _find_project_root(self, start_path: Path) -> Path: return find_project_root(start_path) @contextmanager - def _temporary_project_path(self) -> Iterator[None]: + def _temporary_project_path(self) -> Generator[None, None, None]: """Temporarily add project root to sys.path for imports.""" with temporary_project_path(self.project_root): yield @@ -481,7 +593,7 @@ def _load_module_from_path(self, path: Path) -> Any: stat_result = path.stat() except OSError as e: msg = f"Failed to execute migration module {path}: {e}" - raise MigrationLoadError(msg) from e + raise with_cause(MigrationLoadError(msg), e) from e cached = self._module_cache.get(cache_key) if cached is not None and cached[0] == stat_result.st_mtime_ns and cached[1] == stat_result.st_size: @@ -505,7 +617,7 @@ def _load_module_from_path(self, path: Path) -> Any: except Exception as e: sys.modules.pop(module_name, None) msg = f"Failed to execute migration module {path}: {e}" - raise MigrationLoadError(msg) from e + raise with_cause(MigrationLoadError(msg), e) from e self._module_cache[cache_key] = (stat_result.st_mtime_ns, stat_result.st_size, module) return module diff --git a/sqlspec/migrations/runner.py b/sqlspec/migrations/runner.py index 8fc59afae..3aa1ae2bb 100644 --- a/sqlspec/migrations/runner.py +++ b/sqlspec/migrations/runner.py @@ -13,8 +13,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload -from sqlspec.core import SQL -from sqlspec.exceptions import MigrationError +from sqlspec.core.statement import SQL +from sqlspec.exceptions import MigrationError, with_cause from sqlspec.loader import SQLFileLoader from sqlspec.migrations.context import MigrationContext from sqlspec.migrations.loaders import ( @@ -27,17 +27,18 @@ from sqlspec.migrations.templates import TemplateDescriptionHints from sqlspec.migrations.utils import resolve_default_schema as _resolve_default_schema from sqlspec.migrations.version import extract_migration_version, parse_extension_stem, parse_version -from sqlspec.observability import resolve_db_system +from sqlspec.observability._common import resolve_db_system from sqlspec.utils.logging import get_logger, log_with_context from sqlspec.utils.sync_tools import async_ if TYPE_CHECKING: - from collections.abc import Awaitable, Callable, Iterator + from collections.abc import Awaitable, Callable, Generator from sqlspec.config import DatabaseConfigProtocol - from sqlspec.driver import AsyncDriverAdapterBase, SyncDriverAdapterBase + from sqlspec.driver._async import AsyncDriverAdapterBase + from sqlspec.driver._sync import SyncDriverAdapterBase from sqlspec.migrations.base import ExtensionReconcilerDescriptor, LoadedMigrationMetadata - from sqlspec.observability import ObservabilityRuntime + from sqlspec.observability._runtime import ObservabilityRuntime __all__ = ( "AsyncMigrationRunner", @@ -233,7 +234,7 @@ def set_driver(self, driver: "SyncDriverAdapterBase | AsyncDriverAdapterBase") - self.context.driver = driver @contextlib.contextmanager - def use_step_savepoints(self, driver: Any) -> "Iterator[None]": + def use_step_savepoints(self, driver: Any) -> "Generator[None, None, None]": """Isolate transactional migration steps in savepoints when an outer transaction lock is held. Yields: @@ -869,7 +870,7 @@ def _handle_migration_sql_error( ) return msg = f"Failed to load upgrade for migration {migration.get('version')}: {error}" - raise ValueError(msg) from error + raise with_cause(ValueError(msg), error) from error def _finalize_migration_sql(self, sql_statements: Any) -> "list[str] | None": """Normalize loader output into a SQL statement list, or None when absent. @@ -2111,7 +2112,7 @@ def _parse_replaces_tokens(raw_versions: str) -> list[str]: parse_version(version) except ValueError as error: msg = f"Invalid version {version!r} in migration replaces header" - raise MigrationError(msg) from error + raise with_cause(MigrationError(msg), error) from error versions.append(version) return versions diff --git a/sqlspec/migrations/schema.py b/sqlspec/migrations/schema.py index 345e8f3ee..3957f6072 100644 --- a/sqlspec/migrations/schema.py +++ b/sqlspec/migrations/schema.py @@ -8,7 +8,7 @@ from sqlglot.errors import ParseError, TokenError from sqlspec.builder._ddl import AlterTable, ColumnDefinition, CreateTable, _parse_ddl_identifier, _parse_ddl_table -from sqlspec.exceptions import AmbiguousIdentifierError, MigrationError +from sqlspec.exceptions import AmbiguousIdentifierError, MigrationError, with_cause from sqlspec.utils.logging import get_logger, log_with_context from sqlspec.utils.text import resolve_catalog_identifier @@ -1049,7 +1049,7 @@ def _require_unambiguous_names( resolve_catalog_identifier(name, catalog_names) except AmbiguousIdentifierError as exc: msg = f"Cannot reconcile {kind} {name!r} for {target.identity!r}: {exc}" - raise MigrationError(msg) from exc + raise with_cause(MigrationError(msg), exc) from exc def _metadata_names(metadata_rows: Sequence[Any], key: str) -> set[str]: diff --git a/sqlspec/migrations/squash.py b/sqlspec/migrations/squash.py index 1c0bb41af..117553a82 100644 --- a/sqlspec/migrations/squash.py +++ b/sqlspec/migrations/squash.py @@ -25,9 +25,9 @@ from sqlglot.errors import ErrorLevel, ParseError, TokenError, UnsupportedError from sqlglot.tokenizer_core import Token, TokenType -from sqlspec.data_dictionary import DDLResult -from sqlspec.driver import SyncDriverAdapterBase -from sqlspec.exceptions import SquashValidationError +from sqlspec.data_dictionary._types import DDLResult +from sqlspec.driver._sync import SyncDriverAdapterBase +from sqlspec.exceptions import SquashValidationError, with_cause from sqlspec.migrations._backup import create_backup, remove_backup, restore_backup from sqlspec.migrations.base import _ensure_schema_extraction_safe_sync from sqlspec.migrations.loaders import _load_migration_sql @@ -209,7 +209,7 @@ def _split_statements(self, sql: str) -> list[tuple[str, exp.Expr, list[Token]]] tokens = self._dialect_impl.tokenize(sql) except TokenError as error: msg = f"it could not be tokenized ({error})" - raise self._refusal(msg) from error + raise with_cause(self._refusal(msg), error) from error chunks: list[list[Token]] = [[]] for token in tokens: @@ -228,7 +228,7 @@ def _split_statements(self, sql: str) -> list[tuple[str, exp.Expr, list[Token]]] parsed = self._dialect_impl.parser().parse(chunk, sql) except ParseError as error: msg = "SQLGlot cannot parse it" - raise self._refusal(msg) from error + raise with_cause(self._refusal(msg), error) from error statements.extend((text, expr, chunk) for expr in parsed if expr is not None) return statements @@ -1483,7 +1483,7 @@ def _render_expr(self, expr: exp.Expr, entry: "_FoldedStatement") -> str: sql = expr.sql(dialect=self._dialect, unsupported_level=ErrorLevel.RAISE) except UnsupportedError as error: msg = f"SQLGlot cannot render it for this dialect ({error})" - raise self._refusal(msg, entry) from error + raise with_cause(self._refusal(msg, entry), error) from error return _ensure_semicolon(sql) def _is_faithful(self, expr: exp.Expr, tokens: list[Token], spans: list[tuple[exp.Expr, str, int, int]]) -> bool: diff --git a/sqlspec/migrations/templates.py b/sqlspec/migrations/templates.py index 51da54d8a..770aa9734 100644 --- a/sqlspec/migrations/templates.py +++ b/sqlspec/migrations/templates.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Final -from sqlspec.exceptions import SQLSpecError +from sqlspec.exceptions import SQLSpecError, with_cause if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -256,10 +256,10 @@ def _format_template(template: str, context: "Mapping[str, str]", template_kind: except KeyError as exc: # pragma: no cover missing = str(exc).strip("'") msg = f"Missing template variable '{missing}' in {template_kind} template" - raise TemplateValidationError(msg) from exc + raise with_cause(TemplateValidationError(msg), exc) from exc except ValueError as exc: # pragma: no cover msg = f"Invalid {template_kind} template fragment: {exc}" - raise TemplateValidationError(msg) from exc + raise with_cause(TemplateValidationError(msg), exc) from exc _HEADER_UNSAFE_PATTERN: Final[re.Pattern[str]] = re.compile("[\x01-\x1f\x7f-\x9f\u2028\u2029]+") diff --git a/sqlspec/migrations/tracker.py b/sqlspec/migrations/tracker.py index de4fe8fed..80f0dfaad 100644 --- a/sqlspec/migrations/tracker.py +++ b/sqlspec/migrations/tracker.py @@ -15,7 +15,7 @@ from mypy_extensions import mypyc_attr -from sqlspec.exceptions import IntegrityError, MigrationError, SerializationConflictError, SQLSpecError +from sqlspec.exceptions import IntegrityError, MigrationError, SerializationConflictError, SQLSpecError, with_cause from sqlspec.migrations.base import ( _PG_RELATION_GUARD_SQL, BaseMigrationTracker, @@ -24,7 +24,7 @@ ) from sqlspec.migrations.runner import _driver_in_transaction_async, _driver_in_transaction_sync from sqlspec.migrations.schema import ensure_schema_async, ensure_schema_sync -from sqlspec.observability import resolve_db_system +from sqlspec.observability._common import resolve_db_system from sqlspec.utils.logging import get_logger, log_with_context from sqlspec.utils.uuids import uuid4 @@ -34,7 +34,8 @@ from rich.console import Console - from sqlspec.driver import AsyncDriverAdapterBase, SyncDriverAdapterBase + from sqlspec.driver._async import AsyncDriverAdapterBase + from sqlspec.driver._sync import SyncDriverAdapterBase from sqlspec.migrations.base import AppliedMigrationRecord __all__ = ("AsyncMigrationTracker", "SyncMigrationTracker") @@ -1042,7 +1043,7 @@ def _resolve_current_schema(result: Any) -> str: def _validate_pg_relation_rows(rows: "Sequence[Mapping[str, Any]]", version_table: str) -> None: - """Validate pre-existing PostgreSQL tracking relation kind, owner, and trigger state.""" + """Validate pre-existing PostgreSQL tracking relation kind, owner, trigger, rule, and RLS state.""" for row in rows: relkind = row.get("relkind") if isinstance(relkind, str) and relkind.strip() not in {"r", "p"}: @@ -1061,6 +1062,16 @@ def _validate_pg_relation_rows(rows: "Sequence[Mapping[str, Any]]", version_tabl if has_triggers is True or (isinstance(has_triggers, int) and has_triggers > 0): msg = f"Migration tracking table '{version_table}' cannot have non-internal triggers attached" raise MigrationError(msg) + has_rules = row.get("has_rules") + if has_rules is True or (isinstance(has_rules, int) and has_rules > 0): + msg = f"Migration tracking table '{version_table}' cannot have rewrite rules attached" + raise MigrationError(msg) + has_rls = row.get("has_row_security") + if has_rls is True or (isinstance(has_rls, int) and has_rls > 0): + msg = ( + f"Migration tracking table '{version_table}' cannot have row-level security active for the current role" + ) + raise MigrationError(msg) def _log_lock_release_failure(driver: Any, lock_name: str, exc: Exception) -> None: @@ -1190,6 +1201,7 @@ def __enter__(self) -> None: except SQLSpecError as exc: translated = self._tracker._translate_mssql_lock_exception(self._lock_name, self._timeout_seconds, exc) if translated is not None: + translated.__cause__ = exc raise translated from exc raise self._tracker._validate_mssql_lock_result(self._lock_name, ms_result) @@ -1227,7 +1239,7 @@ def _enter_lease(self) -> None: self._driver.rollback() remaining = deadline - _monotonic() if remaining <= 0: - raise _lease_timeout_error(self._lock_name, timeout_seconds) from None + raise with_cause(_lease_timeout_error(self._lock_name, timeout_seconds), None) from None _sleep(min(_LEASE_LOCK_POLL_INTERVAL_SECONDS, remaining)) continue self._owner_token = owner_token @@ -1385,6 +1397,7 @@ async def __aenter__(self) -> None: except SQLSpecError as exc: translated = self._tracker._translate_mssql_lock_exception(self._lock_name, self._timeout_seconds, exc) if translated is not None: + translated.__cause__ = exc raise translated from exc raise self._tracker._validate_mssql_lock_result(self._lock_name, ms_result) @@ -1428,7 +1441,7 @@ async def _enter_lease(self) -> None: await self._driver.rollback() remaining = deadline - _monotonic() if remaining <= 0: - raise _lease_timeout_error(self._lock_name, timeout_seconds) from None + raise with_cause(_lease_timeout_error(self._lock_name, timeout_seconds), None) from None await _async_sleep(min(_LEASE_LOCK_POLL_INTERVAL_SECONDS, remaining)) continue self._owner_token = owner_token diff --git a/sqlspec/migrations/utils.py b/sqlspec/migrations/utils.py index 14f1e98e6..182305f86 100644 --- a/sqlspec/migrations/utils.py +++ b/sqlspec/migrations/utils.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Final, Literal, cast -from sqlspec.exceptions import MigrationError +from sqlspec.exceptions import MigrationError, with_cause from sqlspec.migrations.templates import MigrationTemplateSettings, TemplateValidationError, build_template_settings from sqlspec.utils.adapters import resolve_adapter_name from sqlspec.utils.logging import get_logger @@ -386,7 +386,7 @@ def _resolve_migrations_spec(ext_name: str, spec: str) -> "Path": f"which could not be imported ({exc}); correct " f"extension_config['{ext_name}']['migrations_path'] or install the package." ) - raise MigrationError(msg) from exc + raise with_cause(MigrationError(msg), exc) from exc return module_path / subdir if subdir else module_path @@ -479,7 +479,7 @@ def _raise_callable_error(message: str) -> None: result_value: object = candidate() if param_count == 0 else candidate(config) except Exception as exc: # pragma: no cover msg = f"Author callable '{import_path}' raised an error: {exc}" - raise TemplateValidationError(msg) from exc + raise with_cause(TemplateValidationError(msg), exc) from exc result_str: str = str(result_value) return result_str diff --git a/sqlspec/migrations/validation.py b/sqlspec/migrations/validation.py index e60c7f4f9..52c26ba4e 100644 --- a/sqlspec/migrations/validation.py +++ b/sqlspec/migrations/validation.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from sqlspec.exceptions import OutOfOrderMigrationError, SquashValidationError +from sqlspec.exceptions import OutOfOrderMigrationError, SquashValidationError, with_cause from sqlspec.migrations.version import VersionType, parse_version from sqlspec.utils.logging import get_logger, log_with_context @@ -212,7 +212,7 @@ def validate_squash_range( parsed_end = parse_version(end_version) except ValueError as exc: msg = f"Squash range versions must be numeric: {start_version} to {end_version}" - raise SquashValidationError(msg) from exc + raise with_cause(SquashValidationError(msg), exc) from exc if parsed_start.extension != parsed_end.extension: msg = f"Cannot squash migrations from different extensions: {start_version} to {end_version}" diff --git a/sqlspec/migrations/version.py b/sqlspec/migrations/version.py index c27c6078a..dc24448a4 100644 --- a/sqlspec/migrations/version.py +++ b/sqlspec/migrations/version.py @@ -33,8 +33,7 @@ SEQUENTIAL_PATTERN = re.compile(r"^(?!\d{14}$)\d+$") TIMESTAMP_PATTERN = re.compile(r"^(\d{14})$") -EXTENSION_PATTERN = re.compile(r"^ext_(\w+)_(.+)$") -EXTENSION_STEM_PATTERN = re.compile(r"^ext_(?P\w+?)_(?P\d+)(?:_.*)?$") +_WORD_SEGMENT_RE = re.compile(r"\A\w+\Z") class VersionType(Enum): @@ -232,10 +231,9 @@ def parse_version(version_str: "str | None") -> MigrationVersion: msg = "Invalid migration version: version string is None or empty" raise ValueError(msg) - extension_match = EXTENSION_PATTERN.match(version_str) - if extension_match: - extension_name = extension_match.group(1) - base_version = extension_match.group(2) + extension_parts = _parse_extension_version(version_str) + if extension_parts is not None: + extension_name, base_version = extension_parts parsed = parse_version(base_version) return MigrationVersion( @@ -261,6 +259,16 @@ def parse_version(version_str: "str | None") -> MigrationVersion: raise ValueError(msg) +def _parse_extension_version(version_str: str) -> "tuple[str, str] | None": + """Split ``ext__`` deterministically without regex backtracking.""" + if not version_str.startswith("ext_"): + return None + extension_name, sep, base_version = version_str[4:].rpartition("_") + if not sep or not base_version or _WORD_SEGMENT_RE.fullmatch(extension_name) is None: + return None + return extension_name, base_version + + def parse_extension_stem(stem: str) -> "tuple[str, str] | None": """Split an extension migration filename stem into its extension name and version. @@ -275,10 +283,17 @@ def parse_extension_stem(stem: str) -> "tuple[str, str] | None": Tuple of extension name and version, or None when the stem is not an extension-prefixed migration. """ - match = EXTENSION_STEM_PATTERN.match(stem) - if match is None: + if not stem.startswith("ext_"): return None - return match.group("name"), match.group("version") + parts = stem[4:].split("_") + for index in range(1, len(parts)): + prev = parts[index - 1] + if not prev or _WORD_SEGMENT_RE.fullmatch(prev) is None: + return None + segment = parts[index] + if segment and segment.isascii() and segment.isdigit(): + return "_".join(parts[:index]), segment + return None def extract_migration_version(filename: str) -> "str | None": diff --git a/sqlspec/observability/_common.py b/sqlspec/observability/_common.py index ad82fdae8..641d7e827 100644 --- a/sqlspec/observability/_common.py +++ b/sqlspec/observability/_common.py @@ -3,6 +3,7 @@ import functools import hashlib import re +import sys from typing import Any, Final __all__ = ( @@ -129,15 +130,37 @@ def resolve_effective_db_system(adapter: str, driver: str | None = None, db_syst return resolved +def _resolve_otel_trace() -> Any: + """Resolve and cache the optional ``opentelemetry.trace`` module once per process.""" + global _OTEL_TRACE_MODULE, _OTEL_TRACE_RESOLVED + trace_mod = sys.modules.get("opentelemetry.trace") + if trace_mod is not None: + return trace_mod + otel_pkg = sys.modules.get("opentelemetry") + if otel_pkg is not None: + otel_trace = getattr(otel_pkg, "trace", None) + if otel_trace is not None: + return otel_trace + if _OTEL_TRACE_RESOLVED: + return _OTEL_TRACE_MODULE + try: + from opentelemetry import trace + + _OTEL_TRACE_MODULE = trace + except ImportError: + _OTEL_TRACE_MODULE = None + _OTEL_TRACE_RESOLVED = True + return _OTEL_TRACE_MODULE + + def get_trace_context() -> "tuple[str | None, str | None]": """Extract trace_id and span_id from the current OTel context. Returns: Tuple of trace_id and span_id if available, otherwise (None, None). """ - try: - from opentelemetry import trace - except ImportError: + trace = _resolve_otel_trace() + if trace is None: return None, None span: Any | None = trace.get_current_span() @@ -198,17 +221,9 @@ def redact_sql_literals(sql: str, *, dialect: str | None = None) -> str: def get_w3c_trace_context() -> tuple[str | None, str]: """Extract W3C headers from recording or propagated OpenTelemetry spans.""" - global _OTEL_TRACE_MODULE, _OTEL_TRACE_RESOLVED - if not _OTEL_TRACE_RESOLVED: - try: - from opentelemetry import trace - - _OTEL_TRACE_MODULE = trace - except ImportError: - _OTEL_TRACE_MODULE = None - _OTEL_TRACE_RESOLVED = True - if _OTEL_TRACE_MODULE is not None: - span = _OTEL_TRACE_MODULE.get_current_span() + trace = _resolve_otel_trace() + if trace is not None: + span = trace.get_current_span() context = span.get_span_context() if span is not None else None if context is not None and context.is_valid and context.trace_id and context.span_id: raw_flags = getattr(context, "trace_flags", None) diff --git a/sqlspec/observability/_runtime.py b/sqlspec/observability/_runtime.py index f087b0d40..e34c9cff0 100644 --- a/sqlspec/observability/_runtime.py +++ b/sqlspec/observability/_runtime.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from time import time -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, Final, cast from urllib.parse import urlsplit, urlunsplit from sqlspec.observability._common import ( @@ -22,7 +22,7 @@ if TYPE_CHECKING: from collections.abc import Iterable - from sqlspec.storage import StorageTelemetry + from sqlspec.storage.pipeline import StorageTelemetry __all__ = ("ObservabilityRuntime", "compute_sql_hash") @@ -30,12 +30,15 @@ logger = get_logger("sqlspec.observability.runtime") _MIN_WALL_CLOCK_EPOCH_S = 1_000_000_000.0 +_CORRELATION_UNSET: Final[Any] = object() class ObservabilityRuntime: """Aggregates dispatchers, observers, spans, and custom metrics.""" __slots__ = ( + "_db_system_cache", + "_has_session_lifecycle_cached", "_is_idle_cached", "_metrics", "_redaction", @@ -43,6 +46,7 @@ class ObservabilityRuntime: "bind_key", "config", "config_name", + "hook_generation", "lifecycle", "span_manager", ) @@ -69,7 +73,36 @@ def __init__( self._statement_observers = tuple(observers) self._redaction = config.redaction.copy() if config.redaction else None self._metrics: dict[str, float] = {} - self._is_idle_cached = not self.lifecycle.is_enabled and not self._statement_observers + self._db_system_cache: dict[tuple[str, str | None], str] = {} + self._has_session_lifecycle_cached = False + self._is_idle_cached = True + self.hook_generation = 0 + self._refresh_cached_flags() + + @property + def has_session_lifecycle(self) -> bool: + """Return True when session or connection lifecycle hooks or telemetry spans are active.""" + return self._has_session_lifecycle_cached or self.span_manager.is_enabled + + def _refresh_cached_flags(self) -> None: + """Recompute the cached lifecycle and idle flags from the dispatcher and observers.""" + lifecycle = self.lifecycle + self._has_session_lifecycle_cached = ( + lifecycle.has_connection_create + or lifecycle.has_connection_destroy + or lifecycle.has_session_start + or lifecycle.has_session_end + ) + self._is_idle_cached = not lifecycle.is_enabled and not self._statement_observers + + def _resolve_db_system_cached(self, driver: str, db_system: str | None) -> str: + key = (driver, db_system) + cached = self._db_system_cache.get(key) + if cached is not None: + return cached + resolved = resolve_effective_db_system(self.config_name, driver, db_system) + self._db_system_cache[key] = resolved + return resolved @property def is_idle(self) -> bool: @@ -238,7 +271,8 @@ def register_lifecycle_hook(self, event: str, callback: "LifecycleHook") -> None callback: Callable receiving the lifecycle context. """ self.lifecycle.register_hook(event, callback) # type: ignore[arg-type] - self._is_idle_cached = not self.lifecycle.is_enabled and not self._statement_observers + self._refresh_cached_flags() + self.hook_generation += 1 def emit_connection_create_sync(self, connection: Any) -> None: span = self._start_lifecycle_span("connection.create", subject=connection) @@ -376,10 +410,11 @@ def create_statement_event( script_statement_index: int | None = None, script_statement_count: int | None = None, db_system: str | None = None, + correlation_id: Any = _CORRELATION_UNSET, ) -> StatementEvent: """Create a redacted and enriched StatementEvent instance.""" - correlation_id = CorrelationContext.get() - resolved_db_system = resolve_effective_db_system(self.config_name, driver, db_system) + resolved_correlation_id = CorrelationContext.get() if correlation_id is _CORRELATION_UNSET else correlation_id + resolved_db_system = self._resolve_db_system_cached(driver, db_system) sanitized_sql = self._redact_sql(sql, resolved_db_system) sanitized_params = self._redact_parameters(parameters) sanitized_profile = self._redact_profile(profile, resolved_db_system) @@ -413,7 +448,7 @@ def create_statement_event( is_script=is_script, rows_affected=rows_affected, duration_s=duration_s, - correlation_id=correlation_id, + correlation_id=resolved_correlation_id, storage_backend=storage_backend, started_at=resolved_started_at, sql_hash=sql_hash, @@ -487,6 +522,7 @@ def emit_statement_event( script_statement_index=script_statement_index, script_statement_count=script_statement_count, db_system=db_system, + correlation_id=correlation_id, ) for observer in self._statement_observers: try: @@ -506,7 +542,7 @@ def start_query_span( if self.config.sampling is not None and not self.config.sampling.should_sample(correlation_id=correlation_id): return None - resolved_db_system = resolve_effective_db_system(self.config_name, driver, db_system) + resolved_db_system = self._resolve_db_system_cached(driver, db_system) redacted_sql = ( self._redact_sql(sql, resolved_db_system) if (self._redaction is not None and self._redaction.mask_literals) diff --git a/sqlspec/service/_core.py b/sqlspec/service/_core.py index fa7029b9a..b8f203363 100644 --- a/sqlspec/service/_core.py +++ b/sqlspec/service/_core.py @@ -9,10 +9,12 @@ from typing_extensions import TypeVar -from sqlspec.core import SQL, CursorFilter, CursorPagination, LimitOffsetFilter, OffsetPagination, OrderByFilter -from sqlspec.core.filters import find_filter -from sqlspec.driver import AsyncDriverAdapterBase, SyncDriverAdapterBase -from sqlspec.exceptions import DataError, ImproperConfigurationError, InvalidCursorError, NotFoundError +from sqlspec.core._pagination import CursorPagination, OffsetPagination +from sqlspec.core.filters import CursorFilter, LimitOffsetFilter, OrderByFilter, find_filter +from sqlspec.core.statement import SQL +from sqlspec.driver._async import AsyncDriverAdapterBase +from sqlspec.driver._sync import SyncDriverAdapterBase +from sqlspec.exceptions import DataError, ImproperConfigurationError, InvalidCursorError, NotFoundError, with_cause from sqlspec.utils.logging import get_logger from sqlspec.utils.routing import RoutingContext from sqlspec.utils.sync_tools import exit_async_context @@ -20,10 +22,10 @@ if TYPE_CHECKING: from types import TracebackType - from sqlspec.builder import QueryBuilder + from sqlspec.builder._base import QueryBuilder from sqlspec.core.filters import StatementFilter from sqlspec.core.statement import Statement - from sqlspec.service import SQLSpecAsyncService, SQLSpecSyncService + from sqlspec.service._base import SQLSpecAsyncService, SQLSpecSyncService from sqlspec.typing import SchemaT, StatementParameters _INVALID_CURSOR_VALUE_MESSAGE = "Invalid pagination cursor: value does not match the ordering column" @@ -293,7 +295,7 @@ def _sync_paginate_cursor( except DataError as exc: if cursor_filter.values is None: raise - raise InvalidCursorError(_INVALID_CURSOR_VALUE_MESSAGE) from exc + raise with_cause(InvalidCursorError(_INVALID_CURSOR_VALUE_MESSAGE), exc) from exc page = cursor_filter.build_page(items) if schema_type is None: return page diff --git a/sqlspec/storage/_paths.py b/sqlspec/storage/_paths.py index a9dfa873a..66b87b9ba 100644 --- a/sqlspec/storage/_paths.py +++ b/sqlspec/storage/_paths.py @@ -5,7 +5,7 @@ from typing import Final from urllib.parse import unquote, urlparse -from sqlspec.exceptions import StorageOperationFailedError, StoragePathTraversalError +from sqlspec.exceptions import StorageOperationFailedError, StoragePathTraversalError, with_cause __all__ = ( "FILE_PROTOCOL", @@ -239,7 +239,7 @@ def resolve_storage_path( try: win_rel = PureWindowsPath(path_str).relative_to(PureWindowsPath(base_path)) except ValueError as exc: - raise StoragePathTraversalError(path_str, base_path) from exc + raise with_cause(StoragePathTraversalError(path_str, base_path), exc) from exc if str(win_rel) == ".": return base_path trimmed_base = base_path.rstrip("/\\") @@ -256,7 +256,7 @@ def resolve_storage_path( try: relative = path_obj.relative_to(base_obj) except ValueError as exc: - raise StoragePathTraversalError(path_str, base_path) from exc + raise with_cause(StoragePathTraversalError(path_str, base_path), exc) from exc if str(relative) == ".": return base_path return f"{base_path.rstrip('/')}/{relative.as_posix()}" diff --git a/sqlspec/storage/_staging_sql.py b/sqlspec/storage/_staging_sql.py index b6433ad6d..49e7ad21a 100644 --- a/sqlspec/storage/_staging_sql.py +++ b/sqlspec/storage/_staging_sql.py @@ -10,7 +10,7 @@ from sqlglot.errors import SqlglotError from sqlspec.data_dictionary._registry import normalize_adapter_dialect_name -from sqlspec.exceptions import SQLSpecError +from sqlspec.exceptions import SQLSpecError, with_cause from sqlspec.utils.arrow_helpers import sql_type_from_arrow_type from sqlspec.utils.text import quote_identifier, split_qualified_identifier from sqlspec.utils.uuids import uuid4 @@ -252,7 +252,7 @@ def validate_sql_cast_type(type_sql: str, dialect: str | None = None) -> str: statements = sqlglot.parse(f"SELECT CAST(x AS {stripped})", read=cast_dialect) except SqlglotError as exc: msg = f"Invalid SQL cast type: {type_sql!r}" - raise ValueError(msg) from exc + raise with_cause(ValueError(msg), exc) from exc stmt = statements[0] if len(statements) == 1 else None if not isinstance(stmt, exp.Select) or len(stmt.expressions) != 1: diff --git a/sqlspec/storage/backends/fsspec.py b/sqlspec/storage/backends/fsspec.py index f26cd6d54..9a8c4c46e 100644 --- a/sqlspec/storage/backends/fsspec.py +++ b/sqlspec/storage/backends/fsspec.py @@ -22,6 +22,7 @@ ) from sqlspec.storage._arrow_stream import iter_parquet_row_groups, validate_parquet_stream_options from sqlspec.storage._paths import ( + _GLOB_MAGIC, check_local_delete_target, ensure_path_within_root, is_file_destination, @@ -228,13 +229,16 @@ def resolve_uri(self, path: str | Path) -> str: def read_bytes_sync(self, path: str | Path, **kwargs: Any) -> bytes: """Read bytes from an object synchronously.""" resolved_path = self._resolve_path(path) + read_fn = getattr(self.fs, "cat_file", None) + reader = ( + partial(read_fn, resolved_path, **kwargs) + if callable(read_fn) + else partial(self.fs.cat, resolved_path, **kwargs) + ) result = cast( "bytes", execute_sync_storage_operation( - partial(self.fs.cat, resolved_path, **kwargs), - backend=self.backend_type, - operation="read_bytes", - path=resolved_path, + reader, backend=self.backend_type, operation="read_bytes", path=resolved_path ), ) log_storage_event( @@ -311,12 +315,12 @@ def delete_sync(self, path: str | Path, *, recursive: bool = False, **kwargs: An resolved_path = self._resolve_path(path) if self.protocol in _LOCAL_PROTOCOLS: check_local_delete_target(resolved_path, self.base_path, recursive=recursive) - execute_sync_storage_operation( - partial(self.fs.rm, resolved_path, recursive=recursive, **kwargs), - backend=self.backend_type, - operation="delete", - path=resolved_path, + delete_fn = ( + partial(_delete_fsspec_tree, self.fs, resolved_path, kwargs, self.protocol in _LOCAL_PROTOCOLS) + if recursive + else partial(_delete_fsspec_object, self.fs, resolved_path, kwargs) ) + execute_sync_storage_operation(delete_fn, backend=self.backend_type, operation="delete", path=resolved_path) log_storage_event( "storage.write", backend_type=self.backend_type, @@ -330,7 +334,7 @@ def copy_sync(self, source: str | Path, destination: str | Path, **kwargs: Any) source_path = self._resolve_path(source) dest_path = self._resolve_path(destination) execute_sync_storage_operation( - partial(self.fs.copy, source_path, dest_path, **kwargs), + partial(_copy_fsspec_object, self.fs, source_path, dest_path, kwargs), backend=self.backend_type, operation="copy", path=f"{source_path}->{dest_path}", @@ -349,7 +353,7 @@ def move_sync(self, source: str | Path, destination: str | Path, **kwargs: Any) source_path = self._resolve_path(source) dest_path = self._resolve_path(destination) execute_sync_storage_operation( - partial(self.fs.mv, source_path, dest_path, **kwargs), + partial(_move_fsspec_object, self.fs, source_path, dest_path, kwargs, self.protocol in _LOCAL_PROTOCOLS), backend=self.backend_type, operation="move", path=f"{source_path}->{dest_path}", @@ -798,6 +802,80 @@ def _is_within_local_root(candidate: str | Path, root: str | Path) -> bool: return resolved == root_obj or root_obj in resolved.parents +def _delete_fsspec_object(fs: Any, path: str, options: "dict[str, Any]") -> None: + """Delete one object by its literal key, using ``rm`` when the filesystem has no ``rm_file``.""" + rm_file = getattr(fs, "rm_file", None) + if callable(rm_file): + try: + rm_file(path, **options) + except NotImplementedError: + pass + else: + return + fs.rm(path, recursive=False, **options) + + +def _delete_fsspec_tree(fs: Any, path: str, options: "dict[str, Any]", literal_rm: bool) -> None: + """Delete ``path`` and everything beneath it, never expanding glob characters in the key. + + Args: + fs: The fsspec filesystem. + path: Resolved key or prefix to delete. + options: Additional options forwarded to the filesystem. + literal_rm: Whether ``fs.rm`` already treats every path literally. + + Raises: + FileNotFoundError: If nothing exists at ``path``. + """ + if literal_rm or _GLOB_MAGIC.search(path) is None: + fs.rm(path, recursive=True, **options) + return + found = fs.find(path, withdirs=True, detail=True) + if not found: + raise FileNotFoundError(path) + entries: dict[str, Any] = found if isinstance(found, dict) else {member: {} for member in found} + for member in sorted(entries, reverse=True): + if entries[member].get("type") == "directory": + with contextlib.suppress(FileNotFoundError, FileExistsError, NotImplementedError): + fs.rmdir(member) + else: + _delete_fsspec_object(fs, member, options) + + +def _copy_fsspec_object(fs: Any, source: str, destination: str, options: "dict[str, Any]") -> None: + """Copy one object, treating a source key that contains glob characters literally.""" + if _GLOB_MAGIC.search(source) is None: + fs.copy(source, destination, **options) + return + try: + fs.cp_file(source, destination, **options) + except NotImplementedError: + fs.copy(source, destination, **options) + + +def _move_fsspec_object(fs: Any, source: str, destination: str, options: "dict[str, Any]", literal_mv: bool) -> None: + """Move one object, treating a source key that contains glob characters literally. + + Args: + fs: The fsspec filesystem. + source: Resolved source key. + destination: Resolved destination key. + options: Additional options forwarded to the filesystem. + literal_mv: Whether ``fs.mv`` already treats every path literally. + """ + if literal_mv or _GLOB_MAGIC.search(source) is None: + fs.mv(source, destination, **options) + return + if source == destination: + return + try: + fs.cp_file(source, destination, **options) + except NotImplementedError: + fs.mv(source, destination, **options) + return + _delete_fsspec_object(fs, source, {}) + + def _resolve_fsspec_write_options( protocol: str, overwrite: bool, kwargs: "dict[str, Any]" ) -> "tuple[str, dict[str, Any]]": @@ -969,6 +1047,7 @@ def _close_and_publish_fsspec_stream( file_obj.closed = True mapped = _as_create_conflict(error, destination) if mapped is not error: + mapped.__cause__ = error raise mapped from error raise except BaseException: @@ -984,11 +1063,11 @@ def _close_and_publish_fsspec_stream( assert stage is not None try: file_obj.close() - fs.mv(stage, destination) + _move_fsspec_object(fs, stage, destination, {}, False) finally: with contextlib.suppress(Exception): if fs.exists(stage): - fs.rm(stage) + _delete_fsspec_object(fs, stage, {}) def _abort_fsspec_stream(strategy: "_FsspecStrategy", fs: Any, stage: "str | None", file_obj: Any) -> None: @@ -1013,7 +1092,7 @@ def _abort_fsspec_stream(strategy: "_FsspecStrategy", fs: Any, stage: "str | Non file_obj.close() if stage is not None: with contextlib.suppress(Exception): - fs.rm(stage) + _delete_fsspec_object(fs, stage, {}) async def _abort_fsspec_stream_async(strategy: "_FsspecStrategy", fs: Any, stage: "str | None", file_obj: Any) -> None: diff --git a/sqlspec/storage/backends/local.py b/sqlspec/storage/backends/local.py index 6cb10a148..ab44d065b 100644 --- a/sqlspec/storage/backends/local.py +++ b/sqlspec/storage/backends/local.py @@ -17,7 +17,7 @@ from mypy_extensions import mypyc_attr -from sqlspec.exceptions import FileNotFoundInStorageError, StorageCapabilityError, StoragePathTraversalError +from sqlspec.exceptions import FileNotFoundInStorageError, StorageCapabilityError, StoragePathTraversalError, with_cause from sqlspec.storage._arrow_payload import ( StorageFormat, encode_arrow_to_async_sink, @@ -188,7 +188,7 @@ def read_bytes_sync(self, path: "str | Path", **kwargs: Any) -> bytes: resolved.read_bytes, backend=self.backend_type, operation="read_bytes", path=str(resolved) ) except FileNotFoundInStorageError as error: - raise FileNotFoundError(str(resolved)) from error + raise with_cause(FileNotFoundError(str(resolved)), error) from error def write_bytes_sync(self, path: "str | Path", data: bytes, *, overwrite: bool = True, **kwargs: Any) -> None: """Write bytes to file synchronously.""" @@ -229,7 +229,7 @@ def stream_read_sync(self, path: "str | Path", chunk_size: "int | None" = None, break yield chunk except FileNotFoundError as error: - raise FileNotFoundError(str(resolved)) from error + raise with_cause(FileNotFoundError(str(resolved)), error) from error def list_objects_sync(self, prefix: str = "", recursive: bool = True, **kwargs: Any) -> "list[str]": """List objects in directory synchronously. @@ -329,7 +329,7 @@ def glob_sync(self, pattern: str, **kwargs: Any) -> "list[str]": rel_pattern_path = pattern_path.relative_to(self.base_path) except ValueError as exc: msg = f"Glob pattern resolves outside base_path '{self.base_path}': {pattern!r}" - raise StoragePathTraversalError(msg) from exc + raise with_cause(StoragePathTraversalError(msg), exc) from exc rel_pattern = str(rel_pattern_path) else: rel_pattern = normalized_pattern diff --git a/sqlspec/storage/backends/obstore.py b/sqlspec/storage/backends/obstore.py index 09dd4ab04..efb6fbb16 100644 --- a/sqlspec/storage/backends/obstore.py +++ b/sqlspec/storage/backends/obstore.py @@ -19,7 +19,12 @@ from mypy_extensions import mypyc_attr -from sqlspec.exceptions import StorageCapabilityError, StorageOperationFailedError, StoragePathTraversalError +from sqlspec.exceptions import ( + StorageCapabilityError, + StorageOperationFailedError, + StoragePathTraversalError, + with_cause, +) from sqlspec.storage._arrow_payload import ( StorageFormat, encode_arrow_payload, @@ -195,7 +200,7 @@ def __init__(self, uri: str, **kwargs: Any) -> None: raise except Exception as exc: msg = f"Failed to initialize obstore backend for {sanitize_uri_for_log(uri)}" - raise StorageOperationFailedError(msg) from exc + raise with_cause(StorageOperationFailedError(msg), exc) from exc @property def supports_signing(self) -> bool: @@ -410,7 +415,7 @@ def glob_sync(self, pattern: str, **kwargs: Any) -> "list[str]": try: resolved_pattern = pattern_path.relative_to(Path(self._local_store_root)).as_posix() except ValueError as exc: - raise StoragePathTraversalError(pattern, self._local_store_root) from exc + raise with_cause(StoragePathTraversalError(pattern, self._local_store_root), exc) from exc else: resolved_pattern = pattern else: @@ -781,7 +786,7 @@ def sign_sync( return signed_urls[0] if is_single else signed_urls except Exception as exc: msg = f"Failed to generate signed URL(s) for {resolved_paths}" - raise StorageOperationFailedError(msg) from exc + raise with_cause(StorageOperationFailedError(msg), exc) from exc async def read_bytes_async(self, path: "str | Path", **kwargs: Any) -> bytes: """Read bytes from storage asynchronously.""" @@ -1196,7 +1201,7 @@ async def sign_async( return signed_urls[0] if is_single else signed_urls except Exception as exc: msg = f"Failed to generate signed URL(s) for {resolved_paths}" - raise StorageOperationFailedError(msg) from exc + raise with_cause(StorageOperationFailedError(msg), exc) from exc def _supports_atomic_copy_create(self) -> bool: """Return whether the underlying store supports atomic conditional copy (overwrite=False).""" diff --git a/sqlspec/storage/errors.py b/sqlspec/storage/errors.py index fa4c47a2a..096427cab 100644 --- a/sqlspec/storage/errors.py +++ b/sqlspec/storage/errors.py @@ -4,7 +4,12 @@ import logging from typing import TYPE_CHECKING, NoReturn, TypeVar -from sqlspec.exceptions import FileExistsInStorageError, FileNotFoundInStorageError, StorageOperationFailedError +from sqlspec.exceptions import ( + FileExistsInStorageError, + FileNotFoundInStorageError, + StorageOperationFailedError, + with_cause, +) from sqlspec.utils.logging import get_logger, log_with_context if TYPE_CHECKING: @@ -56,14 +61,14 @@ def raise_storage_error(error: Exception, *, backend: str, operation: str, path: if is_missing: log_with_context(logger, logging.INFO, "storage.object.missing", **log_extra) - raise FileNotFoundInStorageError(normalized.message) from error + raise with_cause(FileNotFoundInStorageError(normalized.message), error) from error if is_exists: log_with_context(logger, logging.INFO, "storage.object.exists", **log_extra) - raise FileExistsInStorageError(normalized.message) from error + raise with_cause(FileExistsInStorageError(normalized.message), error) from error log_with_context(logger, logging.WARNING, "storage.operation.failed", **log_extra) - raise StorageOperationFailedError(normalized.message) from error + raise with_cause(StorageOperationFailedError(normalized.message), error) from error def execute_sync_storage_operation(func: "Callable[[], T]", *, backend: str, operation: str, path: str | None) -> T: diff --git a/sqlspec/storage/pipeline.py b/sqlspec/storage/pipeline.py index 3723bf863..6b5dd3090 100644 --- a/sqlspec/storage/pipeline.py +++ b/sqlspec/storage/pipeline.py @@ -25,8 +25,9 @@ from sqlspec.storage._utils import import_pyarrow from sqlspec.storage.backends.base import AsyncArrowBatchIterator from sqlspec.storage.errors import execute_async_storage_operation, execute_sync_storage_operation -from sqlspec.storage.registry import StorageRegistry, storage_registry -from sqlspec.utils.serializers import get_serializer_metrics, serialize_collection, to_json +from sqlspec.storage.registry import _CLOUD_OBJECT_STORE_SCHEMES, StorageRegistry, storage_registry +from sqlspec.utils.serializers._json import to_json +from sqlspec.utils.serializers._schema import get_serializer_metrics, serialize_collection from sqlspec.utils.sync_tools import async_, exit_async_context from sqlspec.utils.type_guards import ( supports_async_read_bytes, @@ -209,7 +210,6 @@ def reset(self) -> None: _EMPTY_STORAGE_OPTIONS: dict[str, Any] = {} _ARROW_WRITE_FORMATS = frozenset({"parquet", "arrow-ipc", "csv"}) _ROW_WRITE_FORMATS = frozenset({"json", "jsonl"}) -_CLOUD_OBJECT_STORE_SCHEMES = frozenset({"s3", "gs", "gcs", "az", "azure"}) def _strip_write_options(options: "dict[str, Any] | None") -> "dict[str, Any]": diff --git a/sqlspec/storage/registry.py b/sqlspec/storage/registry.py index 1aa6616b1..bc235029d 100644 --- a/sqlspec/storage/registry.py +++ b/sqlspec/storage/registry.py @@ -33,7 +33,16 @@ FSSPEC_ONLY_SCHEMES: Final[frozenset[str]] = frozenset({"http", "https", "ftp", "sftp", "ssh"}) -_CLOUD_OBJECT_STORE_SCHEMES: Final[frozenset[str]] = frozenset({"s3", "gs", "gcs", "az", "azure"}) +_CLOUD_OBJECT_STORE_SCHEMES: Final[frozenset[str]] = frozenset({ + "s3", + "s3a", + "gs", + "gcs", + "az", + "azure", + "abfs", + "abfss", +}) @mypyc_attr(allow_interpreted_subclasses=True) diff --git a/sqlspec/utils/arrow_helpers.py b/sqlspec/utils/arrow_helpers.py index 226c58800..3b5fabc40 100644 --- a/sqlspec/utils/arrow_helpers.py +++ b/sqlspec/utils/arrow_helpers.py @@ -1482,7 +1482,7 @@ def _emit_fallback( def _encode_json_bytes(val: Any) -> bytes: """Encode value to JSON bytes using msgspec or stdlib json without circular imports.""" - encoder = import_optional_attr("sqlspec.utils.serializers._json", "encode_json") + encoder = import_optional_attr("sqlspec.utils.serializers._json", "to_json") if encoder is not None: return cast("bytes", encoder(val, as_bytes=True)) import json diff --git a/sqlspec/utils/config_tools.py b/sqlspec/utils/config_tools.py index 7081ce214..a4b668fe6 100644 --- a/sqlspec/utils/config_tools.py +++ b/sqlspec/utils/config_tools.py @@ -8,14 +8,17 @@ import inspect import os +import stat import tomllib from collections.abc import Sequence +from collections.abc import Set as AbstractSet from pathlib import Path from types import ModuleType -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, Literal, cast from urllib.parse import parse_qs, unquote, urlsplit -from sqlspec.exceptions import ConfigResolverError, ImproperConfigurationError +from sqlspec.exceptions import ConfigResolverError, ImproperConfigurationError, with_cause +from sqlspec.utils.logging import get_logger from sqlspec.utils.module_loader import import_string from sqlspec.utils.sync_tools import async_, await_ from sqlspec.utils.type_guards import ( @@ -34,13 +37,17 @@ "discover_config_from_pyproject", "find_pyproject_toml", "normalize_connection_config", + "normalize_extension_status_codes", "parse_mysql_dsn", "parse_odbc_connection_string", "parse_pyproject_config", "resolve_config_async", "resolve_config_sync", + "resolve_extension_commit_mode", ) +logger = get_logger("sqlspec.utils.config_tools") + def discover_config_from_pyproject() -> str | None: """Find and parse pyproject.toml for SQLSpec config. @@ -55,7 +62,10 @@ def discover_config_from_pyproject() -> str | None: if pyproject_path is None: return None - return parse_pyproject_config(pyproject_path) + data = _load_trusted_pyproject(pyproject_path) + if data is None: + return None + return _config_from_pyproject_data(data, pyproject_path) def find_pyproject_toml() -> "Path | None": @@ -63,8 +73,13 @@ def find_pyproject_toml() -> "Path | None": Starts from current working directory and walks up to filesystem root. Stops at .git directory boundary (repository root) if found. On POSIX, - stops at the first directory owned by another non-root user and refuses - configuration files owned by another non-root user. + stops at the first directory owned by another non-root user and refuses a + configuration file that another non-root user owns. A symlinked + configuration file is accepted when the link and its target both pass the + ownership test. Inside a shared world-writable directory (sticky, or owned + by a different user) the file must also be singly linked and not + world-writable. Every refusal is logged at warning level. Platforms + without ``os.getuid`` skip the ownership and permission tests. Returns: Path to pyproject.toml, or None if not found. @@ -73,16 +88,17 @@ def find_pyproject_toml() -> "Path | None": current = Path.cwd() while True: - if not _is_owned_by_user_or_root(current, current_uid): + if not _is_safe_directory(current, current_uid): return None pyproject = current / "pyproject.toml" - if pyproject.exists(): - if not _is_owned_by_user_or_root(pyproject, current_uid): - return None + pyproject_status = _inspect_path_trust(pyproject, current_uid) + if pyproject_status is True: return pyproject + if pyproject_status is False: + return None - if (current / ".git").exists(): + if _path_exists_or_symlink(current / ".git"): return None if current == current.parent: @@ -108,8 +124,50 @@ def parse_pyproject_config(pyproject_path: "Path") -> str | None: data = tomllib.load(f) except Exception as e: msg = f"Failed to parse {pyproject_path}: {e}" - raise ValueError(msg) from e + raise with_cause(ValueError(msg), e) from e + + return _config_from_pyproject_data(data, pyproject_path) + + +def _load_trusted_pyproject(pyproject_path: "Path") -> "dict[str, Any] | None": + """Read a discovered pyproject.toml, applying the ownership test to the file that was opened. + + The file is opened once, and its ownership and permissions are read from the open + descriptor before its content is parsed, so the file that is checked is the file that is read. + + Args: + pyproject_path: Path returned by ``find_pyproject_toml``. + + Returns: + The parsed document, or None when the opened file is not trusted. + + Raises: + ValueError: If the file cannot be read or parsed. + """ + current_uid: int | None = os.getuid() if hasattr(os, "getuid") else None + try: + with pyproject_path.open("rb") as f: + if current_uid is not None and not _is_trusted_file_stat(pyproject_path, os.fstat(f.fileno()), current_uid): + return None + return tomllib.load(f) + except Exception as e: + msg = f"Failed to parse {pyproject_path}: {e}" + raise with_cause(ValueError(msg), e) from e + + +def _config_from_pyproject_data(data: "dict[str, Any]", pyproject_path: "Path") -> str | None: + """Return the ``[tool.sqlspec].config`` value of a parsed pyproject document. + + Args: + data: Parsed pyproject.toml content. + pyproject_path: Path the content was read from, used in error messages. + Returns: + Config path(s) as string (converts list to comma-separated), or None if not found. + + Raises: + ValueError: If [tool.sqlspec].config has invalid type (not str or list[str]). + """ tool_section = data.get("tool", {}) if not isinstance(tool_section, dict): return None @@ -155,7 +213,7 @@ async def resolve_config_async( config_obj = import_string(_normalize_config_path(config_path)) except ImportError as e: msg = f"Failed to import config from path '{config_path}': {e}" - raise ConfigResolverError(msg) from e + raise with_cause(ConfigResolverError(msg), e) from e if not callable(config_obj): return _validate_config_result(config_obj, config_path) @@ -167,7 +225,7 @@ async def resolve_config_async( result = await async_(config_obj)() except Exception as e: msg = f"Failed to execute callable config '{config_path}': {e}" - raise ConfigResolverError(msg) from e + raise with_cause(ConfigResolverError(msg), e) from e return _validate_config_result(result, config_path) @@ -187,7 +245,7 @@ def resolve_config_sync( config_obj = import_string(_normalize_config_path(config_path)) except ImportError as e: msg = f"Failed to import config from path '{config_path}': {e}" - raise ConfigResolverError(msg) from e + raise with_cause(ConfigResolverError(msg), e) from e if not callable(config_obj): return _validate_config_result(config_obj, config_path) @@ -199,7 +257,7 @@ def resolve_config_sync( result = config_obj() except Exception as e: msg = f"Failed to execute callable config '{config_path}': {e}" - raise ConfigResolverError(msg) from e + raise with_cause(ConfigResolverError(msg), e) from e return _validate_config_result(result, config_path) @@ -509,10 +567,146 @@ def _is_valid_config(config: Any) -> bool: def _is_owned_by_user_or_root(path: Path, current_uid: int | None) -> bool: + try: + path_stat = path.stat(follow_symlinks=False) + except OSError: + return False + mode = getattr(path_stat, "st_mode", 0) + if stat.S_ISLNK(mode): + return False if current_uid is None: return True + return path_stat.st_uid in {current_uid, 0} + + +def _is_safe_directory(path: Path, current_uid: int | None) -> bool: + """Return True when ``path`` is a non-symlinked directory that the current user or root owns.""" + try: + path_stat = path.stat(follow_symlinks=False) + except OSError as exc: + logger.warning("Stopping pyproject.toml discovery at %s: directory cannot be inspected (%s)", path, exc) + return False + if stat.S_ISLNK(getattr(path_stat, "st_mode", 0)): + logger.warning("Stopping pyproject.toml discovery at %s: directory is a symbolic link", path) + return False + if current_uid is not None and path_stat.st_uid not in {current_uid, 0}: + logger.warning( + "Stopping pyproject.toml discovery at %s: directory is owned by another user (uid %s)", + path, + path_stat.st_uid, + ) + return False + return True + + +def _is_shared_directory(path: Path, current_uid: int) -> bool: + """Return True when ``path`` is world-writable and either sticky or owned by a different user.""" try: - path_stat = path.stat() + path_stat = path.stat(follow_symlinks=False) except OSError: + return True + mode = getattr(path_stat, "st_mode", 0) + if not mode & stat.S_IWOTH: return False - return path_stat.st_uid in {current_uid, 0} + return bool(mode & stat.S_ISVTX) or path_stat.st_uid != current_uid + + +def _inspect_path_trust(path: Path, current_uid: int | None) -> "bool | None": + """Return True if present and trusted, False if present and untrusted, or None if missing. + + On POSIX the current user or root must own ``path`` and, for a symbolic link, its target. + Inside a shared world-writable directory ``path`` must also be singly linked unless it is + a directory, and its target must not be world-writable. + """ + try: + link_stat = path.stat(follow_symlinks=False) + except FileNotFoundError: + return None + except OSError as exc: + logger.warning("Refusing %s: it cannot be inspected (%s)", path, exc) + return False + link_mode = getattr(link_stat, "st_mode", 0) + target_stat = link_stat + if stat.S_ISLNK(link_mode): + try: + target_stat = path.stat() + except OSError as exc: + logger.warning("Refusing %s: symbolic link target cannot be inspected (%s)", path, exc) + return False + if current_uid is None: + return True + if link_stat.st_uid not in {current_uid, 0}: + logger.warning("Refusing %s: it is owned by another user (uid %s)", path, link_stat.st_uid) + return False + return _is_trusted_file_stat(path, target_stat, current_uid) + + +def _is_trusted_file_stat(path: Path, file_stat: Any, current_uid: int) -> bool: + """Return True when ``file_stat``, the status of the file at ``path``, passes the ownership test. + + The current user or root must own the file. Inside a shared world-writable directory the + file must also be singly linked unless it is a directory, and must not be world-writable. + """ + if file_stat.st_uid not in {current_uid, 0}: + logger.warning("Refusing %s: it is owned by another user (uid %s)", path, file_stat.st_uid) + return False + if not _is_shared_directory(path.parent, current_uid): + return True + mode = getattr(file_stat, "st_mode", 0) + if not stat.S_ISDIR(mode) and getattr(file_stat, "st_nlink", 1) > 1: + logger.warning("Refusing %s: it has several hard links inside a shared world-writable directory", path) + return False + if mode & stat.S_IWOTH: + logger.warning("Refusing %s: it is world-writable inside a shared world-writable directory", path) + return False + return True + + +def normalize_extension_status_codes(value: Any) -> Any: + """Return framework status-code overrides as a ``set``, treating empty values as unset. + + Args: + value: The configured ``extra_commit_statuses`` or ``extra_rollback_statuses`` value. + + Returns: + ``None`` for unset or empty values, a ``set`` for any set-like value, list, or tuple, + and any other value unchanged. + """ + if value is None or isinstance(value, set): + return value + if not value: + return None + if isinstance(value, (AbstractSet, list, tuple)): + return set(value) + return value + + +def resolve_extension_commit_mode( + config: Any, + framework_config: "Mapping[str, Any]", + default_commit_mode: Literal["manual", "autocommit", "autocommit_include_redirect"] = "autocommit", +) -> Literal["manual", "autocommit", "autocommit_include_redirect"]: + """Resolve web extension ``commit_mode``, requiring manual mode on non-transactional configs.""" + explicit = framework_config.get("commit_mode") + if not getattr(config, "supports_transactions", True): + if explicit is not None and explicit != "manual": + msg = ( + f"{type(config).__name__} does not support transactions " + f"(supports_transactions=False); commit_mode={explicit!r} is not supported, use 'manual'." + ) + raise ImproperConfigurationError(msg) + return "manual" + if explicit is None: + return default_commit_mode + return cast("Literal['manual', 'autocommit', 'autocommit_include_redirect']", explicit) + + +def _path_exists_or_symlink(path: Path) -> bool: + """Return True when ``path`` exists or is a symlink (including dangling symlinks).""" + try: + path.stat(follow_symlinks=False) + except FileNotFoundError: + return False + except OSError: + return True + return True diff --git a/sqlspec/utils/env.py b/sqlspec/utils/env.py index 0f0729b4c..4711ca661 100644 --- a/sqlspec/utils/env.py +++ b/sqlspec/utils/env.py @@ -5,6 +5,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Final, get_args, get_origin, overload +from sqlspec.exceptions import with_cause + if TYPE_CHECKING: from collections.abc import Callable, Sequence @@ -305,13 +307,13 @@ def _coerce_item(key: str, value: Any, item_type: object) -> Any: return int(value) except (TypeError, ValueError): msg = f"Cannot convert list item for key '{key}' to int" - raise ValueError(msg) from None + raise with_cause(ValueError(msg), None) from None if item_type is float: try: return float(value) except (TypeError, ValueError): msg = f"Cannot convert list item for key '{key}' to float" - raise ValueError(msg) from None + raise with_cause(ValueError(msg), None) from None if item_type is str or item_type is _UNSET: return str(value) if isinstance(item_type, type): @@ -319,7 +321,7 @@ def _coerce_item(key: str, value: Any, item_type: object) -> Any: return item_type(value) except (TypeError, ValueError): msg = f"Cannot convert list item for key '{key}'" - raise ValueError(msg) from None + raise with_cause(ValueError(msg), None) from None return value @@ -331,13 +333,13 @@ def _parse_basic_type(key: str, value: str, target_type: object) -> Any: return int(value) except ValueError: msg = f"Cannot convert value for key '{key}' to int" - raise ValueError(msg) from None + raise with_cause(ValueError(msg), None) from None if target_type is float: try: return float(value) except ValueError: msg = f"Cannot convert value for key '{key}' to float" - raise ValueError(msg) from None + raise with_cause(ValueError(msg), None) from None if target_type is Path: return Path(value) return value @@ -350,7 +352,7 @@ def _parse_dict(key: str, value: str) -> dict[str, Any]: parsed = json.loads(stripped) except json.JSONDecodeError: msg = f"'{key}' is not valid JSON" - raise ValueError(msg) from None + raise with_cause(ValueError(msg), None) from None if not isinstance(parsed, dict): msg = f"'{key}' is not a valid dict representation" raise ValueError(msg) @@ -376,7 +378,7 @@ def _parse_list(key: str, value: str, item_type: object) -> list[Any]: parsed = json.loads(stripped) except json.JSONDecodeError: msg = f"'{key}' is not valid JSON" - raise ValueError(msg) from None + raise with_cause(ValueError(msg), None) from None if not isinstance(parsed, list): msg = f"'{key}' is not a valid list representation" raise ValueError(msg) diff --git a/sqlspec/utils/fixtures.py b/sqlspec/utils/fixtures.py index e008cd1a2..c04eb0de3 100644 --- a/sqlspec/utils/fixtures.py +++ b/sqlspec/utils/fixtures.py @@ -23,19 +23,21 @@ from sqlglot import Dialect, exp from sqlspec import dialects as _dialects -from sqlspec.core.parameters import TypedParameter +from sqlspec.core.parameters._types import TypedParameter +from sqlspec.core.query_modifiers import sanitize_parameter_name from sqlspec.core.type_converter import convert_iso_date, convert_iso_datetime, convert_uuid -from sqlspec.storage import storage_registry +from sqlspec.exceptions import with_cause +from sqlspec.storage.registry import storage_registry from sqlspec.utils.logging import get_logger -from sqlspec.utils.serializers import from_json as decode_json -from sqlspec.utils.serializers import schema_dump -from sqlspec.utils.serializers import to_json as encode_json +from sqlspec.utils.serializers._json import from_json, to_json +from sqlspec.utils.serializers._schema import schema_dump from sqlspec.utils.sync_tools import async_ if TYPE_CHECKING: from sqlglot.dialects.dialect import DialectType - from sqlspec.driver import AsyncDriverAdapterBase, SyncDriverAdapterBase + from sqlspec.driver._async import AsyncDriverAdapterBase + from sqlspec.driver._sync import SyncDriverAdapterBase from sqlspec.typing import SupportedSchemaModel _ = _dialects @@ -168,7 +170,7 @@ def open_fixture_sync(fixtures_path: Any, fixture_name: str) -> Any: with fixture_path.open(mode="r", encoding="utf-8") as f: f_data = f.read() - return decode_json(f_data) + return from_json(f_data) async def open_fixture_async(fixtures_path: Any, fixture_name: str) -> Any: @@ -196,7 +198,7 @@ async def open_fixture_async(fixtures_path: Any, fixture_name: str) -> Any: else: f_data = await _async_read_text(fixture_path) - return decode_json(f_data) + return from_json(f_data) def write_fixture_sync( @@ -231,7 +233,7 @@ def write_fixture_sync( storage = storage_registry.get(uri, **storage_kwargs) except Exception as exc: msg = f"Failed to get storage backend for '{storage_backend}': {exc}" - raise ValueError(msg) from exc + raise with_cause(ValueError(msg), exc) from exc json_content = _serialize_data(data) @@ -276,7 +278,7 @@ async def write_fixture_async( storage = storage_registry.get(uri, **storage_kwargs) except Exception as exc: msg = f"Failed to get storage backend for '{storage_backend}': {exc}" - raise ValueError(msg) from exc + raise with_cause(ValueError(msg), exc) from exc json_content = await _async_serialize(data) @@ -874,11 +876,11 @@ def _serialize_data(data: Any) -> str: else: serialized_items.append(schema_dump(item)) - return encode_json(serialized_items) + return to_json(serialized_items) if isinstance(data, (str, int, float, bool, type(None))): - return encode_json(data) + return to_json(data) - return encode_json(schema_dump(data)) + return to_json(schema_dump(data)) def _validated_fixture_name(name: str) -> str: @@ -1156,9 +1158,9 @@ def _table_fixture_rows(file_path: Path, content: str) -> "list[dict[str, Any]]" """ decoded: Any if file_path.name.endswith((".jsonl", ".jsonl.gz")): - decoded = [decode_json(line) for line in content.splitlines() if line.strip()] + decoded = [from_json(line) for line in content.splitlines() if line.strip()] else: - decoded = decode_json(content) + decoded = from_json(content) if not isinstance(decoded, list): msg = f"Table fixture {file_path} must contain a JSON array of row objects" raise TypeError(msg) @@ -1382,7 +1384,7 @@ def _decode_exported_json_value(value: Any) -> Any: if not isinstance(value, str): return value try: - return decode_json(value) + return from_json(value) except (ValueError, TypeError): return value @@ -1643,7 +1645,7 @@ def _sqlite_vector_converter(value: Any) -> Any: return value if is_blob: return struct.pack(f"<{len(items)}f", *items) - return encode_json(items) + return to_json(items) return _sqlite_vector_converter @@ -1663,7 +1665,7 @@ def _column_value_decoder(family: str, data_type: str) -> "Callable[[Any], Any] """ normalized_type = data_type.lower() if family == "snowflake" and normalized_type in {"variant", "object", "array"}: - return encode_json + return to_json if _is_vector_column(normalized_type): if family == "postgres": return _pg_vector_decoder(normalized_type) @@ -1683,7 +1685,7 @@ def _column_value_decoder(family: str, data_type: str) -> "Callable[[Any], Any] if _is_binary_type_info(data_type): return _decode_bytes if data_type in _JSON_TYPE_NAMES: - return encode_json if family in _JSON_VALUE_FAMILIES else None + return to_json if family in _JSON_VALUE_FAMILIES else None if family == "sqlite": return None if data_type.startswith(("timestamp", "datetime", "bigdatetime", "smalldatetime", "datetimetz")): @@ -1755,6 +1757,22 @@ def _conflict_clause( return exp.OnConflict(conflict_keys=keys, action=exp.var("DO UPDATE"), expressions=assignments) +def _fixture_placeholder_map(columns: "Sequence[str]") -> "dict[str, str]": + """Return a collision-free mapping from fixture column names to safe placeholder names.""" + mapping: dict[str, str] = {} + used: set[str] = set() + for column in columns: + base = sanitize_parameter_name(column) + candidate = base + suffix = 2 + while candidate in used: + candidate = f"{base}_{suffix}" + suffix += 1 + used.add(candidate) + mapping[column] = candidate + return mapping + + def _table_insert_statement( dialect: "DialectType", family: str, @@ -1763,6 +1781,7 @@ def _table_insert_statement( conflict_keys: "tuple[str, ...]", table_columns: "list[_TableColumn]", excluded_updates: "set[str] | None" = None, + placeholder_names: "Mapping[str, str] | None" = None, ) -> str: """Return INSERT SQL text with a named placeholder per quoted column, upserting on conflict keys. @@ -1771,10 +1790,11 @@ def _table_insert_statement( ``GENERATED ALWAYS`` identity columns carries ``OVERRIDING SYSTEM VALUE``. """ col_map = {col.name: col for col in table_columns} + param_map = placeholder_names if placeholder_names is not None else _fixture_placeholder_map(columns) value_expressions: list[exp.Expr] = [] for column in columns: col = col_map.get(column) - ph = exp.Placeholder(this=column) + ph = exp.Placeholder(this=param_map[column]) if col is not None and family == "snowflake" and col.data_type in {"variant", "object", "array"}: value_expressions.append(exp.Anonymous(this="PARSE_JSON", expressions=[ph])) elif col is not None and family == "postgres" and _postgres_vector_kind(col.data_type): @@ -1852,19 +1872,25 @@ def _insert_table_rows_sync( return 0 _decode_row_values(rows, family, columns) inserted_columns = list(rows[0]) + param_map = _fixture_placeholder_map(inserted_columns) statement = _table_insert_statement( - dialect, family, table, inserted_columns, conflict_keys, columns, excluded_updates + dialect, family, table, inserted_columns, conflict_keys, columns, excluded_updates, placeholder_names=param_map + ) + bound_rows = ( + [{param_map.get(key, key): value for key, value in row.items()} for row in rows] + if any(key != value for key, value in param_map.items()) + else rows ) identity_insert = _identity_insert_statements(dialect, family, table, inserted_columns, columns) if identity_insert is not None: driver.execute(identity_insert[0]) try: - for start in range(0, len(rows), batch_size): - driver.execute_many(statement, rows[start : start + batch_size]) + for start in range(0, len(bound_rows), batch_size): + driver.execute_many(statement, bound_rows[start : start + batch_size]) finally: if identity_insert is not None: driver.execute(identity_insert[1]) - return len(rows) + return len(bound_rows) async def _insert_table_rows_async( @@ -1888,23 +1914,29 @@ async def _insert_table_rows_async( return 0 _decode_row_values(rows, family, columns) inserted_columns = list(rows[0]) + param_map = _fixture_placeholder_map(inserted_columns) statement = _table_insert_statement( - dialect, family, table, inserted_columns, conflict_keys, columns, excluded_updates + dialect, family, table, inserted_columns, conflict_keys, columns, excluded_updates, placeholder_names=param_map + ) + bound_rows = ( + [{param_map.get(key, key): value for key, value in row.items()} for row in rows] + if any(key != value for key, value in param_map.items()) + else rows ) identity_insert = _identity_insert_statements(dialect, family, table, inserted_columns, columns) if identity_insert is not None: await driver.execute(identity_insert[0]) insert_error: BaseException | None = None try: - for start in range(0, len(rows), batch_size): - await driver.execute_many(statement, rows[start : start + batch_size]) + for start in range(0, len(bound_rows), batch_size): + await driver.execute_many(statement, bound_rows[start : start + batch_size]) except BaseException as exc: insert_error = exc if identity_insert is not None: await driver.execute(identity_insert[1]) if insert_error is not None: raise insert_error - return len(rows) + return len(bound_rows) def _is_orderable_type(family: str, data_type: str) -> bool: @@ -2310,8 +2342,8 @@ def _table_fixture_text(rows: "list[dict[str, Any]]", jsonl: bool) -> str: """Serialize table rows as a JSON array or as JSON lines, encoding bytes as base64 text.""" exportable = [{key: _exportable_value(value) for key, value in row.items()} for row in rows] if not jsonl: - return encode_json(exportable) - return "".join(f"{encode_json(row)}\n" for row in exportable) + return to_json(exportable) + return "".join(f"{to_json(row)}\n" for row in exportable) def _partition_table_fixture_shards( @@ -2333,7 +2365,7 @@ def _partition_table_fixture_shards( current_lines: list[str] = [] current_bytes = 0 for row in exportable: - line = f"{encode_json(row)}\n" + line = f"{to_json(row)}\n" line_bytes = len(line.encode("utf-8")) if current_lines and ( (max_rows_per_shard is not None and len(current_lines) >= max_rows_per_shard) @@ -2351,7 +2383,7 @@ def _partition_table_fixture_shards( current_rows: list[str] = [] current_bytes = 2 for row in exportable: - row_json = encode_json(row) + row_json = to_json(row) row_bytes = len(row_json.encode("utf-8")) row_cost = row_bytes if not current_rows else (1 + row_bytes) if current_rows and ( diff --git a/sqlspec/utils/logging.py b/sqlspec/utils/logging.py index 9fa946c2a..cd1563923 100644 --- a/sqlspec/utils/logging.py +++ b/sqlspec/utils/logging.py @@ -248,7 +248,7 @@ def _to_json_str(data: Any) -> str: """Serialize log record dict to JSON string using cached serializer lookup.""" global _TO_JSON_FN if _TO_JSON_FN is None: - from sqlspec.utils.serializers import to_json + from sqlspec.utils.serializers._json import to_json _TO_JSON_FN = to_json return cast("str", _TO_JSON_FN(data)) @@ -261,7 +261,7 @@ def _get_trace_context() -> "tuple[str | None, str | None]": Tuple of (trace_id, span_id) or (None, None) if unavailable. """ try: - from sqlspec.observability import get_trace_context + from sqlspec.observability._common import get_trace_context except Exception: return (None, None) return get_trace_context() diff --git a/sqlspec/utils/module_loader.py b/sqlspec/utils/module_loader.py index 560e643bc..3bfa9f316 100644 --- a/sqlspec/utils/module_loader.py +++ b/sqlspec/utils/module_loader.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, TypeVar, cast -from sqlspec.exceptions import MissingDependencyError, SQLSpecError +from sqlspec.exceptions import MissingDependencyError, SQLSpecError, with_cause if TYPE_CHECKING: from types import ModuleType @@ -216,7 +216,7 @@ def module_to_os_path(dotted_path: str = "app") -> "Path": try: src = find_spec(dotted_path) except ModuleNotFoundError as e: - raise ModuleNotFoundError(msg, name=dotted_path) from e + raise with_cause(ModuleNotFoundError(msg, name=dotted_path), e) from e if src is None: raise ModuleNotFoundError(msg, name=dotted_path) @@ -284,8 +284,8 @@ def import_string(dotted_path: str) -> "Any": _raise_import_error(f"Could not import '{dotted_path}': {e}", e) except SQLSpecError: raise - except Exception as e: - _raise_import_error(f"Could not import '{dotted_path}': {e}", e) + except Exception as exc: + _raise_import_error(f"Could not import '{dotted_path}': {exc}", exc) return obj @@ -385,7 +385,7 @@ def _require_dependency( def _raise_import_error(msg: str, exc: "Exception | None" = None) -> None: """Raise an ImportError with optional exception chaining.""" if exc is not None: - raise ImportError(msg) from exc + raise with_cause(ImportError(msg), exc) from exc raise ImportError(msg) diff --git a/sqlspec/utils/portal.py b/sqlspec/utils/portal.py index bc2cf59df..4c29f4206 100644 --- a/sqlspec/utils/portal.py +++ b/sqlspec/utils/portal.py @@ -13,7 +13,7 @@ import threading from typing import TYPE_CHECKING, Any, ClassVar, Final, Self, TypeVar, cast -from sqlspec.exceptions import ImproperConfigurationError +from sqlspec.exceptions import ImproperConfigurationError, with_cause from sqlspec.utils.logging import get_logger if TYPE_CHECKING: @@ -166,7 +166,7 @@ def call(self, func: "Callable[..., Coroutine[Any, Any, _R]]", *args: Any, **kwa if future_holder: future_holder[0].cancel() msg = f"Portal call timed out after {timeout} seconds" - raise ImproperConfigurationError(msg) from None + raise with_cause(ImproperConfigurationError(msg), None) from None if exception is not None: raise exception diff --git a/sqlspec/utils/routing.py b/sqlspec/utils/routing.py index 45c4293fa..b05130f94 100644 --- a/sqlspec/utils/routing.py +++ b/sqlspec/utils/routing.py @@ -391,12 +391,12 @@ def sign_primary_pin( def verify_primary_pin( - token: str, secret: str | bytes, *, bind_key: str | None = None, clock: Callable[[], float] = time.time + token: Any, secret: str | bytes, *, bind_key: str | None = None, clock: Callable[[], float] = time.time ) -> tuple[bool, str | None]: """Verify an HMAC-signed primary pin cookie value and check expiration and bind key. Args: - token: Raw cookie value supplied by the client. + token: Raw cookie value supplied by the client; a missing or non-string value is invalid. secret: Shared HMAC secret key (non-empty ``str`` or ``bytes``). bind_key: Logical database bind key expected for this check. clock: Wall-clock epoch provider in seconds. @@ -466,7 +466,9 @@ def validate_routing_middleware_settings( routing_header.encode("ascii") except UnicodeEncodeError as exc: msg = "routing_header must be None or a non-empty printable ASCII header name" - raise ImproperConfigurationError(msg) from exc + error = ImproperConfigurationError(msg) + error.__cause__ = exc + raise error from exc if primary_pin_cookie is not None and ( not isinstance(primary_pin_cookie, str) or not primary_pin_cookie.strip() diff --git a/sqlspec/utils/schema.py b/sqlspec/utils/schema.py index d6036af23..87538a501 100644 --- a/sqlspec/utils/schema.py +++ b/sqlspec/utils/schema.py @@ -12,11 +12,11 @@ from typing_extensions import TypeVar -from sqlspec.exceptions import SQLSpecError +from sqlspec.exceptions import SQLSpecError, with_cause from sqlspec.typing import CATTRS_INSTALLED, NUMPY_INSTALLED, MsgspecValidationError, SchemaT, convert, get_type_adapter from sqlspec.utils.dispatch import TypeDispatcher from sqlspec.utils.module_loader import import_optional_attr -from sqlspec.utils.serializers._json import decode_json as from_json +from sqlspec.utils.serializers._json import from_json from sqlspec.utils.type_guards import ( is_attrs_instance, is_attrs_schema, @@ -187,7 +187,7 @@ def to_value_type(value: Any, value_type: "type[ValueT]") -> "ValueT": return value_type(value) # type: ignore[call-arg] except (TypeError, ValueError) as e: msg = f"Cannot convert {type(value).__name__} to {value_type.__name__}" - raise TypeError(msg) from e + raise with_cause(TypeError(msg), e) from e def _safe_convert_key(key: Any, converter: Callable[[str], str]) -> Any: @@ -773,7 +773,7 @@ def _convert_to_int(value: Any) -> int: return int(value) except (ValueError, OverflowError) as e: msg = f"Cannot convert {type(value).__name__} to int" - raise TypeError(msg) from e + raise with_cause(TypeError(msg), e) from e if isinstance(value, str): try: return int(value) diff --git a/sqlspec/utils/serializers/__init__.py b/sqlspec/utils/serializers/__init__.py index 5ba2229a4..3167ce9e6 100644 --- a/sqlspec/utils/serializers/__init__.py +++ b/sqlspec/utils/serializers/__init__.py @@ -6,9 +6,7 @@ from sqlspec import _COMPILED if TYPE_CHECKING: - from sqlspec.utils.serializers._json import DEFAULT_TYPE_ENCODERS, TypeEncodersMap - from sqlspec.utils.serializers._json import decode_json as from_json - from sqlspec.utils.serializers._json import encode_json as to_json + from sqlspec.utils.serializers._json import DEFAULT_TYPE_ENCODERS, TypeEncodersMap, from_json, to_json from sqlspec.utils.serializers._numpy import numpy_array_dec_hook, numpy_array_enc_hook from sqlspec.utils.serializers._schema import ( SchemaSerializer, @@ -38,7 +36,7 @@ "DEFAULT_TYPE_ENCODERS": ("sqlspec.utils.serializers._json", "DEFAULT_TYPE_ENCODERS"), "SchemaSerializer": ("sqlspec.utils.serializers._schema", "SchemaSerializer"), "TypeEncodersMap": ("sqlspec.utils.serializers._json", "TypeEncodersMap"), - "from_json": ("sqlspec.utils.serializers._json", "decode_json"), + "from_json": ("sqlspec.utils.serializers._json", "from_json"), "get_collection_serializer": ("sqlspec.utils.serializers._schema", "get_collection_serializer"), "get_serializer_metrics": ("sqlspec.utils.serializers._schema", "get_serializer_metrics"), "numpy_array_dec_hook": ("sqlspec.utils.serializers._numpy", "numpy_array_dec_hook"), @@ -46,7 +44,7 @@ "reset_serializer_cache": ("sqlspec.utils.serializers._schema", "reset_serializer_cache"), "schema_dump": ("sqlspec.utils.serializers._schema", "schema_dump"), "serialize_collection": ("sqlspec.utils.serializers._schema", "serialize_collection"), - "to_json": ("sqlspec.utils.serializers._json", "encode_json"), + "to_json": ("sqlspec.utils.serializers._json", "to_json"), } diff --git a/sqlspec/utils/serializers/_json.py b/sqlspec/utils/serializers/_json.py index 823659487..7e944e3ea 100644 --- a/sqlspec/utils/serializers/_json.py +++ b/sqlspec/utils/serializers/_json.py @@ -16,6 +16,7 @@ if TYPE_CHECKING: from typing import Self +from sqlspec.exceptions import with_cause from sqlspec.typing import MSGSPEC_INSTALLED, NUMPY_INSTALLED, ORJSON_INSTALLED, PYDANTIC_INSTALLED, UNSET from sqlspec.utils.logging import get_logger from sqlspec.utils.module_loader import import_optional_attr @@ -32,9 +33,9 @@ "TypeEncodersMap", "convert_date_to_iso", "convert_datetime_to_gmt_iso", - "decode_json", - "encode_json", + "from_json", "get_default_serializer", + "to_json", ) @@ -291,7 +292,7 @@ def encode(self, data: Any, *, as_bytes: bool = False) -> "str | bytes": raise if "type is not json serializable" in str(exc).lower(): unsupported_msg = "unsupported JSON value" - raise TypeError(unsupported_msg) from exc + raise with_cause(TypeError(unsupported_msg), exc) from exc raise return encoded if as_bytes else encoded.decode("utf-8") @@ -346,19 +347,19 @@ def get_default_serializer() -> JSONSerializer: @overload -def encode_json(data: Any, *, as_bytes: Literal[False] = ...) -> str: ... +def to_json(data: Any, *, as_bytes: Literal[False] = ...) -> str: ... @overload -def encode_json(data: Any, *, as_bytes: Literal[True]) -> bytes: ... +def to_json(data: Any, *, as_bytes: Literal[True]) -> bytes: ... -def encode_json(data: Any, *, as_bytes: bool = False) -> "str | bytes": +def to_json(data: Any, *, as_bytes: bool = False) -> "str | bytes": """Encode Python data into JSON.""" return get_default_serializer().encode(data, as_bytes=as_bytes) -def decode_json(data: "str | bytes", *, decode_bytes: bool = True) -> Any: +def from_json(data: "str | bytes", *, decode_bytes: bool = True) -> Any: """Decode JSON input into Python data.""" return get_default_serializer().decode(data, decode_bytes=decode_bytes) diff --git a/sqlspec/utils/sync_tools.py b/sqlspec/utils/sync_tools.py index 023b363be..9742e73e8 100644 --- a/sqlspec/utils/sync_tools.py +++ b/sqlspec/utils/sync_tools.py @@ -19,6 +19,7 @@ from typing_extensions import ParamSpec +from sqlspec.exceptions import with_cause from sqlspec.utils.env import get_env from sqlspec.utils.module_loader import module_available from sqlspec.utils.portal import get_global_portal @@ -425,7 +426,7 @@ def __call__(self, *args: "ParamSpecT.args", **kwargs: "ParamSpecT.kwargs") -> " except RuntimeError: if self._raise_sync_error: msg = "Cannot run async function" - raise RuntimeError(msg) from None + raise with_cause(RuntimeError(msg), None) from None portal = get_global_portal() typed_partial = cast("Callable[[], Coroutine[Any, Any, ReturnT]]", partial_f) return portal.call(typed_partial) diff --git a/sqlspec/utils/type_guards.py b/sqlspec/utils/type_guards.py index cff985bda..c5a8e4e3a 100644 --- a/sqlspec/utils/type_guards.py +++ b/sqlspec/utils/type_guards.py @@ -44,8 +44,8 @@ from typing import TypeGuard from sqlspec._typing import AttrsInstanceStub, BaseModelStub, DTODataStub, StructStub - from sqlspec.core import StatementFilter - from sqlspec.core.parameters import TypedParameter + from sqlspec.core.filters import StatementFilter + from sqlspec.core.parameters._types import TypedParameter from sqlspec.protocols import ( ArrowTableStatsProtocol, AsyncDeleteProtocol, @@ -1335,7 +1335,7 @@ def is_typed_parameter(obj: Any) -> "TypeGuard[TypedParameter]": Returns: True if the object is a TypedParameter, False otherwise """ - from sqlspec.core.parameters import TypedParameter + from sqlspec.core.parameters._types import TypedParameter return isinstance(obj, TypedParameter) diff --git a/tests/integration/adapters/_shared/behaviors.py b/tests/integration/adapters/_shared/behaviors.py index 9adde4e5e..9d7d52428 100644 --- a/tests/integration/adapters/_shared/behaviors.py +++ b/tests/integration/adapters/_shared/behaviors.py @@ -5,7 +5,7 @@ import json import math import sqlite3 -from collections.abc import Awaitable, Callable, Iterator, Mapping +from collections.abc import Awaitable, Callable, Mapping from decimal import Decimal from typing import TYPE_CHECKING, Any, Literal, Protocol, cast from uuid import NAMESPACE_DNS, UUID, uuid1, uuid4, uuid5 @@ -51,6 +51,8 @@ ) if TYPE_CHECKING: + from collections.abc import Generator + from sqlspec.protocols import AsyncDataDictionaryProtocol, SyncDataDictionaryProtocol from sqlspec.typing import ArrowRecordBatch @@ -4553,7 +4555,7 @@ def _oracle_lob_select_sql(table: str) -> str: @contextlib.contextmanager -def _oracle_driver_feature(driver: object, key: str, value: object) -> "Iterator[None]": +def _oracle_driver_feature(driver: object, key: str, value: object) -> "Generator[None, None, None]": features = cast("Any", driver).driver_features missing = object() previous = features.get(key, missing) diff --git a/tests/integration/adapters/_shared/transaction_behaviors.py b/tests/integration/adapters/_shared/transaction_behaviors.py index 09efdd706..f73f7af1f 100644 --- a/tests/integration/adapters/_shared/transaction_behaviors.py +++ b/tests/integration/adapters/_shared/transaction_behaviors.py @@ -1,7 +1,7 @@ """Nested transaction block behaviors shared by adapter contract tests.""" import contextlib -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncGenerator, Generator from typing import Any, cast import pytest @@ -30,7 +30,7 @@ def __init__(self, driver: Any) -> None: self.sessions = 0 @contextlib.contextmanager - def provide_session(self) -> Iterator[Any]: + def provide_session(self) -> Generator[Any, None, None]: self.sessions += 1 yield self.driver @@ -46,7 +46,7 @@ def __init__(self, driver: Any) -> None: self.sessions = 0 @contextlib.asynccontextmanager - async def provide_session(self) -> AsyncIterator[Any]: + async def provide_session(self) -> AsyncGenerator[Any, None]: self.sessions += 1 yield self.driver diff --git a/tests/integration/adapters/db2/test_tenant.py b/tests/integration/adapters/db2/test_tenant.py index 0db1159f4..753bc6f01 100644 --- a/tests/integration/adapters/db2/test_tenant.py +++ b/tests/integration/adapters/db2/test_tenant.py @@ -1,6 +1,6 @@ """Db2 RCAC row isolation through verified session global variables.""" -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncGenerator, Iterator from contextlib import asynccontextmanager from inspect import isawaitable from typing import Any @@ -28,7 +28,7 @@ async def _invoke(function: Any, *args: Any) -> Any: @asynccontextmanager -async def _session(config: Any) -> AsyncIterator[Any]: +async def _session(config: Any) -> AsyncGenerator[Any, None]: if isinstance(config, Db2AsyncConfig): async with config.provide_session() as session: yield session diff --git a/tests/integration/adapters/mysql/aiomysql/test_config.py b/tests/integration/adapters/mysql/aiomysql/test_config.py index a6fc70b29..66855ef48 100644 --- a/tests/integration/adapters/mysql/aiomysql/test_config.py +++ b/tests/integration/adapters/mysql/aiomysql/test_config.py @@ -11,6 +11,7 @@ AiomysqlPoolParams, ) from sqlspec.core import StatementConfig +from sqlspec.exceptions import SQLSpecError pytestmark = pytest.mark.xdist_group("mysql") @@ -221,3 +222,30 @@ def custom_serializer(data: object) -> str: ) assert config.driver_features["json_serializer"] is custom_serializer + + +async def test_aiomysql_multi_statement_disabled_by_default(mysql_service: MySQLService) -> None: + """Default AiomysqlConfig rejects stacked SQL via COM_SET_OPTION across checkouts while allowing execute_script.""" + config = AiomysqlConfig( + connection_config={ + "host": mysql_service.host, + "port": mysql_service.port, + "user": mysql_service.user, + "password": mysql_service.password, + "db": mysql_service.db, + "minsize": 1, + "maxsize": 1, + } + ) + try: + async with config.provide_session() as session: + with pytest.raises(SQLSpecError): + await session.execute("SELECT 1; SELECT 2") + script_result = await session.execute_script("SELECT 1; SELECT 2") + assert script_result is not None + async with config.provide_session() as session: + with pytest.raises(SQLSpecError): + await session.execute("SELECT 1; SELECT 2") + assert await session.select_value("SELECT 42") == 42 + finally: + await config.close_pool() diff --git a/tests/integration/adapters/mysql/asyncmy/test_config.py b/tests/integration/adapters/mysql/asyncmy/test_config.py index e98aaa0bd..bdafa0c4e 100644 --- a/tests/integration/adapters/mysql/asyncmy/test_config.py +++ b/tests/integration/adapters/mysql/asyncmy/test_config.py @@ -11,6 +11,7 @@ AsyncmyPoolParams, ) from sqlspec.core import StatementConfig +from sqlspec.exceptions import SQLSpecError pytestmark = pytest.mark.xdist_group("mysql") @@ -218,3 +219,30 @@ def custom_serializer(data: object) -> str: ) assert config.driver_features["json_serializer"] is custom_serializer + + +async def test_asyncmy_multi_statement_disabled_by_default(mysql_service: MySQLService) -> None: + """Default AsyncmyConfig rejects stacked SQL via COM_SET_OPTION across checkouts while allowing execute_script.""" + config = AsyncmyConfig( + connection_config={ + "host": mysql_service.host, + "port": mysql_service.port, + "user": mysql_service.user, + "password": mysql_service.password, + "database": mysql_service.db, + "minsize": 1, + "maxsize": 1, + } + ) + try: + async with config.provide_session() as session: + with pytest.raises(SQLSpecError): + await session.execute("SELECT 1; SELECT 2") + script_result = await session.execute_script("SELECT 1; SELECT 2") + assert script_result is not None + async with config.provide_session() as session: + with pytest.raises(SQLSpecError): + await session.execute("SELECT 1; SELECT 2") + assert await session.select_value("SELECT 42") == 42 + finally: + await config.close_pool() diff --git a/tests/integration/adapters/postgres/psycopg/test_async_copy.py b/tests/integration/adapters/postgres/psycopg/test_async_copy.py index 8b4638958..70d743a45 100644 --- a/tests/integration/adapters/postgres/psycopg/test_async_copy.py +++ b/tests/integration/adapters/postgres/psycopg/test_async_copy.py @@ -1,5 +1,6 @@ -"""Integration tests for psycopg async driver statement-stack pipelines.""" +"""Integration tests for psycopg async driver statement-stack pipelines and COPY.""" +import asyncio from collections.abc import AsyncGenerator import pytest @@ -7,6 +8,7 @@ from sqlspec import StatementStack from sqlspec.adapters.psycopg import PsycopgAsyncConfig, PsycopgAsyncDriver +from sqlspec.exceptions import SQLSpecError pytestmark = pytest.mark.xdist_group("postgres") @@ -119,3 +121,82 @@ async def test_psycopg_async_statement_stack_fail_fast_reports_operation_index( await psycopg_async_session.execute_stack(stack) assert excinfo.value.operation_index == 1 + + +async def test_psycopg_async_statement_stack_accepts_every_parameter_shape( + psycopg_async_session: PsycopgAsyncDriver, +) -> None: + """Tuple, list, scalar, named and parameterless statements all run through the native pipeline.""" + await psycopg_async_session.execute_script("DELETE FROM test_table_psycopg_async") + + insert = "INSERT INTO test_table_psycopg_async (id, name, value) VALUES (%s, %s, %s)" + stack = ( + StatementStack() + .push_execute(insert, (1, "tuple", 10)) + .push_execute(insert, [2, "list", 20]) + .push_execute(insert, 3, "scalars", 30) + .push_execute( + "INSERT INTO test_table_psycopg_async (id, name, value) VALUES (:id, :name, :value)", + id=4, + name="named", + value=40, + ) + .push_execute("INSERT INTO test_table_psycopg_async (id, name, value) VALUES (5, 'literal', 50)") + .push_execute("SELECT name FROM test_table_psycopg_async ORDER BY id") + ) + + results = await psycopg_async_session.execute_stack(stack) + + assert [result.rows_affected for result in results[:5]] == [1, 1, 1, 1, 1] + assert results[5].get_result().get_data() == [ + {"name": "tuple"}, + {"name": "list"}, + {"name": "scalars"}, + {"name": "named"}, + {"name": "literal"}, + ] + + +async def test_psycopg_async_copy_to_stdout_returns_rows(psycopg_async_session: PsycopgAsyncDriver) -> None: + """COPY TO STDOUT issued through execute returns the exported text.""" + await psycopg_async_session.execute_script("DELETE FROM test_table_psycopg_async") + await psycopg_async_session.execute_many( + "INSERT INTO test_table_psycopg_async (id, name, value) VALUES (%s, %s, %s)", [(1, "a", 10), (2, "b", 20)] + ) + + result = await psycopg_async_session.execute( + "COPY (SELECT id, name, value FROM test_table_psycopg_async ORDER BY id) TO STDOUT" + ) + + assert result.get_data() == [{"copy_output": "1\ta\t10\n2\tb\t20\n"}] + + +async def test_psycopg_async_copy_from_stdin_reports_rejected_data(psycopg_async_session: PsycopgAsyncDriver) -> None: + """Data the server rejects when COPY finishes raises a mapped error and leaves the session usable.""" + await psycopg_async_session.execute_script("DELETE FROM test_table_psycopg_async") + + with pytest.raises(SQLSpecError): + await psycopg_async_session.execute( + "COPY test_table_psycopg_async (id, name, value) FROM STDIN", "not-a-number\ta\t1\n" + ) + + verify = await psycopg_async_session.execute("SELECT COUNT(*) AS total FROM test_table_psycopg_async") + assert verify.get_data()[0]["total"] == 0 + + +async def test_psycopg_async_statement_stack_cancellation_finishes_and_keeps_session_usable( + psycopg_async_session: PsycopgAsyncDriver, +) -> None: + """Cancelling a running pipeline ends the call promptly and the session works again after rollback.""" + stack = StatementStack().push_execute("SELECT pg_sleep(%s)", (30,)) + task = asyncio.ensure_future(psycopg_async_session.execute_stack(stack)) + await asyncio.sleep(0.5) + + task.cancel() + done, _ = await asyncio.wait({task}, timeout=15) + + assert task in done + assert task.cancelled() + await psycopg_async_session.rollback() + verify = await psycopg_async_session.execute("SELECT 1 AS value") + assert verify.get_data() == [{"value": 1}] diff --git a/tests/integration/adapters/postgres/psycopg/test_driver.py b/tests/integration/adapters/postgres/psycopg/test_driver.py index 76e2aca2c..ba7dcf596 100644 --- a/tests/integration/adapters/postgres/psycopg/test_driver.py +++ b/tests/integration/adapters/postgres/psycopg/test_driver.py @@ -1,6 +1,6 @@ """Integration tests for psycopg driver implementation.""" -from collections.abc import AsyncIterator, Generator +from collections.abc import AsyncGenerator, Generator from typing import Any, cast import pytest @@ -354,7 +354,7 @@ async def invoke(call: Any, *args: Any) -> Any: return await value if inspect.isawaitable(value) else value @asynccontextmanager - async def session_scope() -> AsyncIterator[Any]: + async def session_scope() -> AsyncGenerator[Any, None]: if asynchronous: async with cast("PsycopgAsyncConfig", config).provide_session() as session: yield session diff --git a/tests/integration/adapters/spanner/spanner/_modes.py b/tests/integration/adapters/spanner/spanner/_modes.py index 333b607d0..719038163 100644 --- a/tests/integration/adapters/spanner/spanner/_modes.py +++ b/tests/integration/adapters/spanner/spanner/_modes.py @@ -1,6 +1,6 @@ """Helpers that run one Spanner test body against the sync and async adapters.""" -from collections.abc import AsyncIterator, Awaitable +from collections.abc import AsyncGenerator, Awaitable from contextlib import asynccontextmanager from inspect import isawaitable from typing import Any, Literal, TypeVar @@ -27,7 +27,7 @@ async def invoke(result: "T | Awaitable[T]") -> T: @asynccontextmanager async def mode_session( config: SpannerModeConfig, kind: Literal["session", "read", "write"] = "session", **kwargs: Any -) -> "AsyncIterator[Any]": +) -> "AsyncGenerator[Any, None]": """Open a default, read, or write session on either adapter.""" provider: Any = getattr(config, _PROVIDERS[kind]) if isinstance(config, SpannerAsyncConfig): diff --git a/tests/integration/adapters/sqlite/aiosqlite/test_load_from_arrow_transaction.py b/tests/integration/adapters/sqlite/aiosqlite/test_load_from_arrow_transaction.py index a90a75bb8..2ca18332a 100644 --- a/tests/integration/adapters/sqlite/aiosqlite/test_load_from_arrow_transaction.py +++ b/tests/integration/adapters/sqlite/aiosqlite/test_load_from_arrow_transaction.py @@ -1,10 +1,14 @@ """aiosqlite load_from_arrow transactional atomicity.""" +import asyncio +from pathlib import Path + import pyarrow as pa +import pyarrow.parquet as pq import pytest from sqlspec.adapters.aiosqlite import AiosqliteConfig -from sqlspec.exceptions import SQLSpecError +from sqlspec.exceptions import SQLSpecError, UniqueViolationError pytestmark = pytest.mark.xdist_group("sqlite") @@ -42,3 +46,60 @@ async def test_load_from_arrow_respects_caller_transaction() -> None: assert result.get_data()[0]["c"] == 0 finally: await config.close_pool() + + +@pytest.mark.parametrize("loader", ["arrow", "arrow_drop_indexes", "storage"]) +async def test_failed_load_raises_unique_violation_and_session_stays_usable(tmp_path: Path, loader: str) -> None: + duplicate_keys = pa.table({"id": [1, 1], "v": ["a", "b"]}) + parquet_path = tmp_path / "duplicate.parquet" + pq.write_table(duplicate_keys, parquet_path) + config = AiosqliteConfig(connection_config={"database": str(tmp_path / "failed_load.db")}) + try: + async with config.provide_session() as driver: + await driver.execute_script("CREATE TABLE failed_load (id INTEGER PRIMARY KEY, v TEXT)") + if loader == "storage": + load = driver.load_from_storage("failed_load", str(parquet_path), file_format="parquet") + else: + load = driver.load_from_arrow( + "failed_load", duplicate_keys, drop_indexes=loader == "arrow_drop_indexes" + ) + + with pytest.raises(UniqueViolationError): + await asyncio.wait_for(load, timeout=10) + + assert not driver.connection.in_transaction + assert await driver.select_value("SELECT COUNT(*) FROM failed_load") == 0 + await driver.load_from_arrow("failed_load", pa.table({"id": [1, 2], "v": ["a", "b"]})) + assert await driver.select_value("SELECT COUNT(*) FROM failed_load") == 2 + finally: + await config.close_pool() + + +async def test_cancelled_load_rolls_back_and_session_stays_usable(tmp_path: Path) -> None: + row_count = 50_000 + loop = asyncio.get_running_loop() + insert_started = asyncio.Event() + + def signal_first_insert(statement: str) -> None: + if statement.startswith("INSERT") and not insert_started.is_set(): + loop.call_soon_threadsafe(insert_started.set) + + config = AiosqliteConfig(connection_config={"database": str(tmp_path / "cancelled_load.db")}) + try: + async with config.provide_session() as driver: + await driver.execute_script("CREATE TABLE cancelled_load (id INTEGER PRIMARY KEY, v TEXT)") + arrow_table = pa.table({"id": list(range(row_count)), "v": ["x"] * row_count}) + await driver.connection.set_trace_callback(signal_first_insert) + load = asyncio.ensure_future(driver.load_from_arrow("cancelled_load", arrow_table)) + await asyncio.wait_for(insert_started.wait(), timeout=10) + load.cancel() + await asyncio.wait({load}, timeout=10) + await driver.connection.set_trace_callback(None) + + assert load.cancelled() + assert not driver.connection.in_transaction + assert await driver.select_value("SELECT COUNT(*) FROM cancelled_load") == 0 + await driver.load_from_arrow("cancelled_load", pa.table({"id": [1, 2], "v": ["a", "b"]})) + assert await driver.select_value("SELECT COUNT(*) FROM cancelled_load") == 2 + finally: + await config.close_pool() diff --git a/tests/integration/adapters/sqlite/test_features.py b/tests/integration/adapters/sqlite/test_features.py index f9fd5b85d..2374caa7e 100644 --- a/tests/integration/adapters/sqlite/test_features.py +++ b/tests/integration/adapters/sqlite/test_features.py @@ -2,10 +2,12 @@ import inspect import math +import sqlite3 from collections.abc import AsyncGenerator, Callable from typing import Any, cast import pytest +from sqlglot import parse_one from sqlspec import SQLResult, StatementStack, sql from sqlspec.adapters.aiosqlite import AiosqliteConfig, AiosqliteDriver @@ -13,6 +15,7 @@ from sqlspec.adapters.sqlite import SqliteConfig, SqliteDriver from sqlspec.adapters.sqlite import core as sqlite_core from sqlspec.exceptions import SQLBuilderError +from sqlspec.utils.correlation import CorrelationContext from tests.conftest import requires_interpreted pytestmark = pytest.mark.xdist_group("sqlite") @@ -207,6 +210,54 @@ def counted_lookup(connection: object, target: "tuple[str | None, str]") -> bool assert lookup_count == 1 +@pytest.mark.parametrize("adapter", ["sqlite", "aiosqlite"], ids=["sync", "async"]) +async def test_rowid_eligibility_probe_is_shared_across_sqlcommenter_comments(adapter: str) -> None: + config: SqliteConfig | AiosqliteConfig = ( + SqliteConfig(connection_config={"database": ":memory:"}) if adapter == "sqlite" else AiosqliteConfig() + ) + config.statement_config = config.statement_config.replace( + enable_sqlcommenter=True, sqlcommenter_enable_context=True + ) + session = config.provide_session() + traced: list[str] = [] + try: + driver = await _invoke(getattr(session, "__enter__" if adapter == "sqlite" else "__aenter__")) + await _invoke( + _method(driver, "execute_script"), "CREATE TABLE commented_rowid (id INTEGER PRIMARY KEY, value TEXT)" + ) + await _invoke(driver.connection.set_trace_callback, traced.append) + for request_number in range(200): + with CorrelationContext.context(f"request-{request_number}"): + result = await _invoke( + _method(driver, "execute"), "INSERT INTO commented_rowid (value) VALUES (?)", ("value",) + ) + assert result.last_inserted_id == request_number + 1 + await _invoke(driver.connection.set_trace_callback, None) + + inserts = {statement for statement in traced if statement.startswith("INSERT INTO commented_rowid")} + probes = [statement for statement in traced if "table_list" in statement.lower()] + assert len(inserts) == 200 + assert len(probes) == 1 + assert list(driver._rowid_target_cache) == [(None, "commented_rowid")] + finally: + await _invoke(getattr(session, "__exit__" if adapter == "sqlite" else "__aexit__"), None, None, None) + await _invoke(config.close_pool) + + +@pytest.mark.parametrize("rowid_core", [sqlite_core, aiosqlite_core], ids=["sync", "async"]) +def test_rowid_eligibility_cache_stays_bounded_across_many_targets(rowid_core: Any) -> None: + connection = sqlite3.connect(":memory:") + try: + cursor = connection.execute("SELECT 1") + eligibility_cache: dict[Any, bool] = {} + for table_number in range(600): + expression = parse_one(f"INSERT INTO bounded_{table_number} (value) VALUES (1)", dialect="sqlite") + rowid_core.resolve_lastrowid(connection, cursor, "INSERT", 1, expression, eligibility_cache) + assert 0 < len(eligibility_cache) <= 512 + finally: + connection.close() + + async def test_schema_change_invalidates_rowid_eligibility_cache(sqlite_family_driver: "SQLiteFamilyDriver") -> None: prefix = _prefix(sqlite_family_driver) table = f"{prefix}_rowid_replaced" diff --git a/tests/integration/adapters/test_tenant_commit_semantics.py b/tests/integration/adapters/test_tenant_commit_semantics.py index fe001ea03..1b499c3e6 100644 --- a/tests/integration/adapters/test_tenant_commit_semantics.py +++ b/tests/integration/adapters/test_tenant_commit_semantics.py @@ -1,6 +1,6 @@ """Tenant leases keep each adapter's native session commit behavior.""" -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncGenerator, Callable from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass from inspect import isawaitable @@ -169,7 +169,7 @@ async def _invoke(function: Any, *args: Any, **kwargs: Any) -> Any: @asynccontextmanager -async def _session(config: Any) -> "AsyncIterator[Any]": +async def _session(config: Any) -> "AsyncGenerator[Any, None]": context = config.provide_session() if isinstance(context, AbstractAsyncContextManager): async with context as session: diff --git a/tests/integration/fixtures/oracle_events.py b/tests/integration/fixtures/oracle_events.py index 64537441c..c31b3d36d 100644 --- a/tests/integration/fixtures/oracle_events.py +++ b/tests/integration/fixtures/oracle_events.py @@ -1,7 +1,7 @@ """Shared provisioning helpers for Oracle events-backend integration tests.""" import contextlib -from collections.abc import Callable, Iterator +from collections.abc import Callable, Generator import pytest from pytest_databases.docker.oracle import OracleService @@ -32,7 +32,7 @@ def classic_aq_queue( queue_table: str = DEFAULT_QUEUE_TABLE, queue_name: str = DEFAULT_QUEUE_NAME, payload_type: str = "JSON", -) -> Iterator[None]: +) -> Generator[None, None, None]: """Provision and tear down a classic Advanced Queuing queue as the app user.""" config = _app_config(oracle_service) @@ -94,7 +94,7 @@ def _factory( @contextlib.contextmanager def transactional_event_queue( oracle_service: OracleService, *, queue_name: str = DEFAULT_TXEVENTQ_NAME, payload_type: str = "JSON" -) -> Iterator[None]: +) -> Generator[None, None, None]: """Provision and tear down a Transactional Event Queue as the app user.""" config = _app_config(oracle_service) diff --git a/tests/integration/migrations/test_tracking_table_guard_postgres.py b/tests/integration/migrations/test_tracking_table_guard_postgres.py index 7ff9d05e9..a4bb357a3 100644 --- a/tests/integration/migrations/test_tracking_table_guard_postgres.py +++ b/tests/integration/migrations/test_tracking_table_guard_postgres.py @@ -71,6 +71,17 @@ async def _close(config: Any) -> None: await result +async def _applied_versions(config: Any, table: str) -> list[str]: + session_context = config.provide_session() + if hasattr(session_context, "__aenter__"): + async with session_context as session: + rows = await session.select(f"SELECT version_num FROM public.{table} ORDER BY version_num") + else: + with session_context as session: + rows = session.select(f"SELECT version_num FROM public.{table} ORDER BY version_num") + return [row["version_num"] for row in rows] + + @pytest.mark.parametrize("mode", ["asyncpg", "psycopg"]) async def test_upgrade_refuses_tracking_table_owned_by_foreign_role( tmp_path: Path, postgres_service: PostgresService, mode: str @@ -109,6 +120,42 @@ async def test_upgrade_refuses_tracking_table_owned_by_foreign_role( ) +@pytest.mark.parametrize("forced", [False, True], ids=["owner-exempt", "forced"]) +@pytest.mark.parametrize("mode", ["asyncpg", "psycopg"]) +async def test_upgrade_refuses_tracking_table_only_when_row_security_applies_to_migrator( + tmp_path: Path, postgres_service: PostgresService, mode: str, forced: bool +) -> None: + suffix = uuid4().hex[:8] + migrator, table = f"guard_rls_{suffix}", f"ddl_guard_{suffix}" + statements = [ + f"CREATE ROLE {migrator} LOGIN PASSWORD 'pw' NOSUPERUSER", + f"GRANT CREATE, USAGE ON SCHEMA public TO {migrator}", + f"CREATE TABLE public.{table} (version_num VARCHAR(32) PRIMARY KEY)", + f"ALTER TABLE public.{table} OWNER TO {migrator}", + f"ALTER TABLE public.{table} ENABLE ROW LEVEL SECURITY", + ] + if forced: + statements.append(f"ALTER TABLE public.{table} FORCE ROW LEVEL SECURITY") + _admin_statements(postgres_service, statements) + migrations_dir = tmp_path / "migrations" + migrations_dir.mkdir() + (migrations_dir / "0001_noop.py").write_text(TEMP_SHADOW_MIGRATION, encoding="utf-8") + config, commands = _build(mode, postgres_service, migrator, "pw", migrations_dir, table) + try: + if forced: + with pytest.raises(MigrationError, match="row-level security"): + await _upgrade(commands) + else: + await _upgrade(commands) + assert await _applied_versions(config, table) == ["0001"] + finally: + await _close(config) + _admin_statements( + postgres_service, + [f"DROP TABLE IF EXISTS public.{table}", f"DROP OWNED BY {migrator}", f"DROP ROLE {migrator}"], + ) + + @pytest.mark.parametrize("mode", ["asyncpg", "psycopg"]) async def test_temp_table_does_not_redirect_tracking_writes( tmp_path: Path, postgres_service: PostgresService, mode: str diff --git a/tests/integration/storage/test_integration.py b/tests/integration/storage/test_integration.py index 28d3c45c1..9312c03d3 100644 --- a/tests/integration/storage/test_integration.py +++ b/tests/integration/storage/test_integration.py @@ -12,6 +12,7 @@ from sqlspec.storage.registry import storage_registry from sqlspec.typing import FSSPEC_INSTALLED, OBSTORE_INSTALLED, PYARROW_INSTALLED from sqlspec.utils.module_loader import import_optional, import_optional_attr +from sqlspec.utils.uuids import uuid4 from tests.fixtures.rustfs import rustfs_fsspec_kwargs, rustfs_obstore_kwargs if TYPE_CHECKING: @@ -251,6 +252,44 @@ def test_fsspec_s3_copy_move_operations(fsspec_s3_backend: "ObjectStoreProtocol" assert fsspec_s3_backend.read_text_sync(move_dest_path) == TEST_TEXT_CONTENT +@pytest.mark.xdist_group("rustfs") +@pytest.mark.skipif(not FSSPEC_INSTALLED, reason="fsspec missing") +def test_fsspec_s3_glob_characters_in_keys_are_literal(fsspec_s3_backend: "ObjectStoreProtocol") -> None: + """FSSpec S3 copy, move, and delete act on the literal key and leave glob-matching siblings alone.""" + root = f"glob_literal_{uuid4().hex}" + siblings = { + f"{root}/t/a.bin": b"sibling-a", + f"{root}/t/b.bin": b"sibling-b", + f"{root}/tenants/one/data.bin": b"one", + f"{root}/tenants/two/nested/data.bin": b"two", + } + for sibling_path, sibling_content in siblings.items(): + fsspec_s3_backend.write_bytes_sync(sibling_path, sibling_content) + literal_key = f"{root}/t/[ab].bin" + fsspec_s3_backend.write_bytes_sync(literal_key, b"literal") + fsspec_s3_backend.write_bytes_sync(f"{root}/tenants/*/data.bin", b"star") + fsspec_s3_backend.write_bytes_sync(f"{root}/tenants/*/nested/data.bin", b"star-nested") + + try: + fsspec_s3_backend.copy_sync(literal_key, f"{root}/out/copy.bin") + assert fsspec_s3_backend.read_bytes_sync(f"{root}/out/copy.bin") == b"literal" + assert len(fsspec_s3_backend.list_objects_sync(f"{root}/out/")) == 1 + + fsspec_s3_backend.move_sync(literal_key, f"{root}/out/moved.bin") + assert fsspec_s3_backend.read_bytes_sync(f"{root}/out/moved.bin") == b"literal" + assert not fsspec_s3_backend.exists_sync(literal_key) + + fsspec_s3_backend.write_bytes_sync(literal_key, b"literal") + fsspec_s3_backend.delete_sync(literal_key) + assert not fsspec_s3_backend.exists_sync(literal_key) + + fsspec_s3_backend.delete_sync(f"{root}/tenants/*", recursive=True) + assert len(fsspec_s3_backend.list_objects_sync(f"{root}/tenants/")) == 2 + assert {path: fsspec_s3_backend.read_bytes_sync(path) for path in siblings} == siblings + finally: + fsspec_s3_backend.delete_sync(root, recursive=True) + + @pytest.mark.xdist_group("rustfs") @pytest.mark.skipif(not OBSTORE_INSTALLED, reason="obstore missing") def test_obstore_s3_basic_operations(obstore_s3_backend: "ObjectStoreProtocol") -> None: diff --git a/tests/integration/storage/test_streaming.py b/tests/integration/storage/test_streaming.py index cb8ce4542..62a531165 100644 --- a/tests/integration/storage/test_streaming.py +++ b/tests/integration/storage/test_streaming.py @@ -5,6 +5,7 @@ import threading from collections.abc import Iterator from typing import TYPE_CHECKING, Any, cast +from uuid import uuid4 import pytest @@ -159,7 +160,7 @@ async def test_rustfs_s3_obstore_create_only_stream( assert plain_backend.supports_direct_arrow_write("parquet", overwrite=False) is False if PYARROW_INSTALLED and pyarrow is not None: tbl = pyarrow.table({"id": [1, 2]}) - plain_key = "streaming/rustfs_obstore_plain_once/table.parquet" + plain_key = f"streaming/rustfs_obstore_plain_once/{uuid4().hex}/table.parquet" plain_backend.write_arrow_sync(plain_key, tbl, overwrite=False) with pytest.raises(FileExistsInStorageError): plain_backend.write_arrow_sync(plain_key, tbl, overwrite=False) @@ -324,6 +325,7 @@ def _probed_close(self_file: Any) -> Any: async def _assert_fsspec_direct_streaming(backend: "FSSpecBackend", prefix: str, abort_probe: "list[bool]") -> None: """Assert multipart success, abort, cancellation, and create-only conflict for one backend.""" + prefix = f"{prefix}/{uuid4().hex}" block_size = 5_242_880 payload_12mb = os.urandom(12 * 1024 * 1024) chunks = [payload_12mb[i : i + 1_048_576] for i in range(0, len(payload_12mb), 1_048_576)] @@ -406,6 +408,7 @@ async def _parked_upload() -> None: async def _assert_obstore_create_only_stream(backend: "ObStoreBackend", prefix: str) -> None: """Assert 12 MiB staged create-only streaming, conflict rejection, and stage cleanup on ObStoreBackend.""" + prefix = f"{prefix}/{uuid4().hex}" assert backend.supports_direct_arrow_write("parquet", overwrite=False) is True payload_12mb = os.urandom(12 * 1024 * 1024) @@ -582,13 +585,14 @@ def _cancellable_fresh_batches() -> Iterator[Any]: assert not await backend.exists_async(f"{prefix}/cancel_fresh.parquet") if check_overwrite_false_conflict: - once_dest = f"alias://{alias_name}/{prefix}/once.parquet" + once_key = f"{prefix}/once_{uuid4().hex}.parquet" + once_dest = f"alias://{alias_name}/{once_key}" sync_pipeline.write_arrow(v1, once_dest, overwrite=False) with pytest.raises(FileExistsInStorageError): sync_pipeline.write_arrow(v1, once_dest, overwrite=False) with pytest.raises(FileExistsInStorageError): await async_pipeline.write_arrow(v1, once_dest, overwrite=False) - assert backend.read_arrow_sync(f"{prefix}/once.parquet").equals(v1) + assert backend.read_arrow_sync(once_key).equals(v1) if check_multipart_12mb: random_chunks = [os.urandom(1024 * 1024) for _ in range(12)] diff --git a/tests/unit/adapters/test_adapter_implementations.py b/tests/unit/adapters/test_adapter_implementations.py index 99b404cf9..560d4f012 100644 --- a/tests/unit/adapters/test_adapter_implementations.py +++ b/tests/unit/adapters/test_adapter_implementations.py @@ -7,7 +7,7 @@ import operator import sqlite3 import types -from collections.abc import AsyncIterator, Generator, Iterator +from collections.abc import AsyncGenerator, Generator from contextlib import asynccontextmanager, contextmanager from datetime import UTC, datetime, timedelta from pathlib import Path @@ -759,7 +759,7 @@ async def execute(self, sql: object, *_args: Any) -> None: executed.append(_decode_sql(sql)) @asynccontextmanager - async def provide_async_fetch(*_a: Any, **_kw: Any) -> AsyncIterator[Any]: + async def provide_async_fetch(*_a: Any, **_kw: Any) -> AsyncGenerator[Any, None]: yield _AsyncFetchConn() return SimpleNamespace(extension_config={"litestar": {}}, provide_connection=provide_async_fetch) @@ -781,7 +781,7 @@ async def rollback(self) -> None: return None @asynccontextmanager - async def provide_async_cursor(*_a: Any, **_kw: Any) -> AsyncIterator[Any]: + async def provide_async_cursor(*_a: Any, **_kw: Any) -> AsyncGenerator[Any, None]: yield _AsyncCursorConn() return SimpleNamespace(extension_config={"litestar": {}}, provide_connection=provide_async_cursor) @@ -806,7 +806,7 @@ def rollback(self) -> None: return None @contextmanager - def provide_sync_cursor(*_a: Any, **_kw: Any) -> Iterator[Any]: + def provide_sync_cursor(*_a: Any, **_kw: Any) -> Generator[Any, None, None]: yield _SyncCursorConn() return SimpleNamespace(extension_config={"litestar": {}}, provide_connection=provide_sync_cursor) @@ -825,7 +825,7 @@ def commit(self) -> None: return None @contextmanager - def provide_direct_sync(*_a: Any, **_kw: Any) -> Iterator[Any]: + def provide_direct_sync(*_a: Any, **_kw: Any) -> Generator[Any, None, None]: yield _DirectSyncConn() return SimpleNamespace(extension_config={"litestar": {}}, provide_connection=provide_direct_sync) @@ -842,7 +842,7 @@ async def commit(self) -> None: return None @asynccontextmanager - async def provide_aiosqlite(*_a: Any, **_kw: Any) -> AsyncIterator[Any]: + async def provide_aiosqlite(*_a: Any, **_kw: Any) -> AsyncGenerator[Any, None]: yield _AiosqliteConn() return SimpleNamespace(extension_config={"litestar": {}}, provide_connection=provide_aiosqlite) @@ -861,7 +861,7 @@ async def commit(self) -> None: return None @asynccontextmanager - async def provide_async_session(*_a: Any, **_kw: Any) -> AsyncIterator[Any]: + async def provide_async_session(*_a: Any, **_kw: Any) -> AsyncGenerator[Any, None]: yield _AsyncDriver() return SimpleNamespace(extension_config={"litestar": {}}, provide_session=provide_async_session) @@ -888,7 +888,7 @@ def commit(self) -> None: return None @contextmanager - def provide_sync_session(*_a: Any, **_kw: Any) -> Iterator[Any]: + def provide_sync_session(*_a: Any, **_kw: Any) -> Generator[Any, None, None]: yield _SyncDriver() return SimpleNamespace( diff --git a/tests/unit/adapters/test_adbc/test_extension_stores.py b/tests/unit/adapters/test_adbc/test_extension_stores.py index fa629df6d..926bf7ec6 100644 --- a/tests/unit/adapters/test_adbc/test_extension_stores.py +++ b/tests/unit/adapters/test_adbc/test_extension_stores.py @@ -430,7 +430,7 @@ def commit(self) -> None: if expect_future: assert before + timedelta(milliseconds=200) <= bound_expires_at <= after + timedelta(milliseconds=300) else: - assert before - timedelta(microseconds=1) <= bound_expires_at <= after - timedelta(microseconds=1) + assert before - timedelta(seconds=1) <= bound_expires_at <= after - timedelta(seconds=1) @pytest.mark.parametrize( diff --git a/tests/unit/adapters/test_aiomysql/test_cache_hit_executor.py b/tests/unit/adapters/test_aiomysql/test_cache_hit_executor.py index 1f4bfe48c..08824beda 100644 --- a/tests/unit/adapters/test_aiomysql/test_cache_hit_executor.py +++ b/tests/unit/adapters/test_aiomysql/test_cache_hit_executor.py @@ -39,7 +39,7 @@ def _cached(compiled_sql: str, operation_type: Any, *, returns_rows: bool, param def _cursor() -> MagicMock: cursor = MagicMock() cursor.execute = AsyncMock(return_value=None) - cursor.fetchall = AsyncMock(return_value=[]) + cursor.fetchall = AsyncMock(return_value=()) cursor.close = AsyncMock(return_value=None) return cursor @@ -54,7 +54,7 @@ async def test_execute_cache_hit_returns_rows_with_json_decoded() -> None: """A row-returning statement yields decoded tuple rows, column names, and the row count.""" cursor = _cursor() cursor.description = [("id", 3), ("payload", JSON_TYPE_CODE)] - cursor.fetchall = AsyncMock(return_value=[(1, '{"name": "alpha"}'), (2, None)]) + cursor.fetchall = AsyncMock(return_value=((1, '{"name": "alpha"}'), (2, None))) connection = _connection(cursor) sql = "SELECT id, payload FROM items WHERE id > %s" cached = _cached(sql, "SELECT", returns_rows=True, param_count=1) @@ -133,7 +133,7 @@ async def test_driver_runs_repeated_statement_through_executor() -> None: """A repeated raw statement on the driver returns the same rows on the cached execution.""" cursor = _cursor() cursor.description = [("value", 3)] - cursor.fetchall = AsyncMock(return_value=[(1,)]) + cursor.fetchall = AsyncMock(return_value=((1,),)) connection = _connection(cursor) connection.server_status = 0 driver = AiomysqlDriver(cast("Any", connection)) @@ -153,7 +153,7 @@ async def test_execute_cache_hit_matches_driver_cache_hit_method() -> None: """The executor and the driver cache-hit method produce the same rows, columns, and count.""" cursor = _cursor() cursor.description = [("id", 3), ("payload", JSON_TYPE_CODE)] - cursor.fetchall = AsyncMock(side_effect=[[(1, '{"name": "alpha"}')], [(1, '{"name": "alpha"}')]]) + cursor.fetchall = AsyncMock(side_effect=[((1, '{"name": "alpha"}'),), ((1, '{"name": "alpha"}'),)]) connection = _connection(cursor) connection.server_status = 0 driver = AiomysqlDriver(cast("Any", connection)) diff --git a/tests/unit/adapters/test_aiomysql/test_config.py b/tests/unit/adapters/test_aiomysql/test_config.py index 8651a174e..5b4f44475 100644 --- a/tests/unit/adapters/test_aiomysql/test_config.py +++ b/tests/unit/adapters/test_aiomysql/test_config.py @@ -578,3 +578,30 @@ async def on_create(_conn: Any) -> None: expected = ["sql:SET SESSION TRANSACTION READ ONLY", "hook"] if read_only else ["hook"] assert events == expected + + +async def test_aiomysql_disables_multi_statements_once_and_after_reset() -> None: + """Default AiomysqlConfig sends COM_SET_OPTION off once per connection and again after reset.""" + from sqlspec.adapters.aiomysql.config import _COM_SET_OPTION, _MYSQL_OPTION_MULTI_STATEMENTS_OFF, _reset_session + + conn = MagicMock(closed=False, _execute_command=AsyncMock(), _read_packet=AsyncMock(), _read_ok_packet=AsyncMock()) + config = AiomysqlConfig(connection_config={"host": "localhost", "db": "test"}) + + await config._ensure_connection(conn) + await config._ensure_connection(conn) + assert conn._execute_command.await_count == 1 + assert conn._execute_command.await_args.args == (_COM_SET_OPTION, _MYSQL_OPTION_MULTI_STATEMENTS_OFF) + assert conn._read_packet.await_count == 1 + + await _reset_session(config, conn) + assert conn._execute_command.await_count == 3 + assert conn._execute_command.await_args_list[2].args == (_COM_SET_OPTION, _MYSQL_OPTION_MULTI_STATEMENTS_OFF) + + +async def test_aiomysql_preserves_explicit_multi_statements_client_flag() -> None: + """Explicit CLIENT.MULTI_STATEMENTS opt-in skips COM_SET_OPTION multi-statement disabling.""" + conn = MagicMock(closed=False, _execute_command=AsyncMock(), _read_packet=AsyncMock()) + config = AiomysqlConfig(connection_config={"host": "localhost", "db": "test", "client_flag": 1 << 16}) + + await config._ensure_connection(conn) + conn._execute_command.assert_not_awaited() diff --git a/tests/unit/adapters/test_aiomysql/test_driver.py b/tests/unit/adapters/test_aiomysql/test_driver.py index 8acbbf1f9..ed4397c17 100644 --- a/tests/unit/adapters/test_aiomysql/test_driver.py +++ b/tests/unit/adapters/test_aiomysql/test_driver.py @@ -113,7 +113,7 @@ async def test_aiomysql_execute_cache_hit_invokes_collect_rows_on_clean_query(mo cursor = MagicMock() cursor.description = [("id", 3, None, None, None, None, None)] - cursor.fetchall = AsyncMock(return_value=[(1,)]) + cursor.fetchall = AsyncMock(return_value=((1,),)) cursor.execute = AsyncMock(return_value=None) conn = MagicMock() @@ -158,7 +158,7 @@ async def test_aiomysql_dispatch_execute_uses_statement_statement_config() -> No cursor = MagicMock() cursor.description = [("1", 3, None, None, None, None, None)] cursor.execute = AsyncMock() - cursor.fetchall = AsyncMock(return_value=[(1,)]) + cursor.fetchall = AsyncMock(return_value=((1,),)) cursor.rowcount = 1 cursor.lastrowid = None @@ -309,3 +309,30 @@ def test_aiomysql_maps_1040_and_bare_connection_error_classes() -> None: assert mapped_iface.__cause__ is iface_err assert type(mapped_op) is OperationalError assert mapped_op.__cause__ is op_err + + +async def test_aiomysql_maps_client_bind_type_error_to_data_error() -> None: + """A TypeError raised by the client while binding parameters maps to DataError.""" + from pymysql.converters import escape_item + + from sqlspec.adapters.aiomysql.driver import AiomysqlExceptionHandler + from sqlspec.exceptions import DataError + + handler = AiomysqlExceptionHandler() + async with handler: + escape_item({"nested": 1}, "utf8mb4") + + assert type(handler.pending_exception) is DataError + assert isinstance(handler.pending_exception.__cause__, TypeError) + + +async def test_aiomysql_does_not_map_type_error_raised_outside_the_client() -> None: + """A TypeError that does not originate in the client propagates unchanged.""" + from sqlspec.adapters.aiomysql.driver import AiomysqlExceptionHandler + + handler = AiomysqlExceptionHandler() + with pytest.raises(TypeError, match="list object expected; got tuple"): + async with handler: + raise TypeError("list object expected; got tuple") + + assert handler.pending_exception is None diff --git a/tests/unit/adapters/test_aiosqlite/test_driver.py b/tests/unit/adapters/test_aiosqlite/test_driver.py index 90974ffe3..e59f60743 100644 --- a/tests/unit/adapters/test_aiosqlite/test_driver.py +++ b/tests/unit/adapters/test_aiosqlite/test_driver.py @@ -28,7 +28,7 @@ from sqlspec.adapters.aiosqlite.pool import AiosqliteConnectionPool from sqlspec.core import ParameterStyle from sqlspec.core.compiler import OperationType -from sqlspec.exceptions import SQLSpecError +from sqlspec.exceptions import SQLSpecError, UniqueViolationError def test_rowid_eligibility_falls_back_when_table_list_is_unavailable() -> None: @@ -548,3 +548,88 @@ def _spy_read(raw_conn: Any) -> bool: assert all(tid != threading.get_ident() for tid in observed_threads) finally: await conn.close() + + +def _record_worker_round_trips(connection: aiosqlite.Connection, monkeypatch: pytest.MonkeyPatch) -> "list[str]": + round_trips: list[str] = [] + execute_on_worker: Any = connection._execute + + async def recorded(function: Any, *args: Any, **kwargs: Any) -> Any: + round_trips.append(getattr(function, "__name__", repr(function))) + return await execute_on_worker(function, *args, **kwargs) + + monkeypatch.setattr(connection, "_execute", recorded) + return round_trips + + +async def test_each_statement_takes_one_worker_round_trip(monkeypatch: pytest.MonkeyPatch) -> None: + connection = await aiosqlite.connect(":memory:") + try: + driver = AiosqliteDriver(connection=connection) + await driver.execute_script("CREATE TABLE round_trips (id INTEGER PRIMARY KEY, value TEXT)") + round_trips = _record_worker_round_trips(connection, monkeypatch) + + await driver.begin() + begin_round_trips = len(round_trips) + round_trips.clear() + many = await driver.execute_many("INSERT INTO round_trips (value) VALUES (?)", [("a",), ("b",), ("c",)]) + many_round_trips = len(round_trips) + round_trips.clear() + inserted = await driver.execute("INSERT INTO round_trips (value) VALUES (:value)", {"value": "d"}) + insert_round_trips = len(round_trips) + round_trips.clear() + await driver.execute_script( + "UPDATE round_trips SET value = 'z' WHERE id = 1; DELETE FROM round_trips WHERE id = 2" + ) + script_round_trips = len(round_trips) + selected = await driver.execute("SELECT value FROM round_trips WHERE id = :id", {"id": 1}) + + assert (begin_round_trips, many_round_trips, insert_round_trips, script_round_trips) == (1, 1, 1, 2) + assert many.rows_affected == 3 + assert inserted.rows_affected == 1 + assert inserted.last_inserted_id == 4 + assert selected.get_data() == [{"value": "z"}] + assert connection.in_transaction + finally: + await connection.close() + + +async def test_failing_statement_without_cursor_round_trip_maps_error_and_keeps_connection_usable() -> None: + connection = await aiosqlite.connect(":memory:") + try: + driver = AiosqliteDriver(connection=connection) + await driver.execute_script("CREATE TABLE unique_values (id INTEGER PRIMARY KEY)") + + with pytest.raises(UniqueViolationError): + await driver.execute_many("INSERT INTO unique_values (id) VALUES (?)", [(1,), (1,)]) + with pytest.raises(SQLSpecError): + await driver.execute_script( + "INSERT INTO unique_values (id) VALUES (2); INSERT INTO missing_table VALUES (1)" + ) + + assert await driver.select_value("SELECT COUNT(*) FROM unique_values WHERE id = 2") == 1 + finally: + await connection.close() + + +async def test_subclass_dispatch_override_still_receives_an_open_cursor() -> None: + received: list[Any] = [] + + class _CursorUsingDriver(AiosqliteDriver): + async def dispatch_execute_many(self, cursor: Any, statement: Any) -> Any: + received.append(cursor) + await cursor.execute("INSERT INTO subclass_rows (value) VALUES ('from-cursor')") + return await super().dispatch_execute_many(cursor, statement) + + connection = await aiosqlite.connect(":memory:") + try: + driver = _CursorUsingDriver(connection=connection) + await driver.execute_script("CREATE TABLE subclass_rows (id INTEGER PRIMARY KEY, value TEXT)") + + result = await driver.execute_many("INSERT INTO subclass_rows (value) VALUES (?)", [("a",), ("b",)]) + + assert result.rows_affected == 2 + assert isinstance(received[0], aiosqlite.Cursor) + assert await driver.select_value("SELECT COUNT(*) FROM subclass_rows") == 3 + finally: + await connection.close() diff --git a/tests/unit/adapters/test_aiosqlite/test_pool.py b/tests/unit/adapters/test_aiosqlite/test_pool.py index 254c1f587..47bef0108 100644 --- a/tests/unit/adapters/test_aiosqlite/test_pool.py +++ b/tests/unit/adapters/test_aiosqlite/test_pool.py @@ -905,3 +905,31 @@ def test_memory_detection_parses_uri(database: str, uri: "bool | None", expected if uri is not None: params["uri"] = uri assert pool_module._classify_memory_mode(params) == expected + + +@pytest.mark.anyio +async def test_end_transaction_skips_worker_hop_on_clean_idle_commit(monkeypatch: pytest.MonkeyPatch) -> None: + """Clean commit on an idle empty-queue connection avoids dispatching to the worker thread.""" + import sqlspec.adapters.aiosqlite.core as aiosqlite_core + + pool = AiosqliteConnectionPool({"database": ":memory:"}, pool_size=1, min_size=1) + worker_calls = 0 + + try: + async with pool.get_connection() as conn: + await conn.execute("SELECT 1") + original_worker_hop = aiosqlite_core.run_on_worker_thread + + async def _counting_worker_hop(connection: Any, fn: Any, *args: Any, **kwargs: Any) -> Any: + nonlocal worker_calls + worker_calls += 1 + return await original_worker_hop(connection, fn, *args, **kwargs) + + monkeypatch.setattr(aiosqlite_core, "run_on_worker_thread", _counting_worker_hop) + await aiosqlite_core.end_transaction(conn, commit=True) + assert worker_calls == 0 + + await aiosqlite_core.end_transaction(conn, commit=False) + assert worker_calls == 1 + finally: + await pool.close() diff --git a/tests/unit/adapters/test_asyncmy/test_cache_hit_executor.py b/tests/unit/adapters/test_asyncmy/test_cache_hit_executor.py index 022cfcf11..bb3ced493 100644 --- a/tests/unit/adapters/test_asyncmy/test_cache_hit_executor.py +++ b/tests/unit/adapters/test_asyncmy/test_cache_hit_executor.py @@ -39,7 +39,7 @@ def _cached(compiled_sql: str, operation_type: Any, *, returns_rows: bool, param def _cursor() -> MagicMock: cursor = MagicMock() cursor.execute = AsyncMock(return_value=None) - cursor.fetchall = AsyncMock(return_value=[]) + cursor.fetchall = AsyncMock(return_value=()) cursor.close = AsyncMock(return_value=None) return cursor @@ -54,7 +54,7 @@ async def test_execute_cache_hit_returns_rows_with_json_decoded() -> None: """A row-returning statement yields decoded tuple rows, column names, and the row count.""" cursor = _cursor() cursor.description = [("id", 3), ("payload", JSON_TYPE_CODE)] - cursor.fetchall = AsyncMock(return_value=[(1, '{"name": "alpha"}'), (2, None)]) + cursor.fetchall = AsyncMock(return_value=((1, '{"name": "alpha"}'), (2, None))) connection = _connection(cursor) sql = "SELECT id, payload FROM items WHERE id > %s" cached = _cached(sql, "SELECT", returns_rows=True, param_count=1) @@ -133,7 +133,7 @@ async def test_driver_runs_repeated_statement_through_executor() -> None: """A repeated raw statement on the driver returns the same rows on the cached execution.""" cursor = _cursor() cursor.description = [("value", 3)] - cursor.fetchall = AsyncMock(return_value=[(1,)]) + cursor.fetchall = AsyncMock(return_value=((1,),)) connection = _connection(cursor) connection.server_status = 0 driver = AsyncmyDriver(cast("Any", connection)) @@ -153,7 +153,7 @@ async def test_execute_cache_hit_matches_driver_cache_hit_method() -> None: """The executor and the driver cache-hit method produce the same rows, columns, and count.""" cursor = _cursor() cursor.description = [("id", 3), ("payload", JSON_TYPE_CODE)] - cursor.fetchall = AsyncMock(side_effect=[[(1, '{"name": "alpha"}')], [(1, '{"name": "alpha"}')]]) + cursor.fetchall = AsyncMock(side_effect=[((1, '{"name": "alpha"}'),), ((1, '{"name": "alpha"}'),)]) connection = _connection(cursor) connection.server_status = 0 driver = AsyncmyDriver(cast("Any", connection)) diff --git a/tests/unit/adapters/test_asyncmy/test_config.py b/tests/unit/adapters/test_asyncmy/test_config.py index 478ee63a1..0a3098810 100644 --- a/tests/unit/adapters/test_asyncmy/test_config.py +++ b/tests/unit/adapters/test_asyncmy/test_config.py @@ -630,3 +630,30 @@ async def on_create(_conn: Any) -> None: expected = ["sql:SET SESSION TRANSACTION READ ONLY", "hook"] if read_only else ["hook"] assert events == expected + + +async def test_asyncmy_disables_multi_statements_once_and_after_reset() -> None: + """Default AsyncmyConfig sends COM_SET_OPTION off once per connection and again after reset.""" + from sqlspec.adapters.asyncmy.config import _COM_SET_OPTION, _MYSQL_OPTION_MULTI_STATEMENTS_OFF, _reset_session + + conn = MagicMock(closed=False, _execute_command=AsyncMock(), read_packet=AsyncMock(), _read_ok_packet=AsyncMock()) + config = AsyncmyConfig(connection_config={"host": "localhost", "db": "test"}) + + await config._ensure_connection(conn) + await config._ensure_connection(conn) + assert conn._execute_command.await_count == 1 + assert conn._execute_command.await_args.args == (_COM_SET_OPTION, _MYSQL_OPTION_MULTI_STATEMENTS_OFF) + assert conn.read_packet.await_count == 1 + + await _reset_session(config, conn) + assert conn._execute_command.await_count == 3 + assert conn._execute_command.await_args_list[2].args == (_COM_SET_OPTION, _MYSQL_OPTION_MULTI_STATEMENTS_OFF) + + +async def test_asyncmy_preserves_explicit_multi_statements_client_flag() -> None: + """Explicit CLIENT.MULTI_STATEMENTS opt-in skips COM_SET_OPTION multi-statement disabling.""" + conn = MagicMock(closed=False, _execute_command=AsyncMock(), read_packet=AsyncMock()) + config = AsyncmyConfig(connection_config={"host": "localhost", "db": "test", "client_flag": 1 << 16}) + + await config._ensure_connection(conn) + conn._execute_command.assert_not_awaited() diff --git a/tests/unit/adapters/test_asyncmy/test_driver.py b/tests/unit/adapters/test_asyncmy/test_driver.py index d64874bfa..201fff30d 100644 --- a/tests/unit/adapters/test_asyncmy/test_driver.py +++ b/tests/unit/adapters/test_asyncmy/test_driver.py @@ -109,7 +109,7 @@ async def test_asyncmy_execute_cache_hit_invokes_collect_rows_on_clean_query(mon cursor = MagicMock() cursor.description = [("id", 3, None, None, None, None, None)] - cursor.fetchall = AsyncMock(return_value=[(1,)]) + cursor.fetchall = AsyncMock(return_value=((1,),)) cursor.execute = AsyncMock(return_value=None) conn = MagicMock() @@ -172,7 +172,7 @@ async def test_asyncmy_dispatch_execute_uses_statement_statement_config() -> Non cursor = MagicMock() cursor.description = [("1", 3, None, None, None, None, None)] cursor.execute = AsyncMock() - cursor.fetchall = AsyncMock(return_value=[(1,)]) + cursor.fetchall = AsyncMock(return_value=((1,),)) cursor.rowcount = 1 cursor.lastrowid = None diff --git a/tests/unit/adapters/test_asyncpg/test_config.py b/tests/unit/adapters/test_asyncpg/test_config.py index 32ba33a48..00904eb6c 100644 --- a/tests/unit/adapters/test_asyncpg/test_config.py +++ b/tests/unit/adapters/test_asyncpg/test_config.py @@ -817,3 +817,27 @@ def test_asyncpg_replica_dsn_and_direct_dsn_clear_inherited_endpoints() -> None: assert addrs == [("direct", 5439)] assert params.database == "direct_db" assert params.user == "du" + + +def test_resolve_postgres_extension_state_syncs_statement_cache_holder() -> None: + """Promoting dialect during extension detection refreshes _statement_cache_holder to match statement_config.""" + from sqlspec.core.cache import StatementCacheHolder + from sqlspec.core.config_runtime import resolve_postgres_extension_state + + cfg = AsyncpgConfig( + connection_config={"dsn": "postgresql://localhost/db"}, driver_features={"enable_pgvector": True} + ) + holder = StatementCacheHolder(cfg.statement_config, 64) + cfg.driver_features["_statement_cache_holder"] = holder + cfg.driver_features["statement_cache_size"] = 64 + + promoted, has_pgvector, has_paradedb = resolve_postgres_extension_state( + cfg.statement_config, cfg.driver_features, {"vector"} + ) + + assert has_pgvector is True + assert has_paradedb is False + assert promoted.dialect == "pgvector" + refreshed_holder = cfg.driver_features["_statement_cache_holder"] + assert refreshed_holder is not holder + assert refreshed_holder.statement_config is promoted diff --git a/tests/unit/adapters/test_asyncpg/test_transaction_state.py b/tests/unit/adapters/test_asyncpg/test_transaction_state.py index dda9559f0..b0f1182e3 100644 --- a/tests/unit/adapters/test_asyncpg/test_transaction_state.py +++ b/tests/unit/adapters/test_asyncpg/test_transaction_state.py @@ -2,7 +2,7 @@ import inspect from asyncio import CancelledError -from collections.abc import AsyncIterator +from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from pathlib import Path from typing import Any, cast @@ -224,6 +224,55 @@ async def test_begin_resets_stale_transaction_and_maps_exceptions() -> None: await failing_driver.begin() +class _ClaimingTransaction: + """Mirrors asyncpg claiming the connection before BEGIN and keeping the claim when BEGIN fails.""" + + def __init__(self, connection: "_ClaimedConnection") -> None: + self._connection = connection + + async def start(self) -> None: + self._connection._top_xact = self + raise asyncpg.exceptions.DeadlockDetectedError("deadlock") + + +class _ClaimedConnection: + __slots__ = ("_top_xact",) + + def __init__(self) -> None: + self._top_xact: Any = None + + def transaction(self) -> _ClaimingTransaction: + return _ClaimingTransaction(self) + + def is_in_transaction(self) -> bool: + return False + + +class _SlottedPoolProxy: + """Mirrors asyncpg's pool proxy: slotted, forwarding attribute reads to the wrapped connection.""" + + __slots__ = ("_con",) + + def __init__(self, connection: _ClaimedConnection) -> None: + self._con = connection + + def __getattr__(self, name: str) -> Any: + return getattr(self._con, name) + + +@pytest.mark.parametrize("pooled", [False, True], ids=["raw-connection", "pool-proxy"]) +async def test_failed_begin_releases_the_transaction_claim(pooled: bool) -> None: + """A failed BEGIN must not leave the connection claimed, or the next begin becomes a savepoint.""" + raw_connection = _ClaimedConnection() + connection = _SlottedPoolProxy(raw_connection) if pooled else raw_connection + driver = AsyncpgDriver(cast("Any", connection)) + + with pytest.raises(DeadlockError): + await driver.begin() + + assert raw_connection._top_xact is None + + async def test_commit_and_rollback_skip_raw_sql_when_not_in_transaction() -> None: """commit() and rollback() should not execute raw COMMIT/ROLLBACK when connection is not in transaction.""" connection = _FakeConnection() @@ -299,7 +348,7 @@ async def test_events_backends_manage_transactions_correctly() -> None: driver.rollback = AsyncMock(side_effect=lambda: calls.append("rollback")) @asynccontextmanager - async def fake_session(*_a: Any, **_k: Any) -> AsyncIterator[Any]: + async def fake_session(*_a: Any, **_k: Any) -> AsyncGenerator[Any, None]: yield driver config = MagicMock() diff --git a/tests/unit/adapters/test_cockroach_asyncpg/test_core.py b/tests/unit/adapters/test_cockroach_asyncpg/test_core.py index 89e8e0a36..b52786f66 100644 --- a/tests/unit/adapters/test_cockroach_asyncpg/test_core.py +++ b/tests/unit/adapters/test_cockroach_asyncpg/test_core.py @@ -9,6 +9,7 @@ import asyncpg import pytest +from sqlspec.adapters.asyncpg.core import create_mapped_exception as asyncpg_create_mapped_exception from sqlspec.adapters.asyncpg.core import default_statement_config, register_json_codecs from sqlspec.adapters.asyncpg.driver import AsyncpgDriver from sqlspec.adapters.cockroach_asyncpg import CockroachAsyncpgRetryConfig @@ -221,6 +222,26 @@ def test_create_mapped_exception_signature_parity() -> None: assert isinstance(mapped, QueryTimeoutError) +@pytest.mark.parametrize( + "mapper", [asyncpg_create_mapped_exception, create_mapped_exception], ids=["asyncpg", "cockroach_asyncpg"] +) +@pytest.mark.parametrize( + ("error", "expected_cls"), + [ + (TimeoutError("connect timed out"), QueryTimeoutError), + (OSError("connection reset by peer"), DatabaseConnectionError), + (asyncpg.InterfaceError("connection is closed"), DatabaseConnectionError), + ], +) +def test_timeout_and_connection_errors_keep_the_driver_error_as_cause( + mapper: Any, error: BaseException, expected_cls: type[Exception] +) -> None: + mapped = mapper(error) + + assert isinstance(mapped, expected_cls) + assert mapped.__cause__ is error + + @pytest.mark.anyio @pytest.mark.parametrize( ("exc", "expected_cls"), diff --git a/tests/unit/adapters/test_cockroach_retry.py b/tests/unit/adapters/test_cockroach_retry.py index f30a53631..f9cf459b9 100644 --- a/tests/unit/adapters/test_cockroach_retry.py +++ b/tests/unit/adapters/test_cockroach_retry.py @@ -61,11 +61,12 @@ async def test_async_retry_preserves_outcome_when_rollback_fails( outcome: str, monkeypatch: pytest.MonkeyPatch, ) -> None: - """A failed rollback leaves the transaction aborted, so the original error is raised at once.""" + """A failed rollback raises the original error at once, with its cause kept and the rollback failure reported.""" monkeypatch.setattr(driver_type, "_connection_in_transaction", lambda _self: False) begin = AsyncMock() commit = AsyncMock() - rollback = AsyncMock(side_effect=RuntimeError("secondary rollback failure")) + rollback_failure = RuntimeError("secondary rollback failure") + rollback = AsyncMock(side_effect=rollback_failure) monkeypatch.setattr(driver_type, "begin", begin) monkeypatch.setattr(driver_type, "commit", commit) monkeypatch.setattr(driver_type, "rollback", rollback) @@ -73,13 +74,17 @@ async def test_async_retry_preserves_outcome_when_rollback_fails( connection=MagicMock(), driver_features={"max_retries": 1, "retry_delay_base_ms": 0, "enable_retry_logging": False}, ) + driver_error = LookupError("driver failure") original = ValueError("operation failed") if outcome == "nonretryable" else _RetryableError("restart transaction") + original.__cause__ = driver_error operation = AsyncMock(side_effect=original) with pytest.raises(type(original)) as caught: await driver.run_transaction_with_retry(operation) assert caught.value is original + assert caught.value.__cause__ is driver_error + assert caught.value.__context__ is rollback_failure assert operation.await_count == begin.await_count == 1 assert rollback.await_count == 1 assert commit.await_count == 0 diff --git a/tests/unit/adapters/test_contract_extra_assertions.py b/tests/unit/adapters/test_contract_extra_assertions.py index c1a868538..e9f7561a7 100644 --- a/tests/unit/adapters/test_contract_extra_assertions.py +++ b/tests/unit/adapters/test_contract_extra_assertions.py @@ -1,7 +1,7 @@ """Unit tests for the additive extra_assertions hook on the contract harness.""" import contextlib -from collections.abc import Iterator +from collections.abc import Generator import pytest @@ -28,7 +28,7 @@ def _make_case(*extra_assertions: str) -> DriverCase: @contextlib.contextmanager -def _registered_sync(key: str, scope: str) -> "Iterator[list[tuple[object, DriverCase]]]": +def _registered_sync(key: str, scope: str) -> "Generator[list[tuple[object, DriverCase]], None, None]": calls: list[tuple[object, DriverCase]] = [] register_sync_extra_assertion(key, scope, lambda driver, case: calls.append((driver, case))) try: @@ -40,7 +40,7 @@ def _registered_sync(key: str, scope: str) -> "Iterator[list[tuple[object, Drive @contextlib.contextmanager -def _registered_async(key: str, scope: str) -> "Iterator[list[tuple[object, DriverCase]]]": +def _registered_async(key: str, scope: str) -> "Generator[list[tuple[object, DriverCase]], None, None]": calls: list[tuple[object, DriverCase]] = [] async def _fn(driver: object, case: DriverCase) -> None: diff --git a/tests/unit/adapters/test_contract_lifecycle.py b/tests/unit/adapters/test_contract_lifecycle.py index 2412e2a59..4508ed090 100644 --- a/tests/unit/adapters/test_contract_lifecycle.py +++ b/tests/unit/adapters/test_contract_lifecycle.py @@ -122,7 +122,7 @@ def test_session_contexts_forward_exception_info_to_release() -> None: ("sqlspec.adapters.aiomysql._typing", "AiomysqlSessionContext", True), ("sqlspec.adapters.mysqlconnector._typing", "MysqlConnectorSyncSessionContext", False), ("sqlspec.adapters.mysqlconnector._typing", "MysqlConnectorAsyncSessionContext", True), - ("sqlspec.adapters.asyncmy.config", "AsyncmySessionContext", True), + ("sqlspec.adapters.asyncmy._typing", "AsyncmySessionContext", True), ("sqlspec.adapters.mssql_python._typing", "MssqlPythonSessionContext", False), ("sqlspec.adapters.pymssql._typing", "PymssqlSessionContext", False), ("sqlspec.adapters.sybase._typing", "SybaseSessionContext", False), @@ -199,12 +199,9 @@ def test_detach_contract_covers_every_session_context() -> None: """Every adapter SessionContext class across all 24 adapters is registered in _DETACH_SESSION_CONTEXTS.""" adapters_root = Path(sqlspec.adapters.__file__).resolve().parent candidate_modules = [ - *( - (f"sqlspec.adapters.{pkg.name}._typing", adapters_root / pkg.name / "_typing.py") - for pkg in pkgutil.iter_modules(sqlspec.adapters.__path__) - if (adapters_root / pkg.name / "_typing.py").is_file() - ), - ("sqlspec.adapters.asyncmy.config", adapters_root / "asyncmy" / "config.py"), + (f"sqlspec.adapters.{pkg.name}._typing", adapters_root / pkg.name / "_typing.py") + for pkg in pkgutil.iter_modules(sqlspec.adapters.__path__) + if (adapters_root / pkg.name / "_typing.py").is_file() ] discovered = { pair diff --git a/tests/unit/adapters/test_db2/_fakes.py b/tests/unit/adapters/test_db2/_fakes.py index ddfbb6df0..390b7ac4d 100644 --- a/tests/unit/adapters/test_db2/_fakes.py +++ b/tests/unit/adapters/test_db2/_fakes.py @@ -15,7 +15,7 @@ """ import inspect -from collections.abc import AsyncIterator, Callable, Iterator, Sequence +from collections.abc import AsyncGenerator, Callable, Generator, Sequence from contextlib import asynccontextmanager, contextmanager from typing import Any, Self @@ -499,7 +499,7 @@ class FakeDb2SessionConfig(_FakeDb2SessionState): """Config stand-in whose sessions run a real ``Db2SyncDriver`` over one scripted fake connection.""" @contextmanager - def provide_session(self, **_: Any) -> "Iterator[Db2SyncDriver]": + def provide_session(self, **_: Any) -> "Generator[Db2SyncDriver, None, None]": """Yield a driver bound to the shared fake connection.""" yield Db2SyncDriver( self.connection, statement_config=default_statement_config, driver_features=self.driver_features @@ -514,7 +514,7 @@ class FakeDb2AsyncSessionConfig(_FakeDb2SessionState): """ @asynccontextmanager - async def provide_session(self, **_: Any) -> "AsyncIterator[Any]": + async def provide_session(self, **_: Any) -> "AsyncGenerator[Any, None]": """Yield an async driver bound to the shared fake connection.""" yield db2_driver.Db2AsyncDriver( FakeDb2AsyncConnection(self.connection), @@ -657,7 +657,7 @@ async def call(self, function: "Callable[..., Any]", *args: Any, **kwargs: Any) return result @asynccontextmanager - async def enter(self, manager: Any) -> "AsyncIterator[Any]": + async def enter(self, manager: Any) -> "AsyncGenerator[Any, None]": """Enter a sync or async context manager. Yields: diff --git a/tests/unit/adapters/test_db2/test_exceptions.py b/tests/unit/adapters/test_db2/test_exceptions.py index d2e171440..4e3a73dd8 100644 --- a/tests/unit/adapters/test_db2/test_exceptions.py +++ b/tests/unit/adapters/test_db2/test_exceptions.py @@ -63,6 +63,7 @@ def test_sqlstate_attribute_mapping(sqlstate: str, expected_type: type[SQLSpecEr mapped = create_mapped_exception(exc) assert isinstance(mapped, expected_type) assert sqlstate in str(mapped) + assert mapped.__cause__ is exc @pytest.mark.parametrize( diff --git a/tests/unit/adapters/test_duckdb/test_arrow_streaming.py b/tests/unit/adapters/test_duckdb/test_arrow_streaming.py index b363ad17c..415c83b4d 100644 --- a/tests/unit/adapters/test_duckdb/test_arrow_streaming.py +++ b/tests/unit/adapters/test_duckdb/test_arrow_streaming.py @@ -1,6 +1,6 @@ """Unit tests for DuckDB Arrow streaming paths.""" -from collections.abc import Iterator +from collections.abc import Generator from contextlib import contextmanager from typing import Any from uuid import UUID @@ -16,7 +16,7 @@ @contextmanager -def _seed_driver(*, enable_uuid_conversion: bool = True) -> Iterator[DuckDBDriver]: +def _seed_driver(*, enable_uuid_conversion: bool = True) -> Generator[DuckDBDriver, None, None]: config = DuckDBConfig( connection_config={"database": ":memory:"}, driver_features={"enable_uuid_conversion": enable_uuid_conversion} ) @@ -242,3 +242,45 @@ def test_load_from_arrow_overwrite_rolls_back_truncate_on_insert_failure() -> No rows = driver.execute("SELECT id FROM atomic_target ORDER BY id").get_data() assert rows == [{"id": 1}, {"id": 2}] + + +def test_load_from_arrow_maps_columns_by_name_not_position() -> None: + """DuckDB load_from_arrow inserts by explicit column names when Arrow column order differs from target table.""" + with _seed_driver() as driver: + driver.execute("CREATE OR REPLACE TABLE reordered_target (id INTEGER, email VARCHAR, name VARCHAR)") + table = pa.table({"id": [1], "name": ["Ada"], "email": ["ada@example.com"]}) + + driver.load_from_arrow("reordered_target", table) + + rows = driver.execute("SELECT id, email, name FROM reordered_target").get_data() + assert rows == [{"id": 1, "email": "ada@example.com", "name": "Ada"}] + + +@pytest.mark.parametrize("overwrite", [False, True]) +@pytest.mark.parametrize( + ("arrow_columns", "expected_row"), + [ + pytest.param( + {"ident": [1], "label": ["Ada"], "contact": ["ada@example.com"]}, + {"id": 1, "email": "Ada", "name": "ada@example.com"}, + id="mismatched-names-load-by-position", + ), + pytest.param( + {"name": ["Ada"], "id": [1], "email": ["ada@example.com"]}, + {"id": 1, "email": "ada@example.com", "name": "Ada"}, + id="reordered-names-load-by-name", + ), + ], +) +def test_load_from_arrow_column_mapping_is_the_same_with_and_without_overwrite( + overwrite: bool, arrow_columns: "dict[str, list[Any]]", expected_row: "dict[str, Any]" +) -> None: + seeded_row = {"id": 0, "email": "seed@example.com", "name": "Seed"} + with _seed_driver() as driver: + driver.execute("CREATE OR REPLACE TABLE mapped_target (id INTEGER, email VARCHAR, name VARCHAR)") + driver.execute("INSERT INTO mapped_target VALUES (0, 'seed@example.com', 'Seed')") + + driver.load_from_arrow("mapped_target", pa.table(arrow_columns), overwrite=overwrite) + + rows = driver.execute("SELECT id, email, name FROM mapped_target ORDER BY id").get_data() + assert rows == ([expected_row] if overwrite else [seeded_row, expected_row]) diff --git a/tests/unit/adapters/test_embedded_session_reset.py b/tests/unit/adapters/test_embedded_session_reset.py index 88addc896..3bda986b4 100644 --- a/tests/unit/adapters/test_embedded_session_reset.py +++ b/tests/unit/adapters/test_embedded_session_reset.py @@ -2,7 +2,7 @@ import importlib import inspect -from collections.abc import AsyncIterator +from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from pathlib import Path from typing import Any @@ -41,7 +41,7 @@ async def rows(session: Any, sql: str) -> "list[Any]": @asynccontextmanager -async def session_scope(config: Any) -> "AsyncIterator[Any]": +async def session_scope(config: Any) -> "AsyncGenerator[Any, None]": context = config.provide_session() if hasattr(context, "__aenter__"): async with context as session: diff --git a/tests/unit/adapters/test_oracledb/test_stack_native_transaction.py b/tests/unit/adapters/test_oracledb/test_stack_native_transaction.py index 061e22be5..d748292e7 100644 --- a/tests/unit/adapters/test_oracledb/test_stack_native_transaction.py +++ b/tests/unit/adapters/test_oracledb/test_stack_native_transaction.py @@ -204,5 +204,5 @@ def commit(self) -> None: ) results = driver.execute_stack(stack) assert [result.rows_affected for result in results] == [1, 1] - assert connection.executed == [("INSERT INTO items VALUES (:1)", [1]), ("INSERT INTO items VALUES (:1)", [2])] + assert connection.executed == [("INSERT INTO items VALUES (:1)", [1]), ("INSERT INTO items VALUES (:1)", (2,))] assert connection.commits == 1 diff --git a/tests/unit/adapters/test_psqlpy/test_core.py b/tests/unit/adapters/test_psqlpy/test_core.py index 308d5f038..644319296 100644 --- a/tests/unit/adapters/test_psqlpy/test_core.py +++ b/tests/unit/adapters/test_psqlpy/test_core.py @@ -495,3 +495,17 @@ def test_create_mapped_exception_maps_psqlpy_connection_and_pool_base_errors( """Connection and pool base errors map cleanly even when the message omits generic keywords.""" mapped = psqlpy_core.create_mapped_exception(exc_factory()) assert type(mapped) is expected_cls + + +def test_psqlpy_maps_type_error_to_data_error() -> None: + """PsqlpyExceptionHandler and create_mapped_exception map Python TypeError to DataError.""" + from sqlspec.adapters.psqlpy.driver import PsqlpyExceptionHandler + from sqlspec.exceptions import DataError + + err = TypeError("unexpected bind parameter type") + mapped = psqlpy_core.create_mapped_exception(err) + assert type(mapped) is DataError + + handler = PsqlpyExceptionHandler() + assert handler._handle_exception(TypeError, err) is True + assert type(handler.pending_exception) is DataError diff --git a/tests/unit/adapters/test_psycopg/test_async_context_exit.py b/tests/unit/adapters/test_psycopg/test_async_context_exit.py new file mode 100644 index 000000000..be8081d65 --- /dev/null +++ b/tests/unit/adapters/test_psycopg/test_async_context_exit.py @@ -0,0 +1,251 @@ +"""Async psycopg paths finish when a managed context suspends while it exits.""" + +import asyncio +from collections.abc import AsyncIterator, Awaitable +from types import SimpleNamespace +from typing import Any + +import pytest + +from sqlspec.adapters.cockroach_psycopg.driver import CockroachPsycopgAsyncDriver +from sqlspec.adapters.psycopg.driver import PsycopgAsyncDriver +from sqlspec.core import SQL +from sqlspec.core.stack import StatementStack +from sqlspec.exceptions import StackExecutionError + +_SETTLE_TIMEOUT = 5.0 +_DRIVER_TYPES = [PsycopgAsyncDriver, CockroachPsycopgAsyncDriver] + + +class _Outcome: + """Decides how the awaited operation inside a managed block ends.""" + + def __init__(self, mode: str) -> None: + self.mode = mode + self.started = asyncio.Event() + self.error = RuntimeError("delivered through a future") + + async def run(self, *_: Any) -> None: + self.started.set() + if self.mode == "success": + await asyncio.sleep(0) + return + loop = asyncio.get_running_loop() + pending = loop.create_future() + if self.mode == "error": + loop.call_soon(pending.set_exception, self.error) + await pending + + +class _SuspendingContext: + """Async context manager whose exit yields to the event loop before finishing.""" + + def __init__(self, value: Any, events: "list[str]", label: str) -> None: + self._value = value + self._events = events + self._label = label + self.exit_errors: list[BaseException | None] = [] + + async def __aenter__(self) -> Any: + return self._value + + async def __aexit__(self, exc_type: Any, exc: "BaseException | None", traceback: Any) -> bool: + await asyncio.sleep(0) + self.exit_errors.append(exc) + self._events.append(self._label) + return False + + +class _Cursor: + description = None + rowcount = 1 + statusmessage = "INSERT 0 1" + + def __init__(self, connection: "_Connection", index: int) -> None: + self._connection = connection + self._index = index + + def copy(self, _sql: Any) -> _SuspendingContext: + return self._connection.copy_context + + async def execute(self, _sql: Any, _parameters: Any = None) -> None: + await self._connection.outcome.run() + + async def fetchall(self) -> "list[Any]": + return [] + + async def close(self) -> None: + await asyncio.sleep(0) + self._connection.events.append(f"cursor-{self._index}") + + +class _CopyRows: + """Copy handle whose writes and row iteration follow the outcome.""" + + def __init__(self, outcome: _Outcome) -> None: + self._outcome = outcome + self.write = outcome.run + self.write_row = outcome.run + + async def __aiter__(self) -> "AsyncIterator[Any]": + yield b"1\talice\n" + await self._outcome.run() + yield memoryview(b"2\tbob\n") + + +class _Connection: + def __init__(self, outcome: _Outcome) -> None: + self.autocommit = False + self.info = SimpleNamespace(transaction_status=0) + self.outcome = outcome + self.events: list[str] = [] + self.copy_context = _SuspendingContext(_CopyRows(outcome), self.events, "copy") + self.pipeline_context = _SuspendingContext(SimpleNamespace(sync=self._sync), self.events, "pipeline") + self._cursor_count = 0 + + def cursor(self) -> _Cursor: + cursor = _Cursor(self, self._cursor_count) + self._cursor_count += 1 + return cursor + + def pipeline(self) -> _SuspendingContext: + return self.pipeline_context + + async def commit(self) -> None: + self.events.append("commit") + + async def rollback(self) -> None: + self.events.append("rollback") + + async def _sync(self) -> None: + return None + + +def _copy_from(driver: Any, connection: _Connection) -> Any: + statement = SQL("COPY users FROM STDIN", "1\talice\n") + statement.compile() + return driver.dispatch_special_handling(connection.cursor(), statement) + + +def _copy_to(driver: Any, connection: _Connection) -> Any: + statement = SQL("COPY users TO STDOUT") + statement.compile() + return driver.dispatch_special_handling(connection.cursor(), statement) + + +def _arrow_copy(driver: Any, _connection: _Connection) -> Any: + import pyarrow as pa + + return driver.load_from_arrow("users", pa.table({"id": [1]})) + + +def _stack_pipeline(driver: Any, _connection: _Connection) -> Any: + return driver.execute_stack(StatementStack().push_execute("INSERT INTO users (id) VALUES (:id)", id=1)) + + +_SITES: "dict[str, tuple[Any, str, list[str], list[str]]]" = { + "copy-from": (_copy_from, "copy", ["copy"], ["copy"]), + "copy-to": (_copy_to, "copy", ["copy"], ["copy"]), + "arrow-copy": (_arrow_copy, "copy", ["copy", "cursor-0"], ["copy", "cursor-0"]), + "stack-pipeline": ( + _stack_pipeline, + "pipeline", + ["cursor-0", "pipeline", "commit"], + ["cursor-0", "pipeline", "rollback"], + ), +} + + +async def _settle(awaitable: "Awaitable[Any]", outcome: _Outcome) -> "asyncio.Task[Any]": + """Run the driver call to completion, cancelling it mid-block when the outcome asks for it.""" + task = asyncio.ensure_future(awaitable) + await asyncio.wait_for(outcome.started.wait(), _SETTLE_TIMEOUT) + if outcome.mode == "cancel": + task.cancel() + done, _ = await asyncio.wait({task}, timeout=_SETTLE_TIMEOUT) + assert task in done + return task + + +def _driver(driver_type: Any, connection: _Connection) -> Any: + return driver_type(connection, driver_features={"storage_capabilities": {"arrow_import_enabled": True}}) + + +@pytest.mark.parametrize("driver_type", _DRIVER_TYPES) +@pytest.mark.parametrize("site", list(_SITES)) +async def test_block_success_exits_each_context_once(driver_type: Any, site: str) -> None: + """A completed block exits its contexts once, inner first, without an error.""" + start, context_name, success_events, _ = _SITES[site] + outcome = _Outcome("success") + connection = _Connection(outcome) + + task = await _settle(start(_driver(driver_type, connection), connection), outcome) + + assert task.exception() is None + assert getattr(connection, f"{context_name}_context").exit_errors == [None] + assert connection.events == success_events + + +@pytest.mark.parametrize("driver_type", _DRIVER_TYPES) +@pytest.mark.parametrize("site", list(_SITES)) +async def test_block_error_delivered_through_a_future_propagates_after_exit(driver_type: Any, site: str) -> None: + """An error raised while the block awaits reaches the context exit once and then the caller.""" + start, context_name, _, failure_events = _SITES[site] + outcome = _Outcome("error") + connection = _Connection(outcome) + + task = await _settle(start(_driver(driver_type, connection), connection), outcome) + + raised = task.exception() + if site == "stack-pipeline": + assert isinstance(raised, StackExecutionError) + assert raised.__cause__ is outcome.error + else: + assert raised is outcome.error + assert getattr(connection, f"{context_name}_context").exit_errors == [raised] + assert connection.events == failure_events + + +@pytest.mark.parametrize("driver_type", _DRIVER_TYPES) +@pytest.mark.parametrize("site", list(_SITES)) +async def test_block_cancellation_propagates_after_exit(driver_type: Any, site: str) -> None: + """Cancelling the block runs the context exit once and leaves the task cancelled.""" + start, context_name, _, failure_events = _SITES[site] + outcome = _Outcome("cancel") + connection = _Connection(outcome) + + task = await _settle(start(_driver(driver_type, connection), connection), outcome) + + assert task.cancelled() + exit_errors = getattr(connection, f"{context_name}_context").exit_errors + assert [type(error) for error in exit_errors] == [asyncio.CancelledError] + assert connection.events == [event for event in failure_events if event != "rollback"] + + +@pytest.mark.parametrize("driver_type", _DRIVER_TYPES) +async def test_copy_to_returns_rows_read_before_the_context_exits(driver_type: Any) -> None: + """COPY TO STDOUT output is collected inside the block and returned after the exit.""" + outcome = _Outcome("success") + connection = _Connection(outcome) + + result = await _copy_to(_driver(driver_type, connection), connection) + + assert result.get_data() == [{"copy_output": "1\talice\n2\tbob\n"}] + assert result.rows_affected == 1 + + +@pytest.mark.parametrize("driver_type", _DRIVER_TYPES) +async def test_stack_pipeline_closes_cursors_in_reverse_order_before_the_pipeline(driver_type: Any) -> None: + """Pipeline cursors close last-opened first, then the pipeline exits, then the transaction commits.""" + outcome = _Outcome("success") + connection = _Connection(outcome) + stack = ( + StatementStack() + .push_execute("INSERT INTO users (id) VALUES (:id)", id=1) + .push_execute("INSERT INTO users (id) VALUES (:id)", id=2) + ) + + results = await _driver(driver_type, connection).execute_stack(stack) + + assert [result.rows_affected for result in results] == [1, 1] + assert connection.events == ["cursor-1", "cursor-0", "pipeline", "commit"] diff --git a/tests/unit/adapters/test_psycopg/test_driver.py b/tests/unit/adapters/test_psycopg/test_driver.py index 8dc72990d..2804cc26f 100644 --- a/tests/unit/adapters/test_psycopg/test_driver.py +++ b/tests/unit/adapters/test_psycopg/test_driver.py @@ -281,6 +281,13 @@ async def test_async_rollback_restores_changed_autocommit() -> None: ("SELECT %s AS value WHERE '100%' LIKE '100%'", (1,), "SELECT %s AS value WHERE '100%%' LIKE '100%%'"), ("SELECT %s AS value WHERE 'a%%b' LIKE 'a%%b'", (1,), "SELECT %s AS value WHERE 'a%%b' LIKE 'a%%b'"), ("SELECT ? AS value WHERE 'a%%b' LIKE 'a%%b'", (1,), "SELECT %s AS value WHERE 'a%%%%b' LIKE 'a%%%%b'"), + ("SELECT $1 AS value WHERE 'axb' LIKE '%x%'", (1,), "SELECT %s AS value WHERE 'axb' LIKE '%%x%%'"), + ("SELECT $1 AS value WHERE 10 % 3 = 1", (1,), "SELECT %s AS value WHERE 10 %% 3 = 1"), + ( + "SELECT %s AS value WHERE 10 %% 3 = 1 AND 'a%' <> '%s'", + (1,), + "SELECT %s AS value WHERE 10 %% 3 = 1 AND 'a%%' <> '%%s'", + ), ], ) async def test_cached_percent_sql_preserves_native_binding( @@ -673,3 +680,40 @@ def test_sync_load_from_arrow_preserves_cause_on_failure() -> None: driver.load_from_arrow("test_table", table, overwrite=True) assert exc_info.value.__cause__ is not None + + +class _CountingDescriptionCursor: + """Cursor whose description builds its column list on every read, as psycopg does.""" + + rowcount = 1 + + def __init__(self) -> None: + self.description_reads = 0 + + @property + def description(self) -> "list[SimpleNamespace]": + self.description_reads += 1 + return [SimpleNamespace(name="id", type_code=23), SimpleNamespace(name="name", type_code=25)] + + +@pytest.mark.parametrize("is_async", [False, True]) +async def test_select_dispatch_resolves_names_and_types_from_one_description_read(is_async: bool) -> None: + """A row-returning statement resolves column names and types from a single cursor description.""" + cursor: Any = _CountingDescriptionCursor() + if is_async: + cursor.execute = AsyncMock() + cursor.fetchall = AsyncMock(return_value=[(1, "a")]) + else: + cursor.execute = MagicMock() + cursor.fetchall = MagicMock(return_value=[(1, "a")]) + driver = PsycopgAsyncDriver(MagicMock()) if is_async else PsycopgSyncDriver(MagicMock()) + statement = SQL("SELECT id, name FROM users WHERE id = $1", 1, statement_config=driver.statement_config) + statement.compile() + + result = driver.dispatch_execute(cursor, statement) + if inspect.isawaitable(result): + result = await result + + assert result.column_names == ["id", "name"] + assert result.column_types == {"id": "int32", "name": "string"} + assert cursor.description_reads == 1 diff --git a/tests/unit/adapters/test_psycopg/test_percent.py b/tests/unit/adapters/test_psycopg/test_percent.py index a917b3b03..19fcf2f98 100644 --- a/tests/unit/adapters/test_psycopg/test_percent.py +++ b/tests/unit/adapters/test_psycopg/test_percent.py @@ -49,6 +49,7 @@ def test_qmark_source_keeps_double_percent_literal(sql: str, expected: str) -> N ("raw_sql", "expected"), [ ("SELECT %s", True), + ("SELECT %S", True), ("SELECT %(v)s", True), ("SELECT ?", False), ("SELECT :v", False), @@ -58,3 +59,126 @@ def test_qmark_source_keeps_double_percent_literal(sql: str, expected: str) -> N ) def test_source_uses_pyformat_detects_styles(raw_sql: str, expected: bool) -> None: assert source_uses_pyformat(raw_sql, default_statement_config.parameter_validator) is expected + + +@pytest.mark.parametrize( + ("sql", "parameters", "from_pyformat", "from_other_styles"), + [ + pytest.param( + "SELECT id FROM t WHERE id = %s AND status = %s", + (1, "a"), + "SELECT id FROM t WHERE id = %s AND status = %s", + "SELECT id FROM t WHERE id = %s AND status = %s", + id="placeholders-only", + ), + pytest.param( + "SELECT id FROM t WHERE id = %(id)s AND status = %(status)s", + {"id": 1, "status": "a"}, + "SELECT id FROM t WHERE id = %(id)s AND status = %(status)s", + "SELECT id FROM t WHERE id = %(id)s AND status = %(status)s", + id="named-placeholders-only", + ), + pytest.param( + "SELECT '100%' AS label FROM t WHERE id = %s", + (1,), + "SELECT '100%%' AS label FROM t WHERE id = %s", + "SELECT '100%%' AS label FROM t WHERE id = %s", + id="string-literal", + ), + pytest.param( + "SELECT id FROM t WHERE name LIKE '%x%' AND id = %s", + (1,), + "SELECT id FROM t WHERE name LIKE '%%x%%' AND id = %s", + "SELECT id FROM t WHERE name LIKE '%%x%%' AND id = %s", + id="like-pattern", + ), + pytest.param( + "SELECT id FROM t WHERE name LIKE '%%x%%' AND id = %s", + (1,), + "SELECT id FROM t WHERE name LIKE '%%x%%' AND id = %s", + "SELECT id FROM t WHERE name LIKE '%%%%x%%%%' AND id = %s", + id="doubled-percent", + ), + pytest.param( + "SELECT id % 2, 10%3, 7 %% 4 FROM t WHERE id = %s", + (1,), + "SELECT id %% 2, 10%%3, 7 %% 4 FROM t WHERE id = %s", + "SELECT id %% 2, 10%%3, 7 %%%% 4 FROM t WHERE id = %s", + id="modulo", + ), + pytest.param( + "SELECT '%s', '%(id)s' FROM t WHERE id = %(id)s -- 50% of %s\n AND name = %(name)s", + {"id": 1, "name": "a"}, + "SELECT '%%s', '%%(id)s' FROM t WHERE id = %(id)s -- 50%% of %%s\n AND name = %(name)s", + "SELECT '%%s', '%%(id)s' FROM t WHERE id = %(id)s -- 50%% of %%s\n AND name = %(name)s", + id="placeholder-text-in-literal-and-comment", + ), + pytest.param( + "SELECT $1, :name, ?, %s, %(key)s, 5 % 2, '%'", + (1,), + "SELECT $1, :name, ?, %s, %(key)s, 5 %% 2, '%%'", + "SELECT $1, :name, ?, %s, %(key)s, 5 %% 2, '%%'", + id="mixed-placeholder-styles", + ), + pytest.param( + "SELECT '%%%s', %s, '%%%' || %s || '%", + (1, 2), + "SELECT '%%%%s', %s, '%%%%' || %s || '%%", + "SELECT '%%%%%%s', %s, '%%%%%%' || %s || '%%", + id="odd-percent-runs", + ), + pytest.param( + "SELECT $$50% of %s$$, 'café 5%', %s %", + [1], + "SELECT $$50%% of %%s$$, 'café 5%%', %s %%", + "SELECT $$50%% of %%s$$, 'café 5%%', %s %%", + id="dollar-quote-unicode-trailing-percent", + ), + ], +) +def test_literal_percent_escaping_by_source_style( + sql: str, parameters: object, from_pyformat: str, from_other_styles: str +) -> None: + """Literal percents are doubled, placeholders are kept, and a pyformat source keeps its own escapes.""" + validator = default_statement_config.parameter_validator + + assert escape_literal_percent(sql, parameters, validator, preserve_percent_escapes=True) == from_pyformat + assert escape_literal_percent(sql, parameters, validator, preserve_percent_escapes=False) == from_other_styles + assert escape_literal_percent(sql, parameters, validator) == from_other_styles + + +@pytest.mark.parametrize("parameters", [None, (), [], {}]) +@pytest.mark.parametrize( + "sql", + [ + "SELECT id FROM t WHERE name LIKE '%x%'", + "SELECT id % 2, '100%', '%s', 'a%%b' FROM t", + "SELECT id FROM t WHERE id = %s AND name LIKE '%'", + ], +) +@pytest.mark.parametrize("preserve_percent_escapes", [False, True]) +def test_sql_without_parameters_is_sent_as_written( + sql: str, parameters: object, preserve_percent_escapes: bool +) -> None: + """psycopg leaves percents alone when no parameters are passed, so nothing is escaped.""" + validator = default_statement_config.parameter_validator + + escaped = escape_literal_percent(sql, parameters, validator, preserve_percent_escapes=preserve_percent_escapes) + + assert escaped == sql + + +@pytest.mark.parametrize( + ("raw_sql", "expected"), + [ + ("SELECT id FROM t WHERE id = $1 AND name LIKE '%x%'", False), + ("SELECT id % 2 FROM t WHERE id = :id", False), + ("SELECT '%(id)s' -- %s\n FROM t WHERE id = ?", False), + ("SELECT 10 %% 3 FROM t WHERE id = $1", False), + ("SELECT id FROM t WHERE id = $1 AND name = %s", True), + ("SELECT id % 2 FROM t WHERE id = %(id)s", True), + ("SELECT id FROM t", False), + ], +) +def test_source_uses_pyformat_ignores_percents_that_are_not_placeholders(raw_sql: str, expected: bool) -> None: + assert source_uses_pyformat(raw_sql, default_statement_config.parameter_validator) is expected diff --git a/tests/unit/adapters/test_psycopg/test_stack_parameters.py b/tests/unit/adapters/test_psycopg/test_stack_parameters.py new file mode 100644 index 000000000..eb51416a3 --- /dev/null +++ b/tests/unit/adapters/test_psycopg/test_stack_parameters.py @@ -0,0 +1,255 @@ +"""Pipeline stack preparation hands psycopg the parameters each statement shape needs.""" + +import contextlib +import inspect +from collections.abc import AsyncIterator, Iterator +from types import SimpleNamespace +from typing import Any + +import pytest + +from sqlspec.adapters.cockroach_psycopg.driver import CockroachPsycopgAsyncDriver, CockroachPsycopgSyncDriver +from sqlspec.adapters.psycopg.driver import PsycopgAsyncDriver, PsycopgSyncDriver +from sqlspec.core.stack import StatementStack + +_POSITIONAL_SQL = "INSERT INTO t (id) VALUES (%s)" + +_STACK_CASES = [ + pytest.param(_POSITIONAL_SQL, ((1,),), {}, _POSITIONAL_SQL, [1], id="tuple"), + pytest.param(_POSITIONAL_SQL, ([1],), {}, _POSITIONAL_SQL, [1], id="list"), + pytest.param(_POSITIONAL_SQL, (1,), {}, _POSITIONAL_SQL, [1], id="scalar"), + pytest.param( + "INSERT INTO t (id, name) VALUES (%s, %s)", + (1, "a"), + {}, + "INSERT INTO t (id, name) VALUES (%s, %s)", + [1, "a"], + id="scalars", + ), + pytest.param("INSERT INTO t (id) VALUES (:id)", (), {"id": 1}, _POSITIONAL_SQL, [1], id="named"), + pytest.param( + "INSERT INTO t (id) VALUES (%(id)s)", + (), + {"id": 1}, + "INSERT INTO t (id) VALUES (%(id)s)", + {"id": 1}, + id="named-pyformat", + ), + pytest.param("INSERT INTO t (id) VALUES (1)", (), {}, "INSERT INTO t (id) VALUES (1)", None, id="no-parameters"), +] + +_DRIVER_TYPES = [PsycopgSyncDriver, PsycopgAsyncDriver, CockroachPsycopgSyncDriver, CockroachPsycopgAsyncDriver] + + +class _Cursor: + description = None + rowcount = 1 + statusmessage = "INSERT 0 1" + + def __init__(self, sent: "list[tuple[str, Any]]") -> None: + self._sent = sent + + def execute(self, sql: Any, params: Any = None) -> None: + self._sent.append((str(sql), params)) + + def fetchall(self) -> "list[Any]": + return [] + + def close(self) -> None: + return None + + +class _AsyncCursor(_Cursor): + async def execute(self, sql: Any, params: Any = None) -> None: # type: ignore[override] + self._sent.append((str(sql), params)) + + async def fetchall(self) -> "list[Any]": # type: ignore[override] + return [] + + async def close(self) -> None: # type: ignore[override] + return None + + +class _RowCursor(_Cursor): + """Cursor returning one row whose description builds its column list on every read, as psycopg does.""" + + description_reads = 0 + + @property + def description(self) -> "list[SimpleNamespace]": # type: ignore[override] + self.description_reads += 1 + return [SimpleNamespace(name="id", type_code=23), SimpleNamespace(name="name", type_code=25)] + + def fetchall(self) -> "list[Any]": + return [(1, "a")] + + +class _AsyncRowCursor(_RowCursor): + async def execute(self, sql: Any, params: Any = None) -> None: # type: ignore[override] + self._sent.append((str(sql), params)) + + async def fetchall(self) -> "list[Any]": # type: ignore[override] + return [(1, "a")] + + async def close(self) -> None: # type: ignore[override] + return None + + +class _SyncConnection: + def __init__(self, cursor_type: "type[_Cursor]" = _Cursor) -> None: + self.autocommit = False + self.info = SimpleNamespace(transaction_status=0) + self.sent: list[tuple[str, Any]] = [] + self.transaction_calls: list[str] = [] + self.cursors: list[Any] = [] + self._cursor_type = cursor_type + + def cursor(self) -> _Cursor: + cursor = self._cursor_type(self.sent) + self.cursors.append(cursor) + return cursor + + @contextlib.contextmanager + def pipeline(self) -> "Iterator[Any]": + yield SimpleNamespace(sync=lambda: None) + + def commit(self) -> None: + self.transaction_calls.append("commit") + + def rollback(self) -> None: + self.transaction_calls.append("rollback") + + +class _AsyncPipeline: + async def sync(self) -> None: + return None + + +class _AsyncConnection: + def __init__(self, cursor_type: "type[_Cursor]" = _AsyncCursor) -> None: + self.autocommit = False + self.info = SimpleNamespace(transaction_status=0) + self.sent: list[tuple[str, Any]] = [] + self.transaction_calls: list[str] = [] + self.cursors: list[Any] = [] + self._cursor_type = cursor_type + + def cursor(self) -> _Cursor: + cursor = self._cursor_type(self.sent) + self.cursors.append(cursor) + return cursor + + @contextlib.asynccontextmanager + async def pipeline(self) -> "AsyncIterator[Any]": + yield _AsyncPipeline() + + async def commit(self) -> None: + self.transaction_calls.append("commit") + + async def rollback(self) -> None: + self.transaction_calls.append("rollback") + + +def _driver(driver_type: Any, *, rows: bool = False) -> "tuple[Any, Any]": + connection: Any + if inspect.iscoroutinefunction(driver_type.execute_stack): + connection = _AsyncConnection(_AsyncRowCursor) if rows else _AsyncConnection() + else: + connection = _SyncConnection(_RowCursor) if rows else _SyncConnection() + return driver_type(connection), connection + + +def _psycopg_arguments(parameters: Any) -> Any: + """Return the value psycopg receives: a list for sequences, the mapping itself, or None when omitted.""" + if not parameters: + return None + if isinstance(parameters, dict): + return parameters + assert isinstance(parameters, (list, tuple)) + return list(parameters) + + +@pytest.mark.parametrize("driver_type", _DRIVER_TYPES) +@pytest.mark.parametrize(("sql", "arguments", "keyword_arguments", "expected_sql", "expected_parameters"), _STACK_CASES) +def test_prepared_stack_operation_holds_parameters_psycopg_accepts( + driver_type: Any, + sql: str, + arguments: "tuple[Any, ...]", + keyword_arguments: "dict[str, Any]", + expected_sql: str, + expected_parameters: Any, +) -> None: + """Sequence placeholders get a list or tuple, named placeholders a dict, and no placeholders nothing.""" + driver, _ = _driver(driver_type) + stack = StatementStack().push_execute(sql, *arguments, **keyword_arguments) + + prepared = driver._prepare_pipeline_operations(stack) or [] + + assert len(prepared) == 1 + assert prepared[0].operation_index == 0 + assert prepared[0].sql == expected_sql + assert _psycopg_arguments(prepared[0].parameters) == expected_parameters + + +@pytest.mark.parametrize("driver_type", _DRIVER_TYPES) +@pytest.mark.parametrize(("sql", "arguments", "keyword_arguments", "expected_sql", "expected_parameters"), _STACK_CASES) +async def test_execute_stack_sends_prepared_parameters_through_the_pipeline( + driver_type: Any, + sql: str, + arguments: "tuple[Any, ...]", + keyword_arguments: "dict[str, Any]", + expected_sql: str, + expected_parameters: Any, +) -> None: + """Every statement shape reaches the pipeline cursor and commits the stack transaction.""" + driver, connection = _driver(driver_type) + stack = StatementStack().push_execute(sql, *arguments, **keyword_arguments) + + results = driver.execute_stack(stack) + if inspect.isawaitable(results): + results = await results + + assert [result.rows_affected for result in results] == [1] + assert [(sent_sql, _psycopg_arguments(sent)) for sent_sql, sent in connection.sent] == [ + (expected_sql, expected_parameters) + ] + assert connection.transaction_calls == ["commit"] + + +@pytest.mark.parametrize("driver_type", _DRIVER_TYPES) +async def test_execute_stack_keeps_each_operation_with_its_own_parameters(driver_type: Any) -> None: + """A stack mixing parameter shapes sends each statement with its own prepared value.""" + driver, connection = _driver(driver_type) + stack = ( + StatementStack() + .push_execute(_POSITIONAL_SQL, (1,)) + .push_execute("DELETE FROM t") + .push_execute("UPDATE t SET name = :name WHERE id = :id", name="a", id=2) + .push_execute("INSERT INTO t (id) VALUES (%(id)s)", id=3) + ) + + results = driver.execute_stack(stack) + if inspect.isawaitable(results): + results = await results + + assert len(results) == 4 + assert [(sent_sql, _psycopg_arguments(sent)) for sent_sql, sent in connection.sent] == [ + (_POSITIONAL_SQL, [1]), + ("DELETE FROM t", None), + ("UPDATE t SET name = %s WHERE id = %s", ["a", 2]), + ("INSERT INTO t (id) VALUES (%(id)s)", {"id": 3}), + ] + + +@pytest.mark.parametrize("driver_type", _DRIVER_TYPES) +async def test_execute_stack_select_resolves_columns_from_one_description_read(driver_type: Any) -> None: + """A row-returning stack statement builds its result from a single cursor description.""" + driver, connection = _driver(driver_type, rows=True) + stack = StatementStack().push_execute("SELECT id, name FROM t WHERE id = :id", id=1) + + results = driver.execute_stack(stack) + if inspect.isawaitable(results): + results = await results + + assert results[0].get_result().get_data() == [{"id": 1, "name": "a"}] + assert [cursor.description_reads for cursor in connection.cursors] == [1] diff --git a/tests/unit/adapters/test_pymysql/test_cache_hit_executor.py b/tests/unit/adapters/test_pymysql/test_cache_hit_executor.py index 4dbd5876e..878e8e29f 100644 --- a/tests/unit/adapters/test_pymysql/test_cache_hit_executor.py +++ b/tests/unit/adapters/test_pymysql/test_cache_hit_executor.py @@ -43,7 +43,7 @@ def test_execute_cache_hit_returns_rows_with_json_decoded() -> None: """A row-returning statement yields decoded tuple rows, column names, and the row count.""" cursor = MagicMock() cursor.description = [("id", 3), ("payload", JSON_TYPE_CODE)] - cursor.fetchall.return_value = [(1, '{"name": "alpha"}'), (2, None)] + cursor.fetchall.return_value = ((1, '{"name": "alpha"}'), (2, None)) connection = _connection(cursor) sql = "SELECT id, payload FROM items WHERE id > %s" cached = _cached(sql, "SELECT", returns_rows=True, param_count=1) @@ -122,7 +122,7 @@ def test_driver_runs_repeated_statement_through_executor() -> None: """A repeated raw statement on the driver returns the same rows on the cached execution.""" cursor = MagicMock() cursor.description = [("value", 3)] - cursor.fetchall.return_value = [(1,)] + cursor.fetchall.return_value = ((1,),) connection = _connection(cursor) connection.server_status = 0 driver = PyMysqlDriver(cast("Any", connection)) @@ -142,7 +142,7 @@ def test_execute_cache_hit_matches_driver_cache_hit_method() -> None: """The executor and the driver cache-hit method produce the same rows, columns, and count.""" cursor = MagicMock() cursor.description = [("id", 3), ("payload", JSON_TYPE_CODE)] - cursor.fetchall.side_effect = [[(1, '{"name": "alpha"}')], [(1, '{"name": "alpha"}')]] + cursor.fetchall.side_effect = [((1, '{"name": "alpha"}'),), ((1, '{"name": "alpha"}'),)] connection = _connection(cursor) connection.server_status = 0 driver = PyMysqlDriver(cast("Any", connection)) diff --git a/tests/unit/adapters/test_pymysql/test_driver.py b/tests/unit/adapters/test_pymysql/test_driver.py index b3dccfc7e..73342f5c3 100644 --- a/tests/unit/adapters/test_pymysql/test_driver.py +++ b/tests/unit/adapters/test_pymysql/test_driver.py @@ -30,8 +30,8 @@ def execute(self, statement: str, parameters: Any = None) -> None: def executemany(self, statement: str, parameters: Any = None) -> None: self.connection.executed.append((statement, parameters)) - def fetchall(self) -> "list[Any]": - return [] + def fetchall(self) -> "tuple[Any, ...]": + return () class _FakeConnection: diff --git a/tests/unit/adapters/test_pymysql/test_load_from_arrow.py b/tests/unit/adapters/test_pymysql/test_load_from_arrow.py index dc711f707..7f9685ea7 100644 --- a/tests/unit/adapters/test_pymysql/test_load_from_arrow.py +++ b/tests/unit/adapters/test_pymysql/test_load_from_arrow.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import Any, cast -from unittest.mock import patch import pyarrow as pa import pyarrow.parquet as pq @@ -131,8 +130,7 @@ def test_load_from_arrow_needs_preparation_bypasses_local_infile() -> None: conn = _FakeConnection() driver = _make_driver(conn, enable_local_infile=True) - with patch.object(type(driver), "_arrow_rows_need_preparation", lambda _self, _table: True): - driver.load_from_arrow("orders", pa.table({"id": [1], "name": ["a"]})) + driver.load_from_arrow("orders", pa.table({"id": [1], "tags": [["a", "b"]]})) assert conn._cursor.execute_calls == [] assert len(conn._cursor.executemany_calls) == 1 diff --git a/tests/unit/adapters/test_session_stream_teardown.py b/tests/unit/adapters/test_session_stream_teardown.py index 3c64c32ca..eb91de4bc 100644 --- a/tests/unit/adapters/test_session_stream_teardown.py +++ b/tests/unit/adapters/test_session_stream_teardown.py @@ -4,7 +4,7 @@ import importlib import inspect import sqlite3 -from collections.abc import AsyncIterator +from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from pathlib import Path from typing import Any @@ -132,7 +132,7 @@ def embedded(request: pytest.FixtureRequest, tmp_path: Path) -> "tuple[str, Any, @asynccontextmanager -async def session_scope(config: Any) -> "AsyncIterator[Any]": +async def session_scope(config: Any) -> "AsyncGenerator[Any, None]": if config.is_async: async with config.provide_session() as session: yield session diff --git a/tests/unit/adapters/test_snowflake/_store_helpers.py b/tests/unit/adapters/test_snowflake/_store_helpers.py index d2c4b31b6..d565eb33b 100644 --- a/tests/unit/adapters/test_snowflake/_store_helpers.py +++ b/tests/unit/adapters/test_snowflake/_store_helpers.py @@ -1,6 +1,6 @@ """External database boundary fixtures for Snowflake extension store tests.""" -from collections.abc import Iterator +from collections.abc import Generator from contextlib import contextmanager from typing import Any from unittest.mock import MagicMock @@ -24,7 +24,7 @@ def store_config( driver.select.return_value = [] @contextmanager - def provide_session(self: SnowflakeConfig, *args: Any, **kwargs: Any) -> Iterator[Any]: + def provide_session(self: SnowflakeConfig, *args: Any, **kwargs: Any) -> Generator[Any, None, None]: yield driver monkeypatch.setattr(SnowflakeConfig, "provide_session", provide_session) diff --git a/tests/unit/adapters/test_spanner/test_adk_store.py b/tests/unit/adapters/test_spanner/test_adk_store.py index 005c516c9..4f178a5b7 100644 --- a/tests/unit/adapters/test_spanner/test_adk_store.py +++ b/tests/unit/adapters/test_spanner/test_adk_store.py @@ -1,6 +1,6 @@ """Unit tests for Spanner ADK store behavior.""" -from collections.abc import Iterator +from collections.abc import Generator from contextlib import contextmanager from datetime import UTC, datetime, timedelta from inspect import isawaitable @@ -523,7 +523,7 @@ def _normalized(sql: str) -> str: @contextmanager -def _capture_session_list(config: Any) -> "Iterator[list[tuple[str, dict[str, Any], dict[str, Any]]]]": +def _capture_session_list(config: Any) -> "Generator[list[tuple[str, dict[str, Any], dict[str, Any]]], None, None]": calls: list[tuple[str, dict[str, Any], dict[str, Any]]] = [] def capture( diff --git a/tests/unit/adapters/test_spanner/test_config.py b/tests/unit/adapters/test_spanner/test_config.py index d5444a6b5..2b4060b59 100644 --- a/tests/unit/adapters/test_spanner/test_config.py +++ b/tests/unit/adapters/test_spanner/test_config.py @@ -1,5 +1,5 @@ from asyncio import CancelledError -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncGenerator, Callable from contextlib import asynccontextmanager from datetime import timedelta from types import SimpleNamespace @@ -84,7 +84,7 @@ def _hook_checkout(is_async: bool, hook: Callable[[Any], Any], *, replicas: bool @asynccontextmanager -async def _hook_session(config: Any, **kwargs: Any) -> AsyncIterator[Any]: +async def _hook_session(config: Any, **kwargs: Any) -> AsyncGenerator[Any, None]: if isinstance(config, SpannerAsyncConfig): async with config.provide_session(**kwargs) as session: yield session diff --git a/tests/unit/adapters/test_spanner/test_litestar_store.py b/tests/unit/adapters/test_spanner/test_litestar_store.py index c9ba8b9ca..14d2fabac 100644 --- a/tests/unit/adapters/test_spanner/test_litestar_store.py +++ b/tests/unit/adapters/test_spanner/test_litestar_store.py @@ -411,7 +411,7 @@ def execute_update(self, _sql: str, params: Any = None, param_types: Any = None) if expect_future: assert before + timedelta(milliseconds=200) <= bound_expires_at <= after + timedelta(milliseconds=300) else: - assert before - timedelta(microseconds=1) <= bound_expires_at <= after - timedelta(microseconds=1) + assert before - timedelta(seconds=1) <= bound_expires_at <= after - timedelta(seconds=1) @pytest.mark.parametrize("mode", ["sync", "async"]) diff --git a/tests/unit/adapters/test_sqlite/test_driver.py b/tests/unit/adapters/test_sqlite/test_driver.py index 99514a3a9..87a76ad20 100644 --- a/tests/unit/adapters/test_sqlite/test_driver.py +++ b/tests/unit/adapters/test_sqlite/test_driver.py @@ -6,7 +6,7 @@ import pytest from sqlglot import parse_one -from sqlspec.adapters.sqlite import SqliteConfig, SqliteDriver +from sqlspec.adapters.sqlite import SqliteConfig, SqliteCursor, SqliteDriver from sqlspec.adapters.sqlite.core import SQLITE_CONNECT_SUPPORTS_AUTOCOMMIT, default_statement_config, resolve_lastrowid from sqlspec.adapters.sqlite.pool import SqliteConnectionPool from sqlspec.adapters.sqlite.type_converter import json_converter, register_type_handlers, unregister_type_handlers @@ -234,30 +234,31 @@ def test_execute_many_closes_cursor() -> None: connection.close() -def test_arrow_ingest_rolls_back_when_a_later_batch_conversion_fails(monkeypatch: pytest.MonkeyPatch) -> None: +def test_arrow_ingest_rolls_back_when_a_later_batch_conversion_fails() -> None: import pyarrow as pa - connection = sqlite3.connect(":memory:") - connection.execute("CREATE TABLE target (value INTEGER)") - driver = SqliteDriver( - connection=connection, driver_features={"storage_capabilities": {"arrow_import_enabled": True}} - ) batches_prepared = 0 - def prepare(parameters: Any, *args: Any, **kwargs: Any) -> Any: - nonlocal batches_prepared - batches_prepared += 1 - if batches_prepared == 2: - raise ValueError("conversion failed") - return parameters + class _FailingPrepareSqliteDriver(SqliteDriver): + def prepare_driver_parameters( + self, parameters: Any, statement_config: Any, is_many: bool = False, prepared_statement: Any = None + ) -> Any: + nonlocal batches_prepared + batches_prepared += 1 + if batches_prepared == 2: + raise ValueError("conversion failed") + return super().prepare_driver_parameters(parameters, statement_config, is_many, prepared_statement) - monkeypatch.setattr(SqliteDriver, "_arrow_rows_need_preparation", lambda *args: True) - monkeypatch.setattr( - SqliteDriver, "prepare_driver_parameters", lambda self, *args, **kwargs: prepare(*args, **kwargs) + connection = sqlite3.connect(":memory:") + connection.execute("CREATE TABLE target (value TEXT)") + driver = _FailingPrepareSqliteDriver( + connection=connection, driver_features={"storage_capabilities": {"arrow_import_enabled": True}} ) + nested_values = pa.table({"value": [[number] for number in range(10001)]}) try: with pytest.raises(ValueError, match="conversion failed"): - driver.load_from_arrow("target", pa.table({"value": list(range(10001))})) + driver.load_from_arrow("target", nested_values) + assert batches_prepared == 2 assert connection.in_transaction is False assert connection.execute("SELECT COUNT(*) FROM target").fetchone() == (0,) finally: @@ -528,3 +529,24 @@ def fail_on_second_row(value: int) -> int: session.execute(statement) finally: config.close_pool() + + +def test_cursor_context_closes_the_cursor_and_ignores_a_failing_close() -> None: + closed: list[str] = [] + + class FailingCursor: + def close(self) -> None: + closed.append("close") + raise sqlite3.ProgrammingError("Cannot operate on a closed database.") + + class Connection: + def cursor(self) -> FailingCursor: + return FailingCursor() + + with SqliteCursor(cast("Any", Connection())) as cursor: + assert isinstance(cursor, FailingCursor) + assert closed == ["close"] + + with pytest.raises(ValueError, match="statement failed"), SqliteCursor(cast("Any", Connection())): + raise ValueError("statement failed") + assert closed == ["close", "close"] diff --git a/tests/unit/adapters/test_sqlite/test_load_from_arrow_transaction.py b/tests/unit/adapters/test_sqlite/test_load_from_arrow_transaction.py index 9d1f5e175..d1458ce23 100644 --- a/tests/unit/adapters/test_sqlite/test_load_from_arrow_transaction.py +++ b/tests/unit/adapters/test_sqlite/test_load_from_arrow_transaction.py @@ -45,6 +45,8 @@ def __init__(self, *, in_transaction: bool = False, executemany_error: "Exceptio def execute(self, sql: str, *args: Any) -> "_FakeCursor": self.events.append(sql) + if sql.startswith("BEGIN"): + self.in_transaction = True return self._cursor def cursor(self) -> "_FakeCursor": @@ -52,9 +54,11 @@ def cursor(self) -> "_FakeCursor": def commit(self) -> None: self.events.append("COMMIT") + self.in_transaction = False def rollback(self) -> None: self.events.append("ROLLBACK") + self.in_transaction = False def _driver(conn: "_FakeConnection") -> SqliteDriver: diff --git a/tests/unit/adapters/test_suspending_context_exit.py b/tests/unit/adapters/test_suspending_context_exit.py new file mode 100644 index 000000000..8447f9c4b --- /dev/null +++ b/tests/unit/adapters/test_suspending_context_exit.py @@ -0,0 +1,352 @@ +"""Async driver paths exit a suspending context exactly once and keep the managed block's outcome.""" + +import asyncio +from collections.abc import AsyncIterator, Awaitable, Callable +from typing import Any, cast + +import pytest + +import sqlspec.adapters.mysqlconnector.driver as mysqlconnector_driver +from sqlspec.adapters.asyncpg.driver import AsyncpgDriver +from sqlspec.adapters.db2.driver import Db2AsyncDriver +from sqlspec.adapters.mysqlconnector._typing import MysqlConnectorError +from sqlspec.adapters.mysqlconnector.driver import MysqlConnectorAsyncDriver +from sqlspec.adapters.oracledb.driver import OracleAsyncDriver +from sqlspec.adapters.spanner.driver import SpannerAsyncDriver +from sqlspec.core import StatementStack +from sqlspec.exceptions import SQLSpecError, StackExecutionError + +pytestmark = pytest.mark.anyio + +_OUTCOMES = ("succeeds", "raises", "cancelled") +_STORAGE_CAPABILITIES: dict[str, Any] = { + "arrow_export_enabled": True, + "arrow_import_enabled": True, + "parquet_export_enabled": True, + "parquet_import_enabled": True, + "partition_strategies": [], +} + + +class _SuspendingContext: + """Async context manager whose exit yields to the event loop before recording its arguments.""" + + def __init__(self) -> None: + self.value: Any = None + self.entered = 0 + self.exits: list[tuple[type[BaseException] | None, BaseException | None]] = [] + + async def __aenter__(self) -> Any: + self.entered += 1 + return self.value + + async def __aexit__( + self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: object + ) -> None: + await asyncio.sleep(0) + self.exits.append((exc_type, exc_val)) + + +class _Body: + """Managed block stand-in that returns, raises, or waits until it is cancelled.""" + + def __init__(self, outcome: str, result: Any = None) -> None: + self.outcome = outcome + self.result = result + self.error = RuntimeError("managed block failed") + self.started = asyncio.Event() + + async def run(self, *_args: Any, **_kwargs: Any) -> Any: + self.started.set() + await asyncio.sleep(0) + if self.outcome == "raises": + raise self.error + if self.outcome == "cancelled": + await asyncio.Event().wait() + return self.result + + +class _Cursor: + """Cursor stand-in whose statement calls run the managed block.""" + + def __init__(self, body: _Body) -> None: + self.execute = body.run + self.executemany = body.run + + async def fetchone(self) -> "tuple[str]": + return ("APP",) + + +class _MutationGroups: + """Spanner mutation-group stand-in whose batch write runs the managed block.""" + + def __init__(self, body: _Body) -> None: + self._body = body + self.rows: list[Any] = [] + + def group(self) -> "_MutationGroups": + return self + + def insert_or_update(self, table: str, columns: "list[str]", values: "list[list[Any]]") -> None: + self.rows.extend(values) + + async def batch_write(self) -> "AsyncIterator[Any]": + await self._body.run() + return _no_responses() + + +class _MutationGroupsDatabase: + def __init__(self, context: _SuspendingContext) -> None: + self._context = context + + def mutation_groups(self) -> _SuspendingContext: + return self._context + + +class _AsyncpgConnection: + """asyncpg connection stand-in that suspends on every round trip.""" + + def __init__(self, body: _Body, transaction: "_SuspendingContext | None", *, in_transaction: bool) -> None: + self._body = body + self._transaction = transaction + self._in_transaction = in_transaction + self.commands: list[str] = [] + + def is_in_transaction(self) -> bool: + return self._in_transaction + + def transaction(self) -> "_SuspendingContext | None": + return self._transaction + + async def execute(self, sql: str, *_args: Any, **_kwargs: Any) -> str: + self.commands.append(sql.split()[0]) + if sql.startswith("UPDATE"): + return cast("str", await self._body.run()) + await asyncio.sleep(0) + return sql.split()[0] + + +async def _no_responses() -> "AsyncIterator[Any]": + responses: tuple[Any, ...] = () + for response in responses: + yield response + + +def _db2_driver(body: _Body, context: _SuspendingContext) -> Db2AsyncDriver: + class _Driver(Db2AsyncDriver): + def with_cursor(self, connection: Any) -> Any: + return context + + context.value = _Cursor(body) + return _Driver(connection=object()) + + +def _db2_set_schema( + body: _Body, context: _SuspendingContext, monkeypatch: pytest.MonkeyPatch +) -> "tuple[Callable[[], Awaitable[Any]], Any]": + driver = _db2_driver(body, context) + return lambda: driver.set_migration_session_schema("tenant"), None + + +def _db2_reset_schema( + body: _Body, context: _SuspendingContext, monkeypatch: pytest.MonkeyPatch +) -> "tuple[Callable[[], Awaitable[Any]], Any]": + driver = _db2_driver(body, context) + driver._migration_schema_restore = "APP" # pyright: ignore[reportPrivateUsage] + return driver.reset_migration_session_schema, None + + +def _db2_has_schema( + body: _Body, context: _SuspendingContext, monkeypatch: pytest.MonkeyPatch +) -> "tuple[Callable[[], Awaitable[Any]], Any]": + driver = _db2_driver(body, context) + return lambda: driver.has_schema("tenant"), True + + +def _oracle_load_without_indexes( + body: _Body, context: _SuspendingContext, monkeypatch: pytest.MonkeyPatch +) -> "tuple[Callable[[], Awaitable[Any]], Any]": + class _Driver(OracleAsyncDriver): + def without_indexes(self, table: str, *, schema: "str | None" = None, concurrent: bool = False) -> Any: + return context + + async def load_from_arrow(self, table: str, source: Any, **kwargs: Any) -> Any: + if kwargs.get("drop_indexes"): + return await super().load_from_arrow(table, source, **kwargs) + return await body.run() + + body.result = object() + driver = _Driver(cast("Any", object()), driver_features={"storage_capabilities": _STORAGE_CAPABILITIES}) + return lambda: driver.load_from_arrow("t", object(), drop_indexes=True), body.result + + +def _spanner_load_without_indexes( + body: _Body, context: _SuspendingContext, monkeypatch: pytest.MonkeyPatch +) -> "tuple[Callable[[], Awaitable[Any]], Any]": + class _Driver(SpannerAsyncDriver): + def without_indexes(self, table: str, *, schema: "str | None" = None, concurrent: bool = False) -> Any: + return context + + async def load_from_arrow(self, table: str, source: Any, **kwargs: Any) -> Any: + if kwargs.get("drop_indexes"): + return await super().load_from_arrow(table, source, **kwargs) + return await body.run() + + body.result = object() + driver = _Driver(cast("Any", object()), driver_features={"storage_capabilities": _STORAGE_CAPABILITIES}) + return lambda: driver.load_from_arrow("t", object(), drop_indexes=True), body.result + + +def _spanner_batch_write( + body: _Body, context: _SuspendingContext, monkeypatch: pytest.MonkeyPatch +) -> "tuple[Callable[[], Awaitable[Any]], Any]": + class _Driver(SpannerAsyncDriver): + def _resolve_database(self) -> Any: + return _MutationGroupsDatabase(context) + + context.value = _MutationGroups(body) + driver = _Driver(cast("Any", object())) + return lambda: driver._batch_write_mutations("t", ["id"], [[[1]], [[2]]]), None # pyright: ignore[reportPrivateUsage] + + +def _mysqlconnector_driver( + body: _Body, context: _SuspendingContext, monkeypatch: pytest.MonkeyPatch +) -> MysqlConnectorAsyncDriver: + context.value = _Cursor(body) + monkeypatch.setattr(mysqlconnector_driver, "MysqlConnectorAsyncCursor", lambda *_args: context) + return MysqlConnectorAsyncDriver(cast("Any", object())) + + +def _mysqlconnector_begin( + body: _Body, context: _SuspendingContext, monkeypatch: pytest.MonkeyPatch +) -> "tuple[Callable[[], Awaitable[Any]], Any]": + return _mysqlconnector_driver(body, context, monkeypatch).begin, None + + +def _mysqlconnector_execute( + body: _Body, context: _SuspendingContext, monkeypatch: pytest.MonkeyPatch +) -> "tuple[Callable[[], Awaitable[Any]], Any]": + driver = _mysqlconnector_driver(body, context, monkeypatch) + return lambda: driver._execute_cursor_statement("DELETE FROM t"), None # pyright: ignore[reportPrivateUsage] + + +def _mysqlconnector_execute_unprepared( + body: _Body, context: _SuspendingContext, monkeypatch: pytest.MonkeyPatch +) -> "tuple[Callable[[], Awaitable[Any]], Any]": + driver = _mysqlconnector_driver(body, context, monkeypatch) + return lambda: driver._execute_cursor_statement_unprepared("TRUNCATE TABLE t", ("x",)), None # pyright: ignore[reportPrivateUsage] + + +def _mysqlconnector_executemany( + body: _Body, context: _SuspendingContext, monkeypatch: pytest.MonkeyPatch +) -> "tuple[Callable[[], Awaitable[Any]], Any]": + driver = _mysqlconnector_driver(body, context, monkeypatch) + return lambda: driver._executemany_cursor_statement("INSERT INTO t VALUES (%s)", [(1,)]), None # pyright: ignore[reportPrivateUsage] + + +_SCENARIOS = ( + _db2_set_schema, + _db2_reset_schema, + _db2_has_schema, + _oracle_load_without_indexes, + _spanner_load_without_indexes, + _spanner_batch_write, + _mysqlconnector_begin, + _mysqlconnector_execute, + _mysqlconnector_execute_unprepared, + _mysqlconnector_executemany, +) + + +async def _cancel_once_started(task: "asyncio.Future[Any]", body: _Body) -> None: + await asyncio.wait_for(body.started.wait(), 5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + +@pytest.mark.parametrize("outcome", _OUTCOMES) +@pytest.mark.parametrize("scenario", _SCENARIOS, ids=lambda scenario: scenario.__name__.lstrip("_")) +async def test_suspending_context_exits_once_with_the_block_outcome( + scenario: "Callable[..., tuple[Callable[[], Awaitable[Any]], Any]]", outcome: str, monkeypatch: pytest.MonkeyPatch +) -> None: + body = _Body(outcome) + context = _SuspendingContext() + call, expected = scenario(body, context, monkeypatch) + task = asyncio.ensure_future(call()) + + if outcome == "succeeds": + assert await task is expected + assert context.exits == [(None, None)] + elif outcome == "raises": + with pytest.raises(RuntimeError) as caught: + await task + assert caught.value is body.error + assert context.exits == [(RuntimeError, body.error)] + else: + await _cancel_once_started(task, body) + assert [exc_type for exc_type, _ in context.exits] == [asyncio.CancelledError] + assert context.entered == 1 + + +async def test_mysqlconnector_begin_maps_a_driver_error_after_closing_the_cursor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + body = _Body("raises") + driver_error = MysqlConnectorError("server has gone away") + body.error = cast("Any", driver_error) + context = _SuspendingContext() + driver = _mysqlconnector_driver(body, context, monkeypatch) + + with pytest.raises(SQLSpecError) as caught: + await driver.begin() + + assert caught.value.__cause__ is driver_error + assert context.exits == [(type(driver_error), driver_error)] + + +@pytest.mark.parametrize("outcome", _OUTCOMES) +async def test_asyncpg_stack_exits_its_transaction_once_with_the_stack_outcome(outcome: str) -> None: + body = _Body(outcome, result="UPDATE 1") + transaction = _SuspendingContext() + connection = _AsyncpgConnection(body, transaction, in_transaction=False) + driver = AsyncpgDriver(cast("Any", connection)) + stack = StatementStack().push_execute("UPDATE t SET x = 1") + task = asyncio.ensure_future(driver.execute_stack(stack)) + + if outcome == "succeeds": + results = await task + assert [result.rows_affected for result in results] == [1] + assert transaction.exits == [(None, None)] + elif outcome == "raises": + with pytest.raises(StackExecutionError) as caught: + await task + assert caught.value.__cause__ is body.error + assert transaction.exits == [(StackExecutionError, caught.value)] + else: + await _cancel_once_started(task, body) + assert [exc_type for exc_type, _ in transaction.exits] == [asyncio.CancelledError] + assert transaction.entered == 1 + + +@pytest.mark.parametrize("outcome", _OUTCOMES) +async def test_asyncpg_stack_savepoint_is_released_once_with_the_operation_outcome(outcome: str) -> None: + body = _Body(outcome, result="UPDATE 1") + connection = _AsyncpgConnection(body, None, in_transaction=True) + driver = AsyncpgDriver(cast("Any", connection)) + stack = StatementStack().push_execute("UPDATE t SET x = 1") + task = asyncio.ensure_future(driver.execute_stack(stack, continue_on_error=True)) + + if outcome == "succeeds": + results = await task + assert [result.rows_affected for result in results] == [1] + assert connection.commands == ["SAVEPOINT", "UPDATE", "RELEASE"] + return + if outcome == "raises": + results = await task + error = results[0].error + assert isinstance(error, StackExecutionError) + assert error.original_error is body.error + else: + await _cancel_once_started(task, body) + assert connection.commands == ["SAVEPOINT", "UPDATE", "ROLLBACK", "RELEASE"] diff --git a/tests/unit/base/test_sqlspec_class.py b/tests/unit/base/test_sqlspec_class.py index 555c0357c..dbe014fe1 100644 --- a/tests/unit/base/test_sqlspec_class.py +++ b/tests/unit/base/test_sqlspec_class.py @@ -604,3 +604,44 @@ def config_update_worker() -> None: concurrent.futures.wait(futures) assert len(errors) == 0, f"Concurrent operation errors: {errors}" + + +def test_provide_session_and_connection_bypass_runtime_wrapper_when_lifecycle_idle() -> None: + """Idle session lifecycle skips _RuntimeSyncContext while preserving observers and explicit lifecycle hooks.""" + from sqlspec.adapters.sqlite import SqliteConfig + from sqlspec.adapters.sqlite.config import SqliteConnectionContext, SqliteSessionContext + from sqlspec.observability import ObservabilityConfig + + spec = SQLSpec() + cfg = spec.add_config(SqliteConfig(connection_config={"database": ":memory:"})) + try: + assert isinstance(spec.provide_connection(cfg), SqliteConnectionContext) + assert isinstance(spec.provide_session(cfg), SqliteSessionContext) + finally: + cfg.close_pool() + + observed: list[object] = [] + obs_spec = SQLSpec(observability_config=ObservabilityConfig(statement_observers=(observed.append,))) + obs_cfg = obs_spec.add_config(SqliteConfig(connection_config={"database": ":memory:"})) + try: + session_ctx = obs_spec.provide_session(obs_cfg) + assert isinstance(session_ctx, SqliteSessionContext) + with session_ctx as session: + session.execute("SELECT 1") + assert len(observed) == 1 + finally: + obs_cfg.close_pool() + + lifecycle_starts: list[object] = [] + hook_spec = SQLSpec( + observability_config=ObservabilityConfig(lifecycle={"on_session_start": [lifecycle_starts.append]}) + ) + hook_cfg = hook_spec.add_config(SqliteConfig(connection_config={"database": ":memory:"})) + try: + hook_ctx = hook_spec.provide_session(hook_cfg) + assert not isinstance(hook_ctx, SqliteSessionContext) + with hook_ctx as session: + session.execute("SELECT 1") + assert len(lifecycle_starts) == 1 + finally: + hook_cfg.close_pool() diff --git a/tests/unit/config/test_discovery.py b/tests/unit/config/test_discovery.py index 9c6b6d288..37407fe5e 100644 --- a/tests/unit/config/test_discovery.py +++ b/tests/unit/config/test_discovery.py @@ -1,6 +1,7 @@ """Tests for config discovery functionality.""" import os +import stat from pathlib import Path from types import SimpleNamespace from typing import Any @@ -9,6 +10,25 @@ from sqlspec.utils.config_tools import discover_config_from_pyproject, find_pyproject_toml, parse_pyproject_config +requires_posix_ownership = pytest.mark.skipif(not hasattr(os, "getuid"), reason="POSIX ownership model required") + + +def _override_stat(monkeypatch: "pytest.MonkeyPatch", overrides: "dict[tuple[Path, bool], dict[str, int]]") -> None: + """Replace selected ``os.stat`` fields, keyed by ``(path, follow_symlinks)``.""" + real_stat = os.stat + + def _fake_stat(path: Any, *args: Any, **kwargs: Any) -> Any: + result = real_stat(path, *args, **kwargs) + if not isinstance(path, (str, os.PathLike)): + return result + fields = overrides.get((Path(path), kwargs.get("follow_symlinks", True))) + if fields is None: + return result + base = {"st_uid": result.st_uid, "st_mode": result.st_mode, "st_nlink": result.st_nlink} + return SimpleNamespace(**{**base, **fields}) + + monkeypatch.setattr(os, "stat", _fake_stat) + def test_discover_from_pyproject_single(tmp_path: "Path", monkeypatch: "pytest.MonkeyPatch") -> None: """Test pyproject.toml with single config string.""" @@ -341,3 +361,180 @@ def _fake_stat(path: "str | os.PathLike[str]", *args: Any, **kwargs: Any) -> Any monkeypatch.setattr(os, "stat", _fake_stat) assert find_pyproject_toml() == root_pyproject + + +def test_find_pyproject_treats_symlinked_git_as_repository_boundary( + tmp_path: "Path", monkeypatch: "pytest.MonkeyPatch" +) -> None: + """A symlinked .git entry ends discovery before the parent configuration is reached.""" + (tmp_path / "pyproject.toml").write_text('[tool.sqlspec]\nconfig = "outer:load"\n', encoding="utf-8") + work_dir = tmp_path / "checkout" + work_dir.mkdir() + (work_dir / ".git").symlink_to(tmp_path) + monkeypatch.chdir(work_dir) + + assert find_pyproject_toml() is None + + +def test_find_pyproject_accepts_symlinked_config_owned_by_user( + tmp_path: "Path", monkeypatch: "pytest.MonkeyPatch" +) -> None: + """A symlinked pyproject.toml is returned when the link and its target belong to the current user.""" + shared_config = tmp_path / "shared" / "pyproject.toml" + shared_config.parent.mkdir() + shared_config.write_text('[tool.sqlspec]\nconfig = "myapp.config:get_configs"\n', encoding="utf-8") + work_dir = tmp_path / "project" + work_dir.mkdir() + (work_dir / "pyproject.toml").symlink_to(shared_config) + monkeypatch.chdir(work_dir) + + assert find_pyproject_toml() == work_dir / "pyproject.toml" + assert discover_config_from_pyproject() == "myapp.config:get_configs" + + +@requires_posix_ownership +@pytest.mark.parametrize("follow_symlinks", [False, True], ids=["foreign-link", "foreign-target"]) +def test_find_pyproject_refuses_symlinked_config_owned_by_other_uid( + tmp_path: "Path", monkeypatch: "pytest.MonkeyPatch", caplog: "pytest.LogCaptureFixture", follow_symlinks: bool +) -> None: + """A symlinked pyproject.toml is refused when another user owns the link or its target.""" + target = tmp_path / "real_pyproject.toml" + target.write_text('[tool.sqlspec]\nconfig = "evil:load"\n', encoding="utf-8") + work_dir = tmp_path / "project" + work_dir.mkdir() + link = work_dir / "pyproject.toml" + link.symlink_to(target) + monkeypatch.chdir(work_dir) + foreign_uid = os.getuid() + 1000 + _override_stat(monkeypatch, {(link, follow_symlinks): {"st_uid": foreign_uid}}) + + assert find_pyproject_toml() is None + assert f"Refusing {link}: it is owned by another user (uid {foreign_uid})" in caplog.text + + +@pytest.mark.parametrize("config_mode", [0o644, 0o666], ids=["private-file", "world-writable-file"]) +def test_find_pyproject_discovers_through_user_owned_world_writable_ancestors( + tmp_path: "Path", monkeypatch: "pytest.MonkeyPatch", config_mode: int +) -> None: + """Discovery from a subdirectory reaches pyproject.toml when the user's own tree is mode 0777.""" + project = tmp_path / "project" + work_dir = project / "src" / "app" + work_dir.mkdir(parents=True) + pyproject = project / "pyproject.toml" + pyproject.write_text('[tool.sqlspec]\nconfig = "myapp.config:get_configs"\n', encoding="utf-8") + world_writable = [project, project / "src", work_dir] + for directory in world_writable: + os.chmod(directory, 0o777) + os.chmod(pyproject, config_mode) + monkeypatch.chdir(work_dir) + + try: + assert find_pyproject_toml() == pyproject + finally: + for directory in world_writable: + os.chmod(directory, 0o755) + + +def test_find_pyproject_discovers_through_ancestors_without_posix_ownership( + tmp_path: "Path", monkeypatch: "pytest.MonkeyPatch" +) -> None: + """Without os.getuid, mode bits are ignored and discovery walks through 0777 ancestors.""" + project = tmp_path / "project" + work_dir = project / "src" + work_dir.mkdir(parents=True) + pyproject = project / "pyproject.toml" + pyproject.write_text('[tool.sqlspec]\nconfig = "myapp.config:get_configs"\n', encoding="utf-8") + monkeypatch.chdir(work_dir) + monkeypatch.delattr(os, "getuid", raising=False) + world_writable_dir = {"st_uid": 0, "st_mode": stat.S_IFDIR | 0o777} + _override_stat( + monkeypatch, + { + (work_dir, False): world_writable_dir, + (project, False): world_writable_dir, + (pyproject, False): {"st_uid": 0, "st_mode": stat.S_IFREG | 0o666}, + }, + ) + + assert find_pyproject_toml() == pyproject + + +@requires_posix_ownership +def test_find_pyproject_refuses_foreign_config_in_world_writable_ancestor( + tmp_path: "Path", monkeypatch: "pytest.MonkeyPatch", caplog: "pytest.LogCaptureFixture" +) -> None: + """A pyproject.toml planted by another user in a world-writable ancestor is refused and logged.""" + shared = tmp_path / "shared" + work_dir = shared / "app" + work_dir.mkdir(parents=True) + planted = shared / "pyproject.toml" + planted.write_text('[tool.sqlspec]\nconfig = "evil:load"\n', encoding="utf-8") + monkeypatch.chdir(work_dir) + foreign_uid = os.getuid() + 1000 + _override_stat( + monkeypatch, {(shared, False): {"st_mode": stat.S_IFDIR | 0o777}, (planted, False): {"st_uid": foreign_uid}} + ) + + assert find_pyproject_toml() is None + assert f"Refusing {planted}: it is owned by another user (uid {foreign_uid})" in caplog.text + + +@requires_posix_ownership +@pytest.mark.parametrize( + ("config_fields", "reason"), + [ + ({}, None), + ({"st_mode": stat.S_IFREG | 0o666}, "world-writable inside a shared world-writable directory"), + ({"st_nlink": 2}, "several hard links inside a shared world-writable directory"), + ], + ids=["private-file", "world-writable-file", "hard-linked-file"], +) +def test_find_pyproject_in_sticky_world_writable_ancestor_requires_private_config( + tmp_path: "Path", + monkeypatch: "pytest.MonkeyPatch", + caplog: "pytest.LogCaptureFixture", + config_fields: "dict[str, int]", + reason: "str | None", +) -> None: + """In a root-owned sticky directory, only a singly linked, non-world-writable user config is used.""" + sticky = tmp_path / "sticky" + work_dir = sticky / "app" + work_dir.mkdir(parents=True) + pyproject = sticky / "pyproject.toml" + pyproject.write_text('[tool.sqlspec]\nconfig = "myapp.config:get_configs"\n', encoding="utf-8") + monkeypatch.chdir(work_dir) + _override_stat( + monkeypatch, + {(sticky, False): {"st_uid": 0, "st_mode": stat.S_IFDIR | 0o1777}, (pyproject, False): config_fields}, + ) + + if reason is None: + assert find_pyproject_toml() == pyproject + assert caplog.text == "" + else: + assert find_pyproject_toml() is None + assert str(pyproject) in caplog.text + assert reason in caplog.text + + +@pytest.mark.skipif(not hasattr(os, "getuid"), reason="POSIX ownership model required") +def test_discovery_refuses_config_swapped_for_a_foreign_file_before_reading( + tmp_path: "Path", monkeypatch: "pytest.MonkeyPatch", caplog: "pytest.LogCaptureFixture" +) -> None: + """The ownership test applies to the file that is opened, not only to the path that was found.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text('[tool.sqlspec]\nconfig = "myapp.config:get_configs"\n') + monkeypatch.chdir(tmp_path) + foreign_uid = os.getuid() + 1000 + real_fstat = os.fstat + + def _foreign_fstat(descriptor: int) -> Any: + result = real_fstat(descriptor) + return SimpleNamespace(st_uid=foreign_uid, st_mode=result.st_mode, st_nlink=result.st_nlink) + + assert discover_config_from_pyproject() == "myapp.config:get_configs" + + monkeypatch.setattr(os, "fstat", _foreign_fstat) + + assert discover_config_from_pyproject() is None + assert f"Refusing {pyproject}: it is owned by another user (uid {foreign_uid})" in caplog.text diff --git a/tests/unit/core/test_cache.py b/tests/unit/core/test_cache.py index 12fa9cf0a..34ce80c44 100644 --- a/tests/unit/core/test_cache.py +++ b/tests/unit/core/test_cache.py @@ -946,6 +946,23 @@ def test_lru_cache_size_method_removed() -> None: assert "size" not in LRUCache.__dict__ +def test_statement_cache_holder_keeps_shared_parameter_validator_cache() -> None: + """Building a holder leaves the statement config's shared parameter validator cache enabled.""" + statement_config = StatementConfig() + validator = statement_config.parameter_validator + validator.extract_parameters("SELECT :value") + configured_size = validator.cache_stats()["max_size"] + + cache_module.StatementCacheHolder(statement_config, max_size=2) + validator.extract_parameters("SELECT :value") + + stats = validator.cache_stats() + assert configured_size > 0 + assert stats["max_size"] == configured_size + assert stats["size"] == 1 + assert stats["hits"] == 1 + + def test_l1_statement_cache_holder_lifecycle_and_statistics_coordination() -> None: """L1 StatementCacheHolder and QueryCache coordinate with global cache lifecycle functions.""" holder = cache_module.StatementCacheHolder(StatementConfig(), max_size=2) @@ -1009,3 +1026,96 @@ def test_l1_statement_cache_holder_lifecycle_and_statistics_coordination() -> No clear_all_caches() assert len(holder.query_cache) == 0 + + +def test_cached_query_positional_passthrough_numeric_and_positional_colon() -> None: + """Numeric ($1, $2) and positional colon (:1) styles set positional_passthrough when contiguous.""" + from sqlspec.core.parameters import ParameterInfo, ParameterStyle + + op_profile = OperationProfile(returns_rows=True, modifies_rows=False) + numeric_profile = ParameterProfile([ + ParameterInfo(name="1", style=ParameterStyle.NUMERIC, position=0, ordinal=1, placeholder_text="$1"), + ParameterInfo(name="2", style=ParameterStyle.NUMERIC, position=8, ordinal=2, placeholder_text="$2"), + ]) + numeric_cached = cache_module.CachedQuery( + compiled_sql="SELECT $1, $2", + parameter_profile=numeric_profile, + input_named_parameters=("1", "2"), + applied_wrap_types=False, + parameter_casts={}, + operation_type="SELECT", + operation_profile=op_profile, + param_count=2, + processed_state=ProcessedState(compiled_sql="SELECT $1, $2", execution_parameters=[], operation_type="SELECT"), + ) + assert numeric_cached.positional_passthrough is True + + colon_profile = ParameterProfile([ + ParameterInfo(name="1", style=ParameterStyle.POSITIONAL_COLON, position=7, ordinal=1, placeholder_text=":1") + ]) + colon_cached = cache_module.CachedQuery( + compiled_sql="SELECT :1", + parameter_profile=colon_profile, + input_named_parameters=("1",), + applied_wrap_types=False, + parameter_casts={}, + operation_type="SELECT", + operation_profile=op_profile, + param_count=1, + processed_state=ProcessedState(compiled_sql="SELECT :1", execution_parameters=[], operation_type="SELECT"), + ) + assert colon_cached.positional_passthrough is True + + gapped_cached = cache_module.CachedQuery( + compiled_sql="SELECT $1, $3", + parameter_profile=numeric_profile, + input_named_parameters=("1", "3"), + applied_wrap_types=False, + parameter_casts={}, + operation_type="SELECT", + operation_profile=op_profile, + param_count=2, + processed_state=ProcessedState(compiled_sql="SELECT $1, $3", execution_parameters=[], operation_type="SELECT"), + ) + assert gapped_cached.positional_passthrough is False + + named_profile = ParameterProfile([ + ParameterInfo(name="id", style=ParameterStyle.NAMED_COLON, position=7, ordinal=1, placeholder_text=":id") + ]) + named_cached = cache_module.CachedQuery( + compiled_sql="SELECT :id", + parameter_profile=named_profile, + input_named_parameters=("id",), + applied_wrap_types=False, + parameter_casts={}, + operation_type="SELECT", + operation_profile=op_profile, + param_count=1, + processed_state=ProcessedState(compiled_sql="SELECT :id", execution_parameters=[], operation_type="SELECT"), + ) + assert named_cached.positional_passthrough is False + + +def test_statement_pipeline_epoch_invalidates_frozen_config_cache_on_reset() -> None: + """StatementPipelineRegistry.reset() advances the epoch so frozen StatementConfig drops stale pipeline.""" + from sqlspec.core._pipeline import StatementPipelineRegistry + + registry = StatementPipelineRegistry() + cfg = StatementConfig() + registry.compile(cfg, "SELECT 1", None) + first_pipeline = cfg._cached_pipeline + first_epoch = cfg._cached_pipeline_epoch + assert first_pipeline is not None + assert first_epoch > 0 + + registry.compile(cfg, "SELECT 2", None) + assert cfg._cached_pipeline is first_pipeline + + registry.reset() + assert cfg._cached_pipeline is None + assert cfg._cached_pipeline_epoch == 0 + + registry.compile(cfg, "SELECT 1", None) + assert cfg._cached_pipeline is not None + assert cfg._cached_pipeline is not first_pipeline + assert cfg._cached_pipeline_epoch > first_epoch diff --git a/tests/unit/core/test_query_modifiers.py b/tests/unit/core/test_query_modifiers.py index 2dbcab0a3..be4d59fb5 100644 --- a/tests/unit/core/test_query_modifiers.py +++ b/tests/unit/core/test_query_modifiers.py @@ -744,3 +744,13 @@ def test_parse_column_for_condition_rejects_malformed_names(column: str) -> None def test_parse_column_for_condition_accepts_qualified_names(column: str, expected: str) -> None: """One to four dotted parts build a qualified column.""" assert parse_column_for_condition(column).sql() == expected + + +@pytest.mark.parametrize( + "column", + ["a; DROP TABLE users", "bad column", "users.col--comment", "a' OR '1'='1", "col()", 'col"quote', "col/*comment*/"], +) +def test_parse_column_for_condition_rejects_injection_tokens(column: str) -> None: + """Column names containing SQL operators, whitespace, quotes, or comments raise SQLSpecError.""" + with pytest.raises(SQLSpecError): + parse_column_for_condition(column) diff --git a/tests/unit/core/test_result.py b/tests/unit/core/test_result.py index c60a115ce..cfb147d4a 100644 --- a/tests/unit/core/test_result.py +++ b/tests/unit/core/test_result.py @@ -1068,3 +1068,46 @@ class Item: items = res.get_data(schema_type=Item) assert len(items) == 2 assert res._materialized_dicts is dicts_before + + +def test_cached_select_result_and_dml_result_lazy_lists_and_fast_scalar() -> None: + """CachedSelectResult and DMLResult defer empty lists and scalar() avoids dict materialization.""" + from sqlspec.core import ProcessedState, StatementConfig + from sqlspec.core.result import DMLResult + from sqlspec.core.result._base import CachedSelectResult + + state = ProcessedState(compiled_sql="SELECT ? AS val", execution_parameters=[42], operation_type="SELECT") + res = SQLResult.create_cached_select( + sql="SELECT ? AS val", + config=StatementConfig(), + cached_state=state, + compiled_sql="SELECT ? AS val", + params=(42,), + data=[(42,)], + column_names=["val"], + rows_affected=1, + row_format="tuple", + ) + assert isinstance(res, CachedSelectResult) + assert isinstance(res, SQLResult) + assert res.is_success() is True + assert res.scalar() == 42 + assert res.scalar_or_none() == 42 + assert res._materialized_dicts is None + assert res._inserted_ids is None + assert res._statement_results is None + assert res._errors is None + + one_row = res.one() + assert one_row == {"val": 42} + assert res._materialized_dicts is None + assert res.inserted_ids == [] + + dml = DMLResult("INSERT", rows_affected=1) + assert dml._column_names is None + assert dml._inserted_ids is None + assert dml._statement_results is None + assert dml._errors is None + assert dml.is_success() is True + assert dml._errors is None + assert dml.errors == [] diff --git a/tests/unit/core/test_splitter.py b/tests/unit/core/test_splitter.py index 0670a6512..06c8e2419 100644 --- a/tests/unit/core/test_splitter.py +++ b/tests/unit/core/test_splitter.py @@ -560,12 +560,7 @@ def test_split_bracket_identifier_with_semicolon() -> None: @pytest.mark.parametrize("unterminated", ["/* ", "$$ "]) def test_snowflake_splitter_unterminated_openers_scale_linearly(unterminated: str) -> None: - def elapsed(count: int) -> float: - started = time.perf_counter() - split_sql_script("SELECT " + unterminated * count, "snowflake") - return time.perf_counter() - started - - small = elapsed(4000) - large = elapsed(16000) + small = _fastest_split_seconds("SELECT " + unterminated * 4000, "snowflake") + large = _fastest_split_seconds("SELECT " + unterminated * 16000, "snowflake") assert large < max(small * 12, 0.5) diff --git a/tests/unit/core/test_statement.py b/tests/unit/core/test_statement.py index 143bbcc0d..f3c161ef8 100644 --- a/tests/unit/core/test_statement.py +++ b/tests/unit/core/test_statement.py @@ -18,6 +18,8 @@ import importlib.util import logging import pickle +from dataclasses import dataclass +from functools import partial from typing import Any, cast from unittest.mock import MagicMock, patch @@ -2089,3 +2091,25 @@ def ast_transformer(expr: exp.Expr, params: Any, _profile: Any, _is_many: bool) )._has_uncacheable_transformers is True ) + + +class _DefaultHashTransformer: + def transform(self, sql: str, params: Any) -> tuple[str, Any]: + return sql, params + + +@dataclass(frozen=True) +class _ValueHashTransformer: + tag: str + + def transform(self, sql: str, params: Any) -> tuple[str, Any]: + return f"{sql} {self.tag}", params + + +def test_statement_config_blocks_bound_method_on_default_hash_and_partial() -> None: + """Bound methods of default-hashed instances and functools.partial are treated as uncacheable.""" + assert StatementConfig(output_transformer=_DefaultHashTransformer().transform)._has_uncacheable_transformers is True + assert StatementConfig(output_transformer=partial(_static_output_transformer))._has_uncacheable_transformers is True + assert ( + StatementConfig(output_transformer=_ValueHashTransformer("v1").transform)._has_uncacheable_transformers is False + ) diff --git a/tests/unit/driver/test_deferred_session.py b/tests/unit/driver/test_deferred_session.py index 3c02e0abd..06cf3d715 100644 --- a/tests/unit/driver/test_deferred_session.py +++ b/tests/unit/driver/test_deferred_session.py @@ -6,6 +6,7 @@ import pytest from sqlspec.adapters.aiosqlite import AiosqliteConfig +from sqlspec.adapters.aiosqlite.driver import AiosqliteDriver from sqlspec.driver import DeferredAsyncSession from sqlspec.driver._common import _CLOSED_CONNECTION from sqlspec.exceptions import ImproperConfigurationError @@ -51,3 +52,38 @@ async def open_session() -> Any: if active_session is not None: await session_ctx.__aexit__(None, None, None) await config.close_pool() + + +class _InstanceDialectAiosqliteDriver(AiosqliteDriver): + """aiosqlite driver that stores its dialect on the instance instead of the class.""" + + __slots__ = ("dialect",) + + def __init__(self, connection: Any) -> None: + super().__init__(connection) + self.dialect = "sqlite" + + +@pytest.mark.parametrize("target_type", [None, _InstanceDialectAiosqliteDriver], ids=["class", "instance"]) +async def test_deferred_async_session_serves_dialect_before_connection_opens(target_type: Any) -> None: + """``dialect`` is available before checkout wherever the driver stores it.""" + config = AiosqliteConfig(connection_config={"database": ":memory:"}) + opened = False + + async def open_session() -> Any: + nonlocal opened + opened = True + return None + + deferred = DeferredAsyncSession( + config=config, + open_session=open_session, + current_session=lambda: None, + accessor="await plugin.get_session_async(request)", + target_type=target_type, + ) + try: + assert deferred.dialect == "sqlite" + assert opened is False + finally: + await config.close_pool() diff --git a/tests/unit/driver/test_query_cache.py b/tests/unit/driver/test_query_cache.py index bdfb0e68a..ab31aebe0 100644 --- a/tests/unit/driver/test_query_cache.py +++ b/tests/unit/driver/test_query_cache.py @@ -5,7 +5,7 @@ import sqlite3 import threading from collections import UserDict -from collections.abc import AsyncIterator, Generator +from collections.abc import AsyncGenerator, Generator from contextlib import asynccontextmanager, contextmanager from datetime import date from decimal import Decimal @@ -45,6 +45,7 @@ from sqlspec.core.filters import LimitOffsetFilter from sqlspec.core.parameters._declared import ParameterDeclaration from sqlspec.core.parameters._processor import ParameterProcessor, type_coercion_fallbacks +from sqlspec.core.result import SQLResult from sqlspec.driver._async import AsyncDriverAdapterBase from sqlspec.driver._common import _callable_cache_key, _check_declared_parameters, run_cached_execution from sqlspec.driver._sync import SyncDriverAdapterBase @@ -657,7 +658,7 @@ def execute(sql: str, params: Any = ()) -> None: cursor.fetchall = raw_cursor.fetchall @asynccontextmanager - async def with_cursor(_connection: Any) -> AsyncIterator[Any]: + async def with_cursor(_connection: Any) -> AsyncGenerator[Any, None]: yield cursor async def dispatch(_cursor: Any, statement: SQL) -> Any: @@ -2055,3 +2056,105 @@ async def decline(connection: Any, sql: str, params: Any, cached: CachedQuery, c assert (await aiosqlite_async_driver.execute("SELECT ? AS v", (2,))).get_data() == [{"v": 2}] assert declined == ["SELECT ? AS v"] + + +def test_run_cached_execution_hits_fast_path_for_numeric_positional_params( + sqlite_sync_driver: Any, monkeypatch: Any +) -> None: + """Contiguous $1 positional cached queries take execute_cache_hit while mapping params still rebind.""" + param_profile = ParameterProfile([ + ParameterInfo(name="1", style=ParameterStyle.NUMERIC, position=7, ordinal=1, placeholder_text="$1") + ]) + cached = CachedQuery( + compiled_sql="SELECT $1 AS val", + parameter_profile=param_profile, + input_named_parameters=("1",), + applied_wrap_types=False, + parameter_casts={}, + operation_type="SELECT", + operation_profile=OperationProfile(returns_rows=True, modifies_rows=False), + param_count=1, + processed_state=ProcessedState( + compiled_sql="SELECT $1 AS val", execution_parameters=[1], operation_type="SELECT" + ), + simple_wire_fast_path=True, + ) + sqlite_sync_driver._stmt_cache_enabled = True + sqlite_sync_driver._stmt_cache.set("SELECT $1 AS val", cached) + + fast_calls: list[Any] = [] + rebind_calls: list[Any] = [] + + def fake_executor(_conn: Any, sql: str, params: Any, _cached: Any, _cfg: Any, _state: Any) -> Any: + fast_calls.append((sql, params)) + return SQLResult.create_cached_select( + sql=sql, + config=sqlite_sync_driver.statement_config, + cached_state=cached.processed_state, + compiled_sql=cached.compiled_sql, + params=params, + data=[(params[0],)], + column_names=["val"], + rows_affected=1, + row_format="tuple", + ) + + def fake_rebind(params: Any, _cached: Any) -> Any: + rebind_calls.append(params) + return [params["1"]] + + monkeypatch.setattr(sqlite_sync_driver, "_cache_hit_executor", fake_executor) + monkeypatch.setattr(sqlite_sync_driver, "stmt_cache_rebind", fake_rebind) + + result_pos = run_cached_execution(sqlite_sync_driver, "SELECT $1 AS val", (7,)) + assert isinstance(result_pos, SQLResult) + assert result_pos.scalar() == 7 + assert fast_calls == [("SELECT $1 AS val", (7,))] + assert rebind_calls == [] + + monkeypatch.setattr(sqlite_sync_driver, "_cache_hit_executor", None) + monkeypatch.setattr( + sqlite_sync_driver, + "_execute_cached_statement", + lambda prepared: SQLResult(statement=prepared, data=[{"val": 9}], rows_affected=1), + ) + result_map = run_cached_execution(sqlite_sync_driver, "SELECT $1 AS val", {"1": 9}) + assert isinstance(result_map, SQLResult) + assert result_map.scalar() == 9 + assert rebind_calls == [{"1": 9}] + + +def test_compile_statement_rejects_mismatched_source_guard_on_cache_hit(sqlite_sync_driver: Any) -> None: + """L2 CachedStatement source guard rejects entries whose raw SQL disagrees with the requested statement.""" + clear_all_caches() + config = sqlite_sync_driver.statement_config + stmt = SQL("SELECT 1 AS ok", statement_config=config) + cache_key = sqlite_sync_driver._compile_cache_key(stmt, config, False) + poisoned = CachedStatement( + compiled_sql="SELECT 999 AS ok", + parameters=(), + expression=None, + _source_guard=("SELECT 999 AS ok", False, False, False, config), + ) + get_cache().put_statement(cache_key, poisoned, None) + + compiled_stmt, _ = sqlite_sync_driver._compiled_statement(stmt, config) + assert "999" not in compiled_stmt.compiled_sql + assert "1" in compiled_stmt.compiled_sql + clear_all_caches() + + +def test_open_session_cached_statements_see_query_hooks_registered_later() -> None: + """A query hook registered while a session is open fires for that session's cached statements.""" + spec = SQLSpec() + config = spec.add_config(SqliteConfig(connection_config={"database": ":memory:"})) + seen: list[str] = [] + + with spec.provide_session(config) as session: + for _ in range(2): + session.execute("SELECT ?", (1,)) + config.get_observability_runtime().register_lifecycle_hook("on_query_start", lambda _context: seen.append("q")) + for _ in range(3): + session.execute("SELECT ?", (1,)) + + assert seen == ["q", "q", "q"] diff --git a/tests/unit/driver/test_stream_session_close.py b/tests/unit/driver/test_stream_session_close.py index 90247204f..cf9bb82af 100644 --- a/tests/unit/driver/test_stream_session_close.py +++ b/tests/unit/driver/test_stream_session_close.py @@ -2,7 +2,7 @@ import inspect import sqlite3 -from collections.abc import AsyncIterator +from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from pathlib import Path from typing import Any @@ -60,7 +60,7 @@ async def drain(stream: Any) -> "list[Any]": @asynccontextmanager -async def open_session(mode: str, database: str) -> "AsyncIterator[Any]": +async def open_session(mode: str, database: str) -> "AsyncGenerator[Any, None]": if mode == "sync": config = SqliteConfig(connection_config={"database": database}) try: diff --git a/tests/unit/driver/test_sync_routing_session.py b/tests/unit/driver/test_sync_routing_session.py index 2f10e2dde..744706b65 100644 --- a/tests/unit/driver/test_sync_routing_session.py +++ b/tests/unit/driver/test_sync_routing_session.py @@ -3,7 +3,7 @@ import asyncio import sqlite3 import threading -from collections.abc import Iterator +from collections.abc import Generator, Iterator from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path @@ -780,7 +780,7 @@ def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) @contextmanager - def provide_session(self, *args: Any, **kwargs: Any) -> Iterator[Any]: + def provide_session(self, *args: Any, **kwargs: Any) -> Generator[Any, None, None]: if not self._checkout_gate.acquire(blocking=False): msg = "max_size=1 primary pool exhausted" raise TimeoutError(msg) diff --git a/tests/unit/extensions/_framework_apps.py b/tests/unit/extensions/_framework_apps.py index b14e46e84..ed56c4042 100644 --- a/tests/unit/extensions/_framework_apps.py +++ b/tests/unit/extensions/_framework_apps.py @@ -1,7 +1,7 @@ """Framework test apps that expose SQLSpec request accessors through one neutral interface.""" import json -from collections.abc import Awaitable, Callable, Iterator +from collections.abc import Awaitable, Callable, Generator, Iterator from contextlib import contextmanager from dataclasses import dataclass from typing import Any @@ -25,7 +25,9 @@ class Access: @contextmanager -def serve(framework: str, config: Any, bodies: "dict[str, Body]") -> Iterator[Callable[[str, int], tuple[int, Any]]]: +def serve( + framework: str, config: Any, bodies: "dict[str, Body]" +) -> Generator[Callable[[str, int], tuple[int, Any]], None, None]: """Serve ``//`` routes that run ``bodies[name]`` with the framework's accessors. Yields: diff --git a/tests/unit/extensions/test_events/test_batch_publish.py b/tests/unit/extensions/test_events/test_batch_publish.py index 963b0ed92..ef3e879ee 100644 --- a/tests/unit/extensions/test_events/test_batch_publish.py +++ b/tests/unit/extensions/test_events/test_batch_publish.py @@ -16,7 +16,7 @@ from sqlspec.utils.serializers import from_json, to_json if TYPE_CHECKING: - from collections.abc import AsyncIterator + from collections.abc import AsyncGenerator def _events(count: int = 1_000) -> "list[tuple[str, dict[str, Any], dict[str, Any] | None]]": @@ -280,7 +280,7 @@ def get_observability_runtime(self) -> ObservabilityRuntime: return self.runtime @asynccontextmanager - async def provide_session(self, **_: Any) -> "AsyncIterator[Any]": + async def provide_session(self, **_: Any) -> "AsyncGenerator[Any, None]": self.sessions += 1 yield self.driver @@ -328,7 +328,7 @@ def get_observability_runtime(self) -> ObservabilityRuntime: return self.runtime @asynccontextmanager - async def provide_session(self, **_: Any) -> "AsyncIterator[Any]": + async def provide_session(self, **_: Any) -> "AsyncGenerator[Any, None]": self.sessions += 1 yield self.driver diff --git a/tests/unit/extensions/test_events/test_queue.py b/tests/unit/extensions/test_events/test_queue.py index 703c4e65e..b8529c59e 100644 --- a/tests/unit/extensions/test_events/test_queue.py +++ b/tests/unit/extensions/test_events/test_queue.py @@ -1,7 +1,7 @@ # pyright: reportPrivateUsage=false """Unit tests for SyncTableEventQueue and AsyncTableEventQueue.""" -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncGenerator, Generator from contextlib import asynccontextmanager, contextmanager from datetime import UTC, datetime, timedelta from types import SimpleNamespace @@ -481,7 +481,7 @@ def get_observability_runtime(self) -> Any: return SimpleNamespace(increment_metric=lambda *_args, **_kwargs: None) @contextmanager - def provide_session(self, **_kwargs: Any) -> Iterator[Any]: + def provide_session(self, **_kwargs: Any) -> Generator[Any, None, None]: yield FakeSyncSessionDriver() sync_queue = SyncTableEventQueue(cast("Any", SyncConfigStub()), cleanup_on_ack=False) @@ -513,7 +513,7 @@ def get_observability_runtime(self) -> Any: return SimpleNamespace(increment_metric=lambda *_args, **_kwargs: None) @asynccontextmanager - async def provide_session(self, **_kwargs: Any) -> AsyncIterator[Any]: + async def provide_session(self, **_kwargs: Any) -> AsyncGenerator[Any, None]: yield FakeAsyncSessionDriver() async_queue = AsyncTableEventQueue(cast("Any", AsyncConfigStub()), cleanup_on_ack=False) @@ -535,7 +535,7 @@ def select_one_or_none(self, *_args: Any, **_kwargs: Any) -> None: return None @contextmanager - def _provide_sync_session(*_args: Any, **kwargs: Any) -> Iterator[Any]: + def _provide_sync_session(*_args: Any, **kwargs: Any) -> Generator[Any, None, None]: sync_session_calls.append(kwargs) yield FakeSyncDriver() @@ -559,7 +559,7 @@ async def select_one_or_none(self, *_args: Any, **_kwargs: Any) -> None: return None @asynccontextmanager - async def _provide_async_session(*_args: Any, **kwargs: Any) -> AsyncIterator[Any]: + async def _provide_async_session(*_args: Any, **kwargs: Any) -> AsyncGenerator[Any, None]: async_session_calls.append(kwargs) yield FakeAsyncDriver() @@ -599,7 +599,7 @@ def run_in_transaction(self, fn: Any) -> Any: return fn(FakeSyncTxDriver()) @contextmanager - def _provide_sync_tx_session(*_args: Any, **kwargs: Any) -> Iterator[Any]: + def _provide_sync_tx_session(*_args: Any, **kwargs: Any) -> Generator[Any, None, None]: sync_session_kwargs.append(kwargs) yield FakeSyncOuterDriver() @@ -628,7 +628,7 @@ async def run_in_transaction(self, fn: Any) -> Any: return await fn(FakeAsyncTxDriver()) @asynccontextmanager - async def _provide_async_tx_session(*_args: Any, **kwargs: Any) -> AsyncIterator[Any]: + async def _provide_async_tx_session(*_args: Any, **kwargs: Any) -> AsyncGenerator[Any, None]: async_session_kwargs.append(kwargs) yield FakeAsyncOuterDriver() @@ -855,3 +855,61 @@ def test_nack_ignores_non_leased_rows_and_dequeue_skips_attempt_bound(tmp_path: assert queue.dequeue("alerts") is None finally: config.close_pool() + + +def test_corrupt_payload_nacks_and_raises_event_channel_error_sync(tmp_path: Any) -> None: + """Sync dequeue nacks malformed claimed rows so status leaves leased and transitions via retry policy.""" + config = SqliteConfig(connection_config={"database": str(tmp_path / "corrupt_sync.db")}) + with config.provide_session() as driver: + for ddl in SqliteEventQueueStore(config).create_statements(): + driver.execute_script(ddl) + queue = SyncTableEventQueue(config, max_attempts=1, retry_backoff_seconds=5.0) + try: + event_id = queue.publish("alerts", {"ok": True}) + with config.provide_session(transaction=True) as driver: + driver.execute( + "UPDATE sqlspec_event_queue SET payload_json = '{invalid' WHERE event_id = :event_id", + {"event_id": event_id}, + ) + driver.commit() + with pytest.raises(EventChannelError, match="Failed to decode"): + queue.dequeue("alerts") + with config.provide_session() as driver: + row = driver.select_one( + "SELECT status, attempts, lease_expires_at FROM sqlspec_event_queue WHERE event_id = :event_id", + {"event_id": event_id}, + ) + assert row["status"] == "failed" + assert int(row["attempts"]) == 1 + assert row["lease_expires_at"] is None + finally: + config.close_pool() + + +async def test_corrupt_payload_nacks_and_raises_event_channel_error_async(tmp_path: Any) -> None: + """Async dequeue nacks malformed claimed rows with exponential backoff before reaching failed.""" + config = AiosqliteConfig(connection_config={"database": str(tmp_path / "corrupt_async.db")}) + async with config.provide_session() as driver: + for ddl in AiosqliteEventQueueStore(config).create_statements(): + await driver.execute_script(ddl) + queue = AsyncTableEventQueue(config, max_attempts=3, retry_backoff_seconds=5.0) + try: + event_id = await queue.publish("alerts", {"ok": True}) + async with config.provide_session(transaction=True) as driver: + await driver.execute( + "UPDATE sqlspec_event_queue SET payload_json = '{invalid' WHERE event_id = :event_id", + {"event_id": event_id}, + ) + await driver.commit() + with pytest.raises(EventChannelError, match="Failed to decode"): + await queue.dequeue("alerts") + async with config.provide_session() as driver: + row = await driver.select_one( + "SELECT status, attempts, lease_expires_at FROM sqlspec_event_queue WHERE event_id = :event_id", + {"event_id": event_id}, + ) + assert row["status"] == "pending" + assert int(row["attempts"]) == 1 + assert row["lease_expires_at"] is None + finally: + await config.close_pool() diff --git a/tests/unit/extensions/test_events/test_queue_claim_verification.py b/tests/unit/extensions/test_events/test_queue_claim_verification.py index 51a5bb5c0..80aa34447 100644 --- a/tests/unit/extensions/test_events/test_queue_claim_verification.py +++ b/tests/unit/extensions/test_events/test_queue_claim_verification.py @@ -6,7 +6,7 @@ persisted lease token instead of trusting the zero rowcount. """ -from collections.abc import AsyncIterator +from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from types import SimpleNamespace @@ -157,7 +157,7 @@ async def execute_claim(stmt: Any) -> Any: ) @asynccontextmanager - async def provide_session(*_args: Any, **_kwargs: Any) -> AsyncIterator[Any]: + async def provide_session(*_args: Any, **_kwargs: Any) -> AsyncGenerator[Any, None]: yield driver config_stub = SimpleNamespace( diff --git a/tests/unit/extensions/test_fastapi/test_extension.py b/tests/unit/extensions/test_fastapi/test_extension.py index 5869381f4..684bc038d 100644 --- a/tests/unit/extensions/test_fastapi/test_extension.py +++ b/tests/unit/extensions/test_fastapi/test_extension.py @@ -10,6 +10,8 @@ pytest.importorskip("fastapi") +from collections.abc import AsyncGenerator + from fastapi import Depends, FastAPI, HTTPException, Request from fastapi.responses import StreamingResponse from fastapi.testclient import TestClient @@ -257,7 +259,7 @@ async def close_pool(self) -> None: return None @asynccontextmanager - async def provide_connection(self, *args: Any, **kwargs: Any) -> AsyncIterator[Any]: + async def provide_connection(self, *args: Any, **kwargs: Any) -> AsyncGenerator[Any, None]: payload = TenantContext.get() if payload is None: raise TenantContextError("Tenant context required") diff --git a/tests/unit/extensions/test_flask/test_extension.py b/tests/unit/extensions/test_flask/test_extension.py index 7d08fdb3f..e6e6b1ff3 100644 --- a/tests/unit/extensions/test_flask/test_extension.py +++ b/tests/unit/extensions/test_flask/test_extension.py @@ -24,7 +24,7 @@ from tests.unit.extensions._tenant_configs import SyncRecordingStrategy, TenantSqliteConfig if TYPE_CHECKING: - from collections.abc import AsyncIterator, Iterator + from collections.abc import AsyncGenerator, Generator, Iterator from sqlspec.config import DatabaseConfigProtocol, LifecycleConfig @@ -211,7 +211,7 @@ def test_flask_defers_tenant_connection_and_supports_set_request_tenant( seen: list[tuple[str, TenantContextPayload | None]] = [] @contextmanager - def sync_conn(self: Any, pool: Any = None) -> "Iterator[object]": + def sync_conn(self: Any, pool: Any = None) -> "Generator[object, None, None]": seen.append(("enter", TenantContext.get())) try: yield self.create_connection() @@ -219,7 +219,7 @@ def sync_conn(self: Any, pool: Any = None) -> "Iterator[object]": seen.append(("exit", TenantContext.get())) @asynccontextmanager - async def async_conn(self: Any, pool: Any = None) -> "AsyncIterator[object]": + async def async_conn(self: Any, pool: Any = None) -> "AsyncGenerator[object, None]": seen.append(("enter", TenantContext.get())) conn = await self.create_connection() try: @@ -281,7 +281,7 @@ def test_flask_supports_multi_config_auth_and_deferred_tenant_db(monkeypatch: py original_provide = SqliteConfig.provide_connection @contextmanager - def tracked_provide(self: SqliteConfig, *args: Any, **kwargs: Any) -> "Iterator[Any]": + def tracked_provide(self: SqliteConfig, *args: Any, **kwargs: Any) -> "Generator[Any, None, None]": if self is tenant_cfg: seen_tenant.append(TenantContext.get_tenant_id()) conn = self.create_connection() @@ -567,7 +567,7 @@ def __getattr__(self, name: str) -> Any: original_provide = config.provide_connection @contextmanager - def _proxy_provide(*args: Any, **kwargs: Any) -> "Iterator[Any]": + def _proxy_provide(*args: Any, **kwargs: Any) -> "Generator[Any, None, None]": with original_provide(*args, **kwargs) as raw_conn: proxy = _ConnProxy(raw_conn) proxies.append(proxy) @@ -623,7 +623,7 @@ def test_flask_streamed_response_defers_finalization_and_release_to_close( original_provide = SqliteConfig.provide_connection @contextmanager - def _tracking_provide(self: SqliteConfig, *args: Any, **kwargs: Any) -> "Iterator[Any]": + def _tracking_provide(self: SqliteConfig, *args: Any, **kwargs: Any) -> "Generator[Any, None, None]": nonlocal active_connections with original_provide(self, *args, **kwargs) as conn: active_connections += 1 diff --git a/tests/unit/extensions/test_framework_state.py b/tests/unit/extensions/test_framework_state.py index 125810d02..2aadc43bb 100644 --- a/tests/unit/extensions/test_framework_state.py +++ b/tests/unit/extensions/test_framework_state.py @@ -538,3 +538,183 @@ async def _arollback(_self: Any = None) -> None: monkeypatch.setattr(driver_cls, "rollback", _rollback_impl) monkeypatch.setattr(driver_cls, "_connection_in_transaction", lambda _self: tx_active) return session, counts + + +@pytest.mark.parametrize("framework", ["litestar", "starlette", "fastapi", "flask", "sanic"]) +def test_bigquery_defaults_to_manual_and_rejects_autocommit(framework: str) -> None: + """Non-transactional configs default web plugin commit_mode to manual and reject autocommit.""" + import importlib + + from sqlspec import SQLSpec + from sqlspec.adapters.bigquery import BigQueryConfig + from sqlspec.exceptions import ImproperConfigurationError + + mod_name = "plugin" if framework == "litestar" else "extension" + ext = importlib.import_module(f"sqlspec.extensions.{framework}.{mod_name}") + plugin_cls = ext.SQLSpecPlugin + + spec = SQLSpec() + spec.add_config(BigQueryConfig(connection_config={"project": "test-project"})) + plugin = plugin_cls(spec) + states = getattr(plugin, "_plugin_configs", None) or plugin._config_states + assert len(states) == 1 + assert states[0].commit_mode == "manual" + + for invalid_mode in ("autocommit", "autocommit_include_redirect"): + bad_spec = SQLSpec() + bad_spec.add_config( + BigQueryConfig( + connection_config={"project": "test-project"}, + extension_config=cast("Any", {framework: {"commit_mode": invalid_mode}}), + ) + ) + with pytest.raises(ImproperConfigurationError, match="does not support transactions"): + plugin_cls(bad_spec) + + +_STATE_FRAMEWORKS = ["starlette", "fastapi", "flask", "sanic"] + + +def _build_framework_plugin(framework: str, settings: "dict[str, Any]") -> Any: + """Construct a framework plugin for one SQLite config carrying ``settings`` as its extension config.""" + import importlib + + from sqlspec import SQLSpec + from sqlspec.adapters.sqlite import SqliteConfig + + plugin_cls = importlib.import_module(f"sqlspec.extensions.{framework}.extension").SQLSpecPlugin + spec = SQLSpec() + spec.add_config( + SqliteConfig(connection_config={"database": ":memory:"}, extension_config=cast("Any", {framework: settings})) + ) + return plugin_cls(spec) + + +@pytest.mark.parametrize("framework", _STATE_FRAMEWORKS) +@pytest.mark.parametrize( + ("settings", "message"), + [ + ({"enable_routing_middleware": "yes"}, "enable_routing_middleware must be a boolean"), + ({"enable_routing_middleware": 1}, "enable_routing_middleware must be a boolean"), + ({"routing_header": 1}, "routing_header must be None or a non-empty printable ASCII header name"), + ({"primary_pin_cookie": 1}, "primary_pin_cookie must be None or a non-empty printable token"), + ({"primary_pin_secret": 1}, "primary_pin_secret must be a non-empty str or bytes"), + ], +) +def test_framework_plugins_name_invalid_routing_settings( + framework: str, settings: "dict[str, Any]", message: str +) -> None: + """Invalid routing settings are rejected with an error naming the setting.""" + from sqlspec.exceptions import ImproperConfigurationError + + with pytest.raises(ImproperConfigurationError, match=message): + _build_framework_plugin(framework, settings) + + +@pytest.mark.parametrize("framework", _STATE_FRAMEWORKS) +def test_framework_plugins_normalize_set_like_statuses_and_truthy_flags(framework: str) -> None: + """Set-like status overrides and truthy or falsy flags are stored as ``set`` and ``bool`` values.""" + plugin = _build_framework_plugin( + framework, + { + "extra_commit_statuses": frozenset({404}), + "extra_rollback_statuses": {200: "ok"}.keys(), + "disable_di": 1, + "enable_correlation_middleware": 0, + "auto_trace_headers": "yes", + "enable_sqlcommenter_middleware": None, + "primary_pin_secret": "top-secret", + }, + ) + state = plugin._config_states[0] + + assert (type(state.extra_commit_statuses), state.extra_commit_statuses) == (set, {404}) + assert (type(state.extra_rollback_statuses), state.extra_rollback_statuses) == (set, {200}) + assert state.should_commit(404) + assert state.should_rollback(200) + assert state.disable_di is True + assert state.enable_correlation_middleware is False + assert state.auto_trace_headers is True + assert state.enable_sqlcommenter_middleware is False + assert state.primary_pin_secret == b"top-secret" + + +@pytest.mark.parametrize("framework", _STATE_FRAMEWORKS) +@pytest.mark.parametrize("empty", [frozenset(), [], ()]) +def test_framework_plugins_treat_empty_status_overrides_as_unset(framework: str, empty: Any) -> None: + """Empty status override collections leave the commit mode defaults in effect.""" + state = _build_framework_plugin( + framework, {"extra_commit_statuses": empty, "extra_rollback_statuses": empty} + )._config_states[0] + + assert state.extra_commit_statuses is None + assert state.extra_rollback_statuses is None + assert state.should_commit(200) + assert state.should_rollback(404) + + +@pytest.mark.parametrize("framework", _STATE_FRAMEWORKS) +@pytest.mark.parametrize("statuses", [[404, 409], (404, 409)], ids=["list", "tuple"]) +def test_framework_plugins_accept_status_overrides_as_sequences(framework: str, statuses: Any) -> None: + """List and tuple status overrides are stored as a ``set``.""" + state = _build_framework_plugin(framework, {"extra_commit_statuses": statuses})._config_states[0] + + assert (type(state.extra_commit_statuses), state.extra_commit_statuses) == (set, {404, 409}) + assert state.should_commit(409) + + +@pytest.mark.parametrize("framework", _STATE_FRAMEWORKS) +def test_framework_plugins_reject_overlapping_set_like_statuses(framework: str) -> None: + """Overlapping commit and rollback overrides are rejected for any set-like input.""" + from sqlspec.exceptions import ImproperConfigurationError + + with pytest.raises(ImproperConfigurationError, match="must not share any status codes"): + _build_framework_plugin( + framework, {"extra_commit_statuses": frozenset({404}), "extra_rollback_statuses": frozenset({404})} + ) + + +def test_litestar_get_plugin_state_raises_on_ambiguous_config_type() -> None: + """SQLSpecPlugin._get_plugin_state rejects duplicate config type lookups with KeyError.""" + from litestar import Litestar + + from sqlspec import SQLSpec + from sqlspec.adapters.sqlite import SqliteConfig + from sqlspec.extensions.litestar import SQLSpecPlugin + + spec = SQLSpec() + spec.add_config( + SqliteConfig( + connection_config={"database": ":memory:"}, + bind_key="primary", + extension_config={"litestar": {"session_key": "s1", "connection_key": "c1", "pool_key": "p1"}}, + ) + ) + spec.add_config( + SqliteConfig( + connection_config={"database": ":memory:"}, + bind_key="secondary", + extension_config={"litestar": {"session_key": "s2", "connection_key": "c2", "pool_key": "p2"}}, + ) + ) + plugin = SQLSpecPlugin(spec) + Litestar(plugins=[plugin]) + + with pytest.raises(KeyError, match="Multiple database configurations"): + plugin._get_plugin_state(SqliteConfig) + + +def test_flask_execute_commit_and_rollback_skip_raw_connection_without_tx_methods() -> None: + """Flask autocommit commit and rollback guards ignore raw connections lacking tx methods.""" + from sqlspec import SQLSpec + from sqlspec.adapters.sqlite import SqliteConfig + from sqlspec.extensions.flask.extension import SQLSpecPlugin + + spec = SQLSpec() + spec.add_config(SqliteConfig(connection_config={"database": ":memory:"})) + plugin = SQLSpecPlugin(spec) + state = plugin._config_states[0] + + raw_without_tx = object() + plugin._execute_commit(None, state, connection=raw_without_tx) + plugin._execute_rollback(None, state, connection=raw_without_tx) diff --git a/tests/unit/extensions/test_litestar/test_store_request_session.py b/tests/unit/extensions/test_litestar/test_store_request_session.py index f699323ac..174bf41c3 100644 --- a/tests/unit/extensions/test_litestar/test_store_request_session.py +++ b/tests/unit/extensions/test_litestar/test_store_request_session.py @@ -4,7 +4,7 @@ import json import sqlite3 import threading -from collections.abc import AsyncIterator +from collections.abc import AsyncGenerator, AsyncIterator from contextlib import asynccontextmanager from pathlib import Path from typing import Any, cast @@ -452,11 +452,11 @@ async def test_asyncpg_store_write_after_failed_release_runs_in_rolled_back_tran class FakeAsyncpgConfig(AsyncpgConfig): @asynccontextmanager - async def provide_connection(self, *args: Any, **kwargs: Any) -> AsyncIterator[Any]: + async def provide_connection(self, *args: Any, **kwargs: Any) -> AsyncGenerator[Any, None]: yield connection @asynccontextmanager - async def provide_session(self, *args: Any, **kwargs: Any) -> AsyncIterator[Any]: # type: ignore[override] # pyright: ignore[reportIncompatibleMethodOverride] + async def provide_session(self, *args: Any, **kwargs: Any) -> AsyncGenerator[Any, None]: # type: ignore[override] # pyright: ignore[reportIncompatibleMethodOverride] yield AsyncpgDriver(cast("Any", connection)) config = FakeAsyncpgConfig(extension_config={"litestar": {"session_table": TABLE}}) diff --git a/tests/unit/extensions/test_litestar/test_tenant.py b/tests/unit/extensions/test_litestar/test_tenant.py index 03e4611d3..207759222 100644 --- a/tests/unit/extensions/test_litestar/test_tenant.py +++ b/tests/unit/extensions/test_litestar/test_tenant.py @@ -1,7 +1,7 @@ """Verified tenant resolution and guards for the Litestar integration.""" import asyncio -from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from collections.abc import Awaitable, Callable from contextlib import asynccontextmanager, contextmanager from inspect import isawaitable from types import SimpleNamespace @@ -12,6 +12,8 @@ pytest.importorskip("litestar") +from collections.abc import AsyncGenerator, Generator + from litestar import Controller, Litestar, WebSocket, get, websocket from litestar.connection import ASGIConnection from litestar.di import NamedDependency @@ -517,7 +519,7 @@ def last_seen() -> tuple[str | None, bool]: class _TenantSqliteConfig(SqliteConfig): @contextmanager - def provide_connection(self, *args: Any, **kwargs: Any) -> Iterator[Any]: + def provide_connection(self, *args: Any, **kwargs: Any) -> Generator[Any, None, None]: payload = TenantContext.get() if payload is None: raise TenantContextError("Tenant context required") @@ -621,7 +623,7 @@ async def provide(request_scope: Any = scope) -> Any: async def test_provide_request_session_async_translates_missing_tenant() -> None: @asynccontextmanager - async def never_called(_pool: Any) -> AsyncIterator[Any]: + async def never_called(_pool: Any) -> AsyncGenerator[Any, None]: pytest.fail("provide_connection should not run without an authorized tenant") yield None diff --git a/tests/unit/extensions/test_prometheus/test_metrics.py b/tests/unit/extensions/test_prometheus/test_metrics.py index 9c2e35a61..492e2afae 100644 --- a/tests/unit/extensions/test_prometheus/test_metrics.py +++ b/tests/unit/extensions/test_prometheus/test_metrics.py @@ -290,6 +290,56 @@ def test_observer_handles_negative_duration(monkeypatch) -> None: observer(event) +def test_observer_accumulates_per_label_set_and_exports_rows_only_when_reported(monkeypatch) -> None: + """Repeated events accumulate on one series per label set; row series appear once rows are reported.""" + _force_dependency(monkeypatch, "prometheus_client") + + from prometheus_client import CollectorRegistry + + from sqlspec.extensions import prometheus + + registry = CollectorRegistry() + observer = prometheus.PrometheusStatementObserver(registry=registry) + + def _observe(operation: str, rows_affected: "int | None") -> None: + observer( + create_event( + sql="SELECT 1", + parameters=(), + driver="AsyncpgDriver", + adapter="AsyncpgConfig", + bind_key=None, + operation=operation, + execution_mode="async", + is_many=False, + is_script=False, + rows_affected=rows_affected, + duration_s=0.25, + correlation_id=None, + db_system="postgresql", + ) + ) + + select_labels = {"db_system": "postgresql", "operation": "SELECT"} + insert_labels = {"db_system": "postgresql", "operation": "INSERT"} + + _observe("SELECT", None) + _observe("SELECT", None) + assert registry.get_sample_value("sqlspec_driver_query_total", select_labels) == 2.0 + assert registry.get_sample_value("sqlspec_driver_query_duration_seconds_sum", select_labels) == 0.5 + assert registry.get_sample_value("sqlspec_driver_query_rows_count", select_labels) is None + + _observe("SELECT", 3) + _observe("SELECT", 4) + _observe("INSERT", 1) + assert registry.get_sample_value("sqlspec_driver_query_total", select_labels) == 4.0 + assert registry.get_sample_value("sqlspec_driver_query_duration_seconds_count", select_labels) == 4.0 + assert registry.get_sample_value("sqlspec_driver_query_rows_count", select_labels) == 2.0 + assert registry.get_sample_value("sqlspec_driver_query_rows_sum", select_labels) == 7.0 + assert registry.get_sample_value("sqlspec_driver_query_total", insert_labels) == 1.0 + assert registry.get_sample_value("sqlspec_driver_query_rows_sum", insert_labels) == 1.0 + + def test_custom_namespace_and_subsystem(monkeypatch) -> None: """Verify custom namespace and subsystem are applied.""" _force_dependency(monkeypatch, "prometheus_client") diff --git a/tests/unit/extensions/test_starlette/test_extension.py b/tests/unit/extensions/test_starlette/test_extension.py index 92eb5a22a..1896464a0 100644 --- a/tests/unit/extensions/test_starlette/test_extension.py +++ b/tests/unit/extensions/test_starlette/test_extension.py @@ -39,7 +39,7 @@ from tests.unit.extensions._tenant_configs import AsyncRecordingStrategy, TenantAiosqliteConfig if TYPE_CHECKING: - from collections.abc import AsyncIterator + from collections.abc import AsyncGenerator, AsyncIterator from starlette.types import Message @@ -332,7 +332,7 @@ async def rollback() -> None: connection = SimpleNamespace(commit=commit, rollback=rollback) @asynccontextmanager - async def provide_connection(pool: object) -> "AsyncIterator[SimpleNamespace]": + async def provide_connection(pool: object) -> "AsyncGenerator[SimpleNamespace, None]": nonlocal active active = True events.append("acquire") @@ -448,7 +448,7 @@ async def test_tenant_incomplete_downstream_response_rolls_back() -> None: connection = _Connection() @asynccontextmanager - async def provide_connection(pool: object) -> "AsyncIterator[_Connection]": + async def provide_connection(pool: object) -> "AsyncGenerator[_Connection, None]": yield connection async def app(scope: Any, receive: Any, send: Any) -> None: @@ -505,7 +505,7 @@ async def rollback() -> None: raise RuntimeError("cleanup failed") @asynccontextmanager - async def provide_connection(pool: object) -> "AsyncIterator[SimpleNamespace]": + async def provide_connection(pool: object) -> "AsyncGenerator[SimpleNamespace, None]": try: yield SimpleNamespace(rollback=rollback) except ValueError as exc: @@ -574,7 +574,7 @@ async def close_pool(self) -> None: return None @asynccontextmanager - async def provide_connection(self, *args: Any, **kwargs: Any) -> "AsyncIterator[Any]": + async def provide_connection(self, *args: Any, **kwargs: Any) -> "AsyncGenerator[Any, None]": payload = TenantContext.get() if payload is None: raise TenantContextError("Tenant context required") diff --git a/tests/unit/extensions/test_tenant_request_contract.py b/tests/unit/extensions/test_tenant_request_contract.py index f5b243663..9fefd9bb1 100644 --- a/tests/unit/extensions/test_tenant_request_contract.py +++ b/tests/unit/extensions/test_tenant_request_contract.py @@ -6,7 +6,7 @@ """ import json -from collections.abc import AsyncIterator, Callable, Iterator +from collections.abc import AsyncIterator, Callable, Generator, Iterator from contextlib import contextmanager from dataclasses import dataclass, field from typing import Annotated, Any, cast @@ -82,7 +82,7 @@ def _ambient_extractor(scope: Any) -> str | None: @contextmanager -def _litestar(tenant: bool, asynchronous: bool) -> Iterator[Harness]: +def _litestar(tenant: bool, asynchronous: bool) -> Generator[Harness, None, None]: from litestar import Litestar, get from litestar.background_tasks import BackgroundTask from litestar.middleware import DefineMiddleware @@ -170,7 +170,7 @@ def request(path: str, headers: dict[str, str]) -> tuple[int, Any]: def _starlette_family(framework: str, tenant: bool, asynchronous: bool) -> Any: @contextmanager - def build() -> Iterator[Harness]: + def build() -> Generator[Harness, None, None]: from starlette.background import BackgroundTask from starlette.middleware import Middleware from starlette.requests import Request @@ -283,7 +283,7 @@ def request(path: str, headers: dict[str, str]) -> tuple[int, Any]: @contextmanager -def _flask(tenant: bool) -> Iterator[Harness]: +def _flask(tenant: bool) -> Generator[Harness, None, None]: from flask import Flask, Response, g, request, stream_with_context from sqlspec.extensions.flask import SQLSpecPlugin, set_request_tenant @@ -374,7 +374,7 @@ def run(path: str, headers: dict[str, str]) -> tuple[int, Any]: @contextmanager -def _sanic(tenant: bool, asynchronous: bool) -> Iterator[Harness]: +def _sanic(tenant: bool, asynchronous: bool) -> Generator[Harness, None, None]: from sanic import Sanic, response from sqlspec.extensions.sanic import SQLSpecPlugin, set_request_tenant diff --git a/tests/unit/migrations/test_lease_lock.py b/tests/unit/migrations/test_lease_lock.py index cd9fd6c97..d374c2f37 100644 --- a/tests/unit/migrations/test_lease_lock.py +++ b/tests/unit/migrations/test_lease_lock.py @@ -3,7 +3,7 @@ import logging import sqlite3 import time -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncGenerator, AsyncIterator, Generator from contextlib import asynccontextmanager, contextmanager from pathlib import Path from typing import Any, cast @@ -81,6 +81,39 @@ def test_lock_backend_kind_cockroach_and_bridge_sqlite() -> None: assert SyncMigrationTracker._resolve_lock_backend(bridge_sqlite) == "lease" # pyright: ignore[reportPrivateUsage] +class _InstanceDialectDuckDBDriver(DuckDBDriver): + """DuckDB driver that stores its dialect on the instance instead of the class.""" + + __slots__ = ("dialect",) + + def __init__(self, connection: Any) -> None: + super().__init__(connection) + self.dialect = "duckdb" + + +class _InstanceDialectAiosqliteDriver(AiosqliteDriver): + """aiosqlite driver that stores its dialect on the instance instead of the class.""" + + __slots__ = ("dialect",) + + def __init__(self, connection: Any) -> None: + super().__init__(connection) + self.dialect = "sqlite" + + +@pytest.mark.parametrize( + ("driver_type", "expected"), + [(_InstanceDialectDuckDBDriver, "lease"), (_InstanceDialectAiosqliteDriver, "sqlite")], + ids=["duckdb", "aiosqlite"], +) +def test_lock_backend_kind_reads_instance_dialect_of_driver_adapters(driver_type: Any, expected: str) -> None: + """Driver adapters whose dialect lives on the instance still resolve their lock backend.""" + driver = driver_type(MagicMock()) + + assert not isinstance(driver_type.__dict__["dialect"], str) + assert SyncMigrationTracker._resolve_lock_backend(driver) == expected # pyright: ignore[reportPrivateUsage] + + def test_lease_lock_blocks_second_migrator_duckdb(tmp_path: Path) -> None: db_path = tmp_path / "lease.duckdb" first_config, second_config = _duckdb_config(db_path), _duckdb_config(db_path) @@ -163,7 +196,7 @@ def execute(self, statement: Any, /, *parameters: Any, **kwargs: Any) -> Any: @contextmanager -def _no_rowcount_driver(db_path: Path) -> "Iterator[_NoRowcountDuckDBDriver]": +def _no_rowcount_driver(db_path: Path) -> "Generator[_NoRowcountDuckDBDriver, None, None]": config = _duckdb_config(db_path) try: with config.provide_connection() as connection: @@ -332,7 +365,7 @@ async def test_async_lease_lock_acquires_renews_and_releases(async_lease_driver: @asynccontextmanager -async def _held_lease(tmp_path: Path, is_async: bool, error: "BaseException | None") -> "AsyncIterator[None]": +async def _held_lease(tmp_path: Path, is_async: bool, error: "BaseException | None") -> "AsyncGenerator[None, None]": """Acquire the migration lease, delete it as another migrator would, then raise ``error`` if given.""" if is_async: async_config = AiosqliteConfig(connection_config={"database": str(tmp_path / "lost.db")}) diff --git a/tests/unit/migrations/test_migration_execution.py b/tests/unit/migrations/test_migration_execution.py index 4be0f4815..c73bc7205 100644 --- a/tests/unit/migrations/test_migration_execution.py +++ b/tests/unit/migrations/test_migration_execution.py @@ -11,6 +11,7 @@ import ast import os +import stat from collections.abc import Callable from pathlib import Path from types import SimpleNamespace @@ -1148,3 +1149,181 @@ def _mock_stat_root(self_path: Path, *, follow_symlinks: bool = True) -> Any: monkeypatch.setattr(Path, "stat", _mock_stat_root) assert find_project_root(child_dir) == resolved_root_ws + + +def _override_stat(monkeypatch: pytest.MonkeyPatch, overrides: "dict[tuple[Path, bool], dict[str, int]]") -> None: + """Replace selected ``os.stat`` fields, keyed by ``(path, follow_symlinks)``.""" + real_stat = os.stat + + def _fake_stat(path: Any, *args: Any, **kwargs: Any) -> Any: + result = real_stat(path, *args, **kwargs) + if not isinstance(path, (str, os.PathLike)): + return result + fields = overrides.get((Path(path), kwargs.get("follow_symlinks", True))) + if fields is None: + return result + base = {"st_uid": result.st_uid, "st_mode": result.st_mode, "st_nlink": result.st_nlink} + return SimpleNamespace(**{**base, **fields}) + + monkeypatch.setattr(os, "stat", _fake_stat) + + +def _build_project(tmp_path: Path, *, marker: "str | None" = "pyproject.toml") -> "tuple[Path, Path]": + """Create ``project/migrations`` under ``tmp_path`` and return the resolved project and migrations paths.""" + project = (tmp_path / "project").resolve() + migrations_dir = project / "migrations" + migrations_dir.mkdir(parents=True) + if marker is not None: + (project / marker).write_text("[project]\nname = 'app'\n", encoding="utf-8") + return project, migrations_dir + + +def test_find_project_root_accepts_symlinked_marker_owned_by_user(tmp_path: Path) -> None: + """A symlinked project marker counts when the link and its target belong to the current user.""" + project, migrations_dir = _build_project(tmp_path, marker=None) + outside = tmp_path / "outside.toml" + outside.write_text("[project]\nname = 'outside'\n", encoding="utf-8") + (project / "pyproject.toml").symlink_to(outside) + + assert find_project_root(migrations_dir) == project + + +@pytest.mark.parametrize("marker_mode", [0o644, 0o666], ids=["private-file", "world-writable-file"]) +def test_find_project_root_discovers_marker_in_user_owned_world_writable_tree(tmp_path: Path, marker_mode: int) -> None: + """The project root is found from the migrations directory when the user's own tree is mode 0777.""" + project, migrations_dir = _build_project(tmp_path) + os.chmod(project / "pyproject.toml", marker_mode) + os.chmod(migrations_dir, 0o777) + os.chmod(project, 0o777) + + try: + assert find_project_root(migrations_dir) == project + finally: + os.chmod(migrations_dir, 0o755) + os.chmod(project, 0o755) + + +def test_find_project_root_falls_back_to_user_owned_world_writable_parent(tmp_path: Path) -> None: + """Without any marker, a mode 0777 parent owned by the current user is the fallback root.""" + project, migrations_dir = _build_project(tmp_path, marker=None) + os.chmod(project, 0o777) + + try: + assert find_project_root(migrations_dir) == project + finally: + os.chmod(project, 0o755) + + +def test_find_project_root_discovers_marker_without_posix_ownership( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Without os.getuid, mode bits are ignored and the marker in a 0777 parent is found.""" + project, migrations_dir = _build_project(tmp_path) + monkeypatch.delattr(os, "getuid", raising=False) + world_writable_dir = {"st_uid": 0, "st_mode": stat.S_IFDIR | 0o777} + _override_stat( + monkeypatch, + { + (migrations_dir, False): world_writable_dir, + (project, False): world_writable_dir, + (project / "pyproject.toml", False): {"st_uid": 0, "st_mode": stat.S_IFREG | 0o666}, + }, + ) + + assert find_project_root(migrations_dir) == project + + +@pytest.mark.skipif(not hasattr(os, "getuid"), reason="POSIX ownership model required") +def test_find_project_root_refuses_foreign_marker_in_world_writable_parent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A marker planted by another user in a world-writable parent is refused and logged.""" + project, migrations_dir = _build_project(tmp_path) + marker = project / "pyproject.toml" + foreign_uid = os.getuid() + 1000 + _override_stat( + monkeypatch, {(project, False): {"st_mode": stat.S_IFDIR | 0o777}, (marker, False): {"st_uid": foreign_uid}} + ) + + assert find_project_root(migrations_dir) == migrations_dir + assert f"Refusing project marker {marker}: it is owned by another user (uid {foreign_uid})" in caplog.text + + +@pytest.mark.skipif(not hasattr(os, "getuid"), reason="POSIX ownership model required") +@pytest.mark.parametrize( + ("marker_fields", "reason"), + [ + ({}, None), + ({"st_mode": stat.S_IFREG | 0o666}, "world-writable inside a shared world-writable directory"), + ({"st_nlink": 2}, "several hard links inside a shared world-writable directory"), + ], + ids=["private-file", "world-writable-file", "hard-linked-file"], +) +def test_find_project_root_in_root_owned_world_writable_parent_requires_private_marker( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + marker_fields: "dict[str, int]", + reason: "str | None", +) -> None: + """In a root-owned world-writable directory, only a singly linked, non-world-writable marker is used.""" + project, migrations_dir = _build_project(tmp_path) + marker = project / "pyproject.toml" + _override_stat( + monkeypatch, {(project, False): {"st_uid": 0, "st_mode": stat.S_IFDIR | 0o777}, (marker, False): marker_fields} + ) + + if reason is None: + assert find_project_root(migrations_dir) == project + assert caplog.text == "" + else: + assert find_project_root(migrations_dir) == migrations_dir + assert f"Refusing project marker {marker}" in caplog.text + assert reason in caplog.text + + +@pytest.mark.skipif(not hasattr(os, "getuid"), reason="POSIX ownership model required") +@pytest.mark.parametrize("nested", [False, True], ids=["parent", "grandparent"]) +def test_find_project_root_never_uses_sticky_world_writable_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, nested: bool +) -> None: + """A sticky world-writable directory holding a trusted marker is not used as the project root.""" + shared, migrations_dir = _build_project(tmp_path) + expected = migrations_dir + if nested: + migrations_dir = migrations_dir / "versions" + migrations_dir.mkdir() + _override_stat(monkeypatch, {(shared, False): {"st_uid": 0, "st_mode": stat.S_IFDIR | 0o1777}}) + + assert find_project_root(migrations_dir) == expected + assert f"Ignoring the project marker in {shared}" in caplog.text + + +@pytest.mark.skipif(not hasattr(os, "getuid"), reason="POSIX ownership model required") +def test_find_project_root_never_falls_back_to_shared_world_writable_parent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Without any marker, a sticky world-writable parent is not used as the project root.""" + project, migrations_dir = _build_project(tmp_path, marker=None) + _override_stat(monkeypatch, {(project, False): {"st_uid": 0, "st_mode": stat.S_IFDIR | 0o1777}}) + + assert find_project_root(migrations_dir) == migrations_dir + assert f"parent directory {project} is a shared world-writable directory" in caplog.text + + +@pytest.mark.skipif(not hasattr(os, "getuid"), reason="POSIX ownership model required") +def test_find_project_root_falls_back_without_warning_in_a_tree_owned_by_another_user( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A source tree mounted from another user, as in a container, ends the search without a warning.""" + project, migrations_dir = _build_project(tmp_path) + foreign_uid = os.getuid() + 1000 + _override_stat( + monkeypatch, {(project, False): {"st_uid": foreign_uid}, (migrations_dir, False): {"st_uid": foreign_uid}} + ) + + with caplog.at_level("DEBUG", logger="sqlspec.migrations.loaders"): + assert find_project_root(migrations_dir) == migrations_dir + + assert {record.levelname for record in caplog.records} == {"DEBUG"} + assert f"directory is owned by another user (uid {foreign_uid})" in caplog.text diff --git a/tests/unit/migrations/test_tracking_table_guard.py b/tests/unit/migrations/test_tracking_table_guard.py index a132ab48a..a63f18d57 100644 --- a/tests/unit/migrations/test_tracking_table_guard.py +++ b/tests/unit/migrations/test_tracking_table_guard.py @@ -255,3 +255,30 @@ async def test_schema_pin_is_re_resolved_per_locked_command(is_async: bool) -> N assert tracker.version_table == "second.ddl_migrations" assert harness.guard_calls()[-1][1] == {"schema_name": "second", "table_name": "ddl_migrations"} + + +@pytest.mark.parametrize( + ("extra_row", "pattern"), + [({"has_rules": True}, "rewrite rules"), ({"has_row_security": True}, "row-level security")], + ids=["rules", "row-level-security"], +) +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +async def test_guard_rejects_rules_and_row_security(is_async: bool, extra_row: dict[str, Any], pattern: str) -> None: + """Tracking table guard rejects pre-existing PostgreSQL relations with rewrite rules or RLS.""" + row = {**OWNED_ROW, **extra_row} + harness = _Harness(is_async=is_async, relation_rows=[row]) + + with pytest.raises(MigrationError, match=pattern): + await _ensure(_tracker(is_async), harness, is_async) + + +async def test_pg_relation_guard_and_omissions_sql_contract() -> None: + """PostgreSQL relation guard avoids inverted superuser logic and omission check covers matviews and attacl.""" + from sqlspec.migrations.base import _PG_RELATION_GUARD_SQL, _PG_SCHEMA_OMISSIONS_SQL + + assert "WHERE r.oid = c.relowner AND r.rolsuper" in _PG_RELATION_GUARD_SQL + assert "pg_catalog.pg_has_role(current_user, c.relowner, 'MEMBER')" in _PG_RELATION_GUARD_SQL + assert "pg_catalog.pg_rewrite" in _PG_RELATION_GUARD_SQL + assert "pg_catalog.row_security_active(c.oid)" in _PG_RELATION_GUARD_SQL + assert "'materialized_view'" in _PG_SCHEMA_OMISSIONS_SQL + assert "attacl IS NOT NULL" in _PG_SCHEMA_OMISSIONS_SQL diff --git a/tests/unit/migrations/test_version.py b/tests/unit/migrations/test_version.py index 013a64e92..957674d18 100644 --- a/tests/unit/migrations/test_version.py +++ b/tests/unit/migrations/test_version.py @@ -464,3 +464,21 @@ async def test_async_quoted_and_schema_qualified_version_table_tracker() -> None partial_ddl = partially_quoted._tracking_table_ddl() # pyright: ignore[reportPrivateUsage] assert 'CREATE TABLE IF NOT EXISTS "app"."Tracker"' in str(partial_ddl) assert 'CREATE TABLE IF NOT EXISTS "app"."Tracker"' in partial_ddl.to_statement(pg_config).sql + + +def test_extension_version_parsing_linear_on_long_underscore_inputs() -> None: + """Extension parser rejects pathological underscore-heavy inputs in linear time without backtracking.""" + import time + + long_stem = "ext_" + ("a_" * 25_000) + "tail" + started = time.perf_counter() + assert parse_extension_stem(long_stem) is None + with pytest.raises(ValueError, match="Invalid migration version"): + parse_version(long_stem) + elapsed = time.perf_counter() - started + assert elapsed < 0.2 + + assert parse_extension_stem("ext_litestar_queues_0001_init") == ("litestar_queues", "0001") + parsed = parse_version("ext_litestar_queues_0001") + assert parsed.extension == "litestar_queues" + assert parsed.sequence == 1 diff --git a/tests/unit/observability/test_observability_core.py b/tests/unit/observability/test_observability_core.py index 08984525d..81c3ba6fe 100644 --- a/tests/unit/observability/test_observability_core.py +++ b/tests/unit/observability/test_observability_core.py @@ -18,6 +18,7 @@ from sqlspec import ObservabilityConfig, ObservabilityRuntime, RedactionConfig, SQLSpec, StatementObserver from sqlspec.adapters.adbc.driver import AdbcDriver +from sqlspec.adapters.aiosqlite import AiosqliteConfig from sqlspec.adapters.sqlite import SqliteConfig from sqlspec.config import LifecycleConfig from sqlspec.core import SQL, ArrowResult, StatementConfig @@ -959,6 +960,56 @@ def test_register_lifecycle_hook_updates_is_idle() -> None: assert runtime.is_idle is False +def _drive_registry_provider(spec: SQLSpec, config: Any, provider_name: str) -> None: + """Enter and exit one registry-provided session or connection for a sync or async config.""" + provider = getattr(spec, provider_name) + if not config.is_async: + try: + with provider(config): + pass + finally: + config.close_pool() + return + + async def _drive() -> None: + try: + async with provider(config): + pass + finally: + await config.close_pool() + + asyncio.run(_drive()) + + +@pytest.mark.parametrize("config_type", [SqliteConfig, AiosqliteConfig]) +@pytest.mark.parametrize( + ("event_name", "provider_name"), + [ + ("on_connection_create", "provide_session"), + ("on_session_start", "provide_session"), + ("on_session_end", "provide_session"), + ("on_connection_destroy", "provide_session"), + ("on_connection_create", "provide_connection"), + ("on_connection_destroy", "provide_connection"), + ], +) +def test_runtime_registered_lifecycle_hook_fires_through_registry( + config_type: Any, event_name: str, provider_name: str +) -> None: + """A session or connection hook registered after construction fires with telemetry disabled.""" + spec = SQLSpec() + config = spec.add_config(config_type(connection_config={"database": ":memory:"})) + fired: list[str] = [] + + def _hook(_context: dict[str, Any]) -> None: + fired.append(event_name) + + config.get_observability_runtime().register_lifecycle_hook(event_name, _hook) + _drive_registry_provider(spec, config, provider_name) + + assert fired == [event_name] + + @pytest.mark.parametrize( ("config_name", "driver", "db_system", "expected"), [ @@ -1575,3 +1626,30 @@ def test_create_statement_event_preserves_profile_when_redaction_disabled() -> N ) assert event.profile == {"query_name": "SELECT 'raw' WHERE id = 7"} + + +def test_get_trace_context_caches_otel_trace_module(monkeypatch: pytest.MonkeyPatch) -> None: + """get_trace_context resolves opentelemetry.trace once per process instead of re-importing per query.""" + import builtins + + import sqlspec.observability._common as obs_common + + obs_common._reset_otel_trace_cache() + real_import = builtins.__import__ + import_attempts = 0 + + def _counting_import(name: str, *args: Any, **kwargs: Any) -> Any: + nonlocal import_attempts + if name == "opentelemetry" or name.startswith("opentelemetry."): + import_attempts += 1 + return real_import(name, *args, **kwargs) + + try: + monkeypatch.setattr(builtins, "__import__", _counting_import) + obs_common.get_trace_context() + first_count = import_attempts + obs_common.get_trace_context() + obs_common.get_w3c_trace_context() + assert import_attempts == first_count + finally: + obs_common._reset_otel_trace_cache() diff --git a/tests/unit/storage/test_bridge.py b/tests/unit/storage/test_bridge.py index 00b07b418..a8513d5e3 100644 --- a/tests/unit/storage/test_bridge.py +++ b/tests/unit/storage/test_bridge.py @@ -3,7 +3,7 @@ import contextlib import sqlite3 -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncGenerator, Generator from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -964,13 +964,13 @@ def supports_direct_arrow_write(self, file_format: str, *, overwrite: bool = Tru @contextlib.contextmanager def open_write_stream_sync( self, path: "str | Path", *, overwrite: bool = True, **kwargs: Any - ) -> Iterator[WritableBytesSink]: + ) -> Generator[WritableBytesSink, None, None]: yield _SyncBufSink() @contextlib.asynccontextmanager async def open_write_stream_async( self, path: "str | Path", *, overwrite: bool = True, **kwargs: Any - ) -> AsyncIterator[AsyncWritableBytesSink]: + ) -> AsyncGenerator[AsyncWritableBytesSink, None]: yield _AsyncBufSink() diff --git a/tests/unit/storage/test_fsspec_backend.py b/tests/unit/storage/test_fsspec_backend.py index b08beff84..db77f01b6 100644 --- a/tests/unit/storage/test_fsspec_backend.py +++ b/tests/unit/storage/test_fsspec_backend.py @@ -1221,3 +1221,182 @@ def test_fsspec_file_directory_requires_recursive(tmp_path: Path) -> None: backend.delete_sync("a", recursive=True) assert not (tmp_path / "a").exists() + + +@pytest.mark.skipif(not FSSPEC_INSTALLED, reason="fsspec missing") +def test_fsspec_prefers_cat_file_and_rm_file_for_glob_chars() -> None: + """Single-object read_bytes_sync and non-recursive delete_sync prefer cat_file/rm_file over globbing cat/rm.""" + from sqlspec.storage.backends.fsspec import FSSpecBackend + + calls: list[tuple[str, str]] = [] + + class _FakeFS: + protocol = "memory" + + def cat_file(self, path: str, **_kwargs: Any) -> bytes: + calls.append(("cat_file", path)) + return b"ok" + + def cat(self, path: str, **_kwargs: Any) -> bytes: + calls.append(("cat", path)) + return b"glob" + + def rm_file(self, path: str, **_kwargs: Any) -> None: + calls.append(("rm_file", path)) + + def rm(self, path: str, *, recursive: bool = False, **_kwargs: Any) -> None: + calls.append((f"rm:{recursive}", path)) + + def unstrip_protocol(self, path: str) -> str: + return path + + backend = FSSpecBackend("memory://globtest") + backend.write_bytes_sync("dir/[meta].bin", b"real") + assert backend.read_bytes_sync("dir/[meta].bin") == b"real" + backend.delete_sync("dir/[meta].bin") + + backend.fs = _FakeFS() + assert backend.read_bytes_sync("dir/[meta].bin") == b"ok" + backend.delete_sync("dir/[meta].bin") + backend.delete_sync("dir", recursive=True) + + assert calls == [("cat_file", "dir/[meta].bin"), ("rm_file", "dir/[meta].bin"), ("rm:True", "dir")] + + +async def _call_store(store: "FSSpecBackend", operation: str, use_async: bool, *args: Any, **kwargs: Any) -> Any: + """Invoke ``_async`` or ``_sync`` on ``store``.""" + if use_async: + return await getattr(store, f"{operation}_async")(*args, **kwargs) + return getattr(store, f"{operation}_sync")(*args, **kwargs) + + +@pytest.mark.skipif(not FSSPEC_INSTALLED, reason="fsspec missing") +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("protocol_kind", ["file", "memory"]) +@pytest.mark.parametrize("key", ["t/[ab].bin", "t/*.bin", "t/?.bin"], ids=["brackets", "star", "question"]) +async def test_fsspec_copy_move_delete_treat_glob_characters_literally( + tmp_path: Path, protocol_kind: str, key: str, use_async: bool +) -> None: + """Copy, move, and delete act on the one literal key and leave glob-matching siblings alone.""" + from sqlspec.storage.backends.fsspec import FSSpecBackend + + base = str(tmp_path) if protocol_kind == "file" else f"/literal_{tmp_path.name}" + store = FSSpecBackend(protocol_kind, base_path=base) + store.write_bytes_sync("t/a.bin", b"sibling-a") + store.write_bytes_sync("t/b.bin", b"sibling-b") + store.write_bytes_sync(key, b"literal") + siblings = {"t/a.bin": b"sibling-a", "t/b.bin": b"sibling-b"} + if protocol_kind == "file": + (tmp_path / "out").mkdir() + + await _call_store(store, "copy", use_async, key, "out/copy.bin") + assert store.read_bytes_sync("out/copy.bin") == b"literal" + assert len(store.list_objects_sync("out")) == 1 + + await _call_store(store, "move", use_async, key, key) + assert store.read_bytes_sync(key) == b"literal" + + await _call_store(store, "move", use_async, key, "out/moved.bin") + assert store.read_bytes_sync("out/moved.bin") == b"literal" + assert not store.exists_sync(key) + assert {name: store.read_bytes_sync(name) for name in siblings} == siblings + + store.write_bytes_sync(key, b"literal") + await _call_store(store, "delete", use_async, key) + assert not store.exists_sync(key) + assert {name: store.read_bytes_sync(name) for name in siblings} == siblings + + +@pytest.mark.skipif(not FSSPEC_INSTALLED, reason="fsspec missing") +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("protocol_kind", ["file", "memory"]) +@pytest.mark.parametrize("prefix", ["tenants/*", "tenants/[to]??", "tenants/?ne"], ids=["star", "brackets", "question"]) +async def test_fsspec_recursive_delete_treats_glob_characters_literally( + tmp_path: Path, protocol_kind: str, prefix: str, use_async: bool +) -> None: + """Recursive delete removes only the literal prefix, not the prefixes its glob characters would match.""" + from sqlspec.exceptions import FileNotFoundInStorageError + from sqlspec.storage.backends.fsspec import FSSpecBackend + + base = str(tmp_path) if protocol_kind == "file" else f"/literal_tree_{tmp_path.name}" + store = FSSpecBackend(protocol_kind, base_path=base) + store.write_bytes_sync("tenants/one/data.bin", b"one") + store.write_bytes_sync("tenants/two/nested/data.bin", b"two") + store.write_bytes_sync(f"{prefix}/data.bin", b"literal") + store.write_bytes_sync(f"{prefix}/nested/data.bin", b"literal-nested") + + await _call_store(store, "delete", use_async, prefix, recursive=True) + + assert not store.exists_sync(prefix) + assert store.read_bytes_sync("tenants/one/data.bin") == b"one" + assert store.read_bytes_sync("tenants/two/nested/data.bin") == b"two" + assert len(store.list_objects_sync("tenants")) == 2 + if protocol_kind == "memory": + with pytest.raises(FileNotFoundInStorageError): + await _call_store(store, "delete", use_async, prefix, recursive=True) + + +@pytest.mark.skipif(not FSSPEC_INSTALLED, reason="fsspec missing") +def test_fsspec_recursive_delete_handles_find_returning_a_list(tmp_path: Path) -> None: + """Recursive delete of a literal key works on a filesystem whose ``find`` lists a single file as a list.""" + from fsspec.implementations.memory import MemoryFileSystem + + from sqlspec.storage.backends.fsspec import FSSpecBackend + + class _ListFindFileSystem(MemoryFileSystem): # type: ignore[misc] + def find(self, path: str, *args: Any, **kwargs: Any) -> Any: + found = super().find(path, *args, **kwargs) + return list(found) if self.isfile(path) else found + + store = FSSpecBackend("memory", base_path=f"/list_find_{tmp_path.name}") + store.fs = _ListFindFileSystem() + store.write_bytes_sync("exports/report[2026].parquet", b"report") + store.write_bytes_sync("exports/report2.parquet", b"sibling") + + store.delete_sync("exports/report[2026].parquet", recursive=True) + + assert not store.exists_sync("exports/report[2026].parquet") + assert store.read_bytes_sync("exports/report2.parquet") == b"sibling" + + +@pytest.mark.skipif(not FSSPEC_INSTALLED, reason="fsspec missing") +def test_fsspec_delete_and_move_fall_back_to_rm_without_rm_file(tmp_path: Path) -> None: + """Delete and move work on a filesystem that implements ``rm`` but not ``rm_file``.""" + from fsspec.implementations.memory import MemoryFileSystem + + from sqlspec.storage.backends.fsspec import FSSpecBackend + + class _RmOnlyFileSystem(MemoryFileSystem): # type: ignore[misc] + def rm_file(self, path: str) -> None: + raise NotImplementedError + + def rm(self, path: Any, recursive: bool = False, maxdepth: "int | None" = None) -> None: + self._rm(path) + + store = FSSpecBackend("memory", base_path=f"/rm_only_{tmp_path.name}") + store.fs = _RmOnlyFileSystem() + store.write_bytes_sync("plain.bin", b"plain") + store.write_bytes_sync("glob[1].bin", b"glob") + + store.delete_sync("plain.bin") + assert not store.exists_sync("plain.bin") + + store.move_sync("glob[1].bin", "moved.bin") + assert store.read_bytes_sync("moved.bin") == b"glob" + assert not store.exists_sync("glob[1].bin") + + +@pytest.mark.skipif(not FSSPEC_INSTALLED, reason="fsspec missing") +def test_fsspec_staged_stream_publishes_under_prefix_with_glob_characters(tmp_path: Path) -> None: + """A staged write stream publishes to a destination whose parent prefix contains glob characters.""" + from sqlspec.storage.backends.fsspec import FSSpecBackend + + store = FSSpecBackend("memory", base_path=f"/staged_glob_{tmp_path.name}") + store.write_bytes_sync("part/a/keep.bin", b"keep") + + with store.open_write_stream_sync("part/[ab]/out.bin") as sink: + sink.write(b"payload") + + assert store.read_bytes_sync("part/[ab]/out.bin") == b"payload" + assert store.read_bytes_sync("part/a/keep.bin") == b"keep" + assert _fsspec_stage_entries(store) == [] diff --git a/tests/unit/storage/test_pipeline_streaming.py b/tests/unit/storage/test_pipeline_streaming.py index 89b42a0d4..06c63ec3c 100644 --- a/tests/unit/storage/test_pipeline_streaming.py +++ b/tests/unit/storage/test_pipeline_streaming.py @@ -3,7 +3,7 @@ import asyncio import contextlib import threading -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncGenerator, Generator, Iterator from decimal import Decimal from pathlib import Path from typing import Any, cast @@ -308,13 +308,15 @@ def supports_direct_arrow_write(self, file_format: str, *, overwrite: bool = Tru return file_format == "arrow-ipc" and overwrite @contextlib.contextmanager - def open_write_stream_sync(self, path: str, *, overwrite: bool = True, **kwargs: Any) -> Iterator[Any]: + def open_write_stream_sync( + self, path: str, *, overwrite: bool = True, **kwargs: Any + ) -> Generator[Any, None, None]: raise NotImplementedError @contextlib.asynccontextmanager async def open_write_stream_async( self, path: str, *, overwrite: bool = True, **kwargs: Any - ) -> AsyncIterator[AsyncWritableBytesSink]: + ) -> AsyncGenerator[AsyncWritableBytesSink, None]: yield gated_sink registry = cast("StorageRegistry", _MapRegistry({"gate": _GatedBackend()})) diff --git a/tests/unit/storage/test_storage_registry.py b/tests/unit/storage/test_storage_registry.py index 17c45aee0..9db180e9f 100644 --- a/tests/unit/storage/test_storage_registry.py +++ b/tests/unit/storage/test_storage_registry.py @@ -4,12 +4,14 @@ import inspect import logging from pathlib import Path +from typing import Any import pytest from sqlspec.exceptions import ImproperConfigurationError, MissingDependencyError from sqlspec.protocols import ObjectStoreProtocol from sqlspec.storage.backends.local import LocalStore +from sqlspec.storage.pipeline import SyncStoragePipeline from sqlspec.storage.registry import StorageRegistry from sqlspec.typing import FSSPEC_INSTALLED, OBSTORE_INSTALLED from sqlspec.utils.type_guards import is_local_path @@ -315,3 +317,26 @@ def test_storage_registry_sanitizes_uri_credentials_in_logs_and_errors(caplog: p registry.get("nonexistent_alias?token=should-not-leak") assert "should-not-leak" not in str(exc_info.value) assert "nonexistent_alias" in str(exc_info.value) + + +@pytest.mark.skipif(not OBSTORE_INSTALLED, reason="obstore missing") +@pytest.mark.parametrize( + ("uri", "options"), + [ + ("s3a://example-bucket/dir/key.parquet", {"skip_signature": True, "region": "us-east-1"}), + ("abfs://container@account.dfs.core.windows.net/dir/key.parquet", {"skip_signature": True}), + ("abfss://container@account.dfs.core.windows.net/dir/key.parquet", {"skip_signature": True}), + ], + ids=["s3a", "abfs", "abfss"], +) +def test_storage_pipeline_strips_bucket_from_s3a_abfs_and_abfss_object_keys( + uri: str, options: "dict[str, Any]" +) -> None: + """The pipeline resolves s3a, abfs, and abfss URIs to a bucket-scoped backend and a bucket-relative key.""" + pipeline = SyncStoragePipeline(registry=StorageRegistry()) + + backend, key, backend_name = pipeline._backend(uri, options) + + assert backend_name == "obstore" + assert key == "dir/key.parquet" + assert backend.resolve_uri(key) == uri diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index e3286a1f8..143776235 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -3,6 +3,7 @@ import sqlite3 import asyncpg +import pytest from google.api_core import exceptions as api_exceptions from psycopg import errors as pg_errors @@ -18,6 +19,7 @@ QueryTimeoutError, StackExecutionError, map_sqlstate_to_exception, + with_cause, ) @@ -84,3 +86,25 @@ def test_stack_execution_error_redacts_sql_literals_in_message_while_preserving_ assert "private" not in str(err) assert "'***'" in str(err) assert "top-secret-token" not in err.args[0] + + +def test_with_cause_sets_the_direct_cause_and_returns_the_same_error() -> None: + cause = ValueError("driver failure") + error = OperationalError("mapped failure") + + assert with_cause(error, cause) is error + assert error.__cause__ is cause + assert error.__suppress_context__ is True + + +def test_with_cause_none_suppresses_the_error_being_handled() -> None: + handled = KeyError("lookup") + with pytest.raises(OperationalError) as raised: + try: + raise handled + except KeyError: + raise with_cause(OperationalError("replacement"), None) + + assert raised.value.__cause__ is None + assert raised.value.__context__ is handled + assert raised.value.__suppress_context__ is True diff --git a/tests/unit/test_mypyc_sybase_surface.py b/tests/unit/test_mypyc_sybase_surface.py index 5d9c0b634..655d8295d 100644 --- a/tests/unit/test_mypyc_sybase_surface.py +++ b/tests/unit/test_mypyc_sybase_surface.py @@ -55,8 +55,10 @@ def test_mypyc_inventory_includes_sybase_and_sqlanywhere_hot_surfaces() -> None: "sqlspec/data_dictionary/dialects/sybase/config.py", "sqlspec/data_dictionary/dialects/sqlanywhere/config.py", "sqlspec/adapters/sybase/core.py", + "sqlspec/adapters/sybase/driver.py", "sqlspec/adapters/sybase/pool.py", "sqlspec/adapters/sqlanywhere/core.py", + "sqlspec/adapters/sqlanywhere/driver.py", "sqlspec/adapters/sqlanywhere/pool.py", } for module_path in expected_compiled: @@ -69,11 +71,9 @@ def test_mypyc_inventory_includes_sybase_and_sqlanywhere_hot_surfaces() -> None: "sqlspec/adapters/sybase/_typing.py", "sqlspec/adapters/sybase/config.py", "sqlspec/adapters/sybase/data_dictionary.py", - "sqlspec/adapters/sybase/driver.py", "sqlspec/adapters/sqlanywhere/_typing.py", "sqlspec/adapters/sqlanywhere/config.py", "sqlspec/adapters/sqlanywhere/data_dictionary.py", - "sqlspec/adapters/sqlanywhere/driver.py", } for module_path in expected_interpreted: assert module_path in interpreted, f"Expected interpreted module: {module_path}" diff --git a/tests/unit/test_public_exports.py b/tests/unit/test_public_exports.py index d940250e6..bc5bad52e 100644 --- a/tests/unit/test_public_exports.py +++ b/tests/unit/test_public_exports.py @@ -213,7 +213,7 @@ def resolve(name): ("sqlspec.observability", "ObservabilityConfig", "sqlspec.observability._common", "resolve_db_system"), ("sqlspec.data_dictionary", "get_dialect_config", "sqlspec.data_dictionary._registry", "get_dialect_config"), ("sqlspec.utils.tenant", "TenantContext", "sqlspec.utils.tenant._context", "TenantContext"), - ("sqlspec.utils.serializers", "to_json", "sqlspec.utils.serializers._json", "encode_json"), + ("sqlspec.utils.serializers", "to_json", "sqlspec.utils.serializers._json", "to_json"), ], ) def test_concurrent_package_and_submodule_imports_preserve_identity( diff --git a/tests/unit/test_service_config.py b/tests/unit/test_service_config.py index 329c76f8a..b92f5bb54 100644 --- a/tests/unit/test_service_config.py +++ b/tests/unit/test_service_config.py @@ -3,7 +3,7 @@ import asyncio import sqlite3 import threading -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncGenerator, AsyncIterator, Generator, Iterator from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager, closing, contextmanager, nullcontext from contextvars import copy_context @@ -40,7 +40,7 @@ def sync_config( session.commit() @contextmanager - def observed(self: SqliteConfig, *args: Any, **kwargs: Any) -> Iterator[SqliteDriver]: + def observed(self: SqliteConfig, *args: Any, **kwargs: Any) -> Generator[SqliteDriver, None, None]: driver: SqliteDriver | None = None try: with original(self, *args, **kwargs) as driver: @@ -68,7 +68,7 @@ async def async_config( await session.commit() @asynccontextmanager - async def observed(self: AiosqliteConfig, *args: Any, **kwargs: Any) -> AsyncIterator[AiosqliteDriver]: + async def observed(self: AiosqliteConfig, *args: Any, **kwargs: Any) -> AsyncGenerator[AiosqliteDriver, None]: driver: AiosqliteDriver | None = None try: async with original(self, *args, **kwargs) as driver: @@ -367,7 +367,7 @@ def test_sync_transaction_failure_releases_and_resets( original_provider = SqliteConfig.provide_session @contextmanager - def provider(self: SqliteConfig) -> Iterator[SqliteDriver]: + def provider(self: SqliteConfig) -> Generator[SqliteDriver, None, None]: try: with original_provider(self) as driver: yield driver @@ -408,7 +408,7 @@ async def test_async_transaction_failure_releases_and_resets( original_provider = AiosqliteConfig.provide_session @asynccontextmanager - async def provider(self: AiosqliteConfig) -> AsyncIterator[AiosqliteDriver]: + async def provider(self: AiosqliteConfig) -> AsyncGenerator[AiosqliteDriver, None]: try: async with original_provider(self) as driver: yield driver diff --git a/tests/unit/utils/serializers/test_default_type_encoders.py b/tests/unit/utils/serializers/test_default_type_encoders.py index 53b0ce698..46674ed63 100644 --- a/tests/unit/utils/serializers/test_default_type_encoders.py +++ b/tests/unit/utils/serializers/test_default_type_encoders.py @@ -28,7 +28,7 @@ OrjsonSerializer, StandardLibSerializer, _enc_hook, - encode_json, + to_json, ) @@ -369,7 +369,7 @@ def test_offset_pagination_round_trip_via_dataclass_branch() -> None: from sqlspec.core import OffsetPagination page: OffsetPagination[int] = OffsetPagination(items=[1, 2, 3], limit=10, offset=0, total=3) - assert json.loads(encode_json(page)) == {"items": [1, 2, 3], "limit": 10, "offset": 0, "total": 3} + assert json.loads(to_json(page)) == {"items": [1, 2, 3], "limit": 10, "offset": 0, "total": 3} def test_purepath_subclass_resolves_via_mro() -> None: diff --git a/tests/unit/utils/test_fixtures.py b/tests/unit/utils/test_fixtures.py index 134a0bd9e..4fa8d6fcf 100644 --- a/tests/unit/utils/test_fixtures.py +++ b/tests/unit/utils/test_fixtures.py @@ -3333,3 +3333,55 @@ def test_postgres_fixture_insert_statement_casts_json_placeholders_via_text(json assert sql_text == ( f'INSERT INTO "events" ("id", "payload") VALUES (%(id)s, CAST(CAST(%(payload)s AS TEXT) AS {expected_cast}))' ) + + +def test_fixture_insert_sanitizes_placeholder_names_for_special_columns() -> None: + """Fixture INSERT builder sanitizes columns containing spaces or punctuation while remapping row dicts.""" + columns = [ + fixture_module._TableColumn( + name="first name", + data_type="text", + is_primary=False, + is_identity=False, + identity_always=False, + sequence_name=None, + is_generated=False, + ), + fixture_module._TableColumn( + name="first-name", + data_type="text", + is_primary=False, + is_identity=False, + identity_always=False, + sequence_name=None, + is_generated=False, + ), + ] + sql_text = fixture_module._table_insert_statement( + "sqlite", "sqlite", "users", ["first name", "first-name"], (), columns + ) + assert ":first_name" in sql_text + assert ":first_name_2" in sql_text + + recorded: list[tuple[str, Any]] = [] + + class _FakeDriver: + def execute(self, *_args: Any) -> None: + return None + + def execute_many(self, stmt: str, batch: list[dict[str, Any]]) -> None: + recorded.append((stmt, batch)) + + count = fixture_module._insert_table_rows_sync( + cast("Any", _FakeDriver()), + "sqlite", + "sqlite", + "users", + [{"first name": "Ada", "first-name": "Lovelace"}], + columns, + (), + set(), + 10, + ) + assert count == 1 + assert recorded[0][1] == [{"first_name": "Ada", "first_name_2": "Lovelace"}] diff --git a/tests/unit/utils/test_routing_context.py b/tests/unit/utils/test_routing_context.py index b65daaebf..f9ffe2633 100644 --- a/tests/unit/utils/test_routing_context.py +++ b/tests/unit/utils/test_routing_context.py @@ -168,6 +168,12 @@ def test_signed_primary_pin_roundtrip_expiry_and_bind_isolation() -> None: assert verify_primary_pin("x" * 1025, "top-secret", clock=lambda: now) == (False, None) +@pytest.mark.parametrize("token", [None, 123, b"v1.1.a.b.c"]) +def test_verify_primary_pin_rejects_missing_and_non_string_tokens(token: Any) -> None: + """A missing cookie or a non-string value is reported as an invalid pin instead of raising.""" + assert verify_primary_pin(token, "top-secret") == (False, None) + + @pytest.mark.parametrize( ("secret", "kwargs", "exc_type"), [ diff --git a/tools/perf_baselines/sqlite.json b/tools/perf_baselines/sqlite.json index 86ae88998..c39449fa1 100644 --- a/tools/perf_baselines/sqlite.json +++ b/tools/perf_baselines/sqlite.json @@ -2,21 +2,35 @@ "metadata": { "rows": 1000, "pool_size": 1, - "iterations": 5, - "warmup": 2, + "iterations": 7, + "warmup": 3, "mypyc_compiled": false, "environment": { - "python_version": "3.11.9", - "platform": "Linux-7.1.6-1rodete1-amd64-x86_64-with-glibc2.43", + "python_version": "3.11.14", + "platform": "Linux-7.0.0-43-generic-x86_64-with-glibc2.43", "packages": { - "duckdb": "1.5.6", - "sqlglot": "30.22.0", + "duckdb": "2.0.0", + "sqlglot": "30.30.0", "aiosqlite": "0.22.1", "pyarrow": "26.0.0", - "sqlspec": "0.66.0" - } - }, - "timestamp": "2026-10-13T21:37:48+0000" + "sqlspec": "0.72.0" + }, + "cpu_count": 12, + "sqlglot_version": "30.30.0", + "driver_versions": { + "sqlite3": "3.50.4", + "adbc-driver-manager": "1.13.0", + "aiosqlite": "0.22.1", + "asyncmy": "0.2.17", + "asyncpg": "0.32.0", + "duckdb": "2.0.0", + "oracledb": "26.1.0", + "psqlpy": "0.13.1", + "psycopg": "3.3.6" + }, + "service_id": null + }, + "timestamp": "2026-11-17T16:13:10+0000" }, "results": [ { @@ -24,17 +38,19 @@ "library": "raw", "library_key": "raw", "scenario": "initialization", - "time": 0.002675202995305881, + "time": 0.004335767938755453, "times": [ - 0.0027365599962649867, - 0.002725298996665515, - 0.002611039992189035, - 0.002675202995305881, - 0.002634947988553904 + 0.004370943992398679, + 0.004535520914942026, + 0.004312830977141857, + 0.004294575075618923, + 0.004526066943071783, + 0.004335767938755453, + 0.004099272075109184 ], - "stddev": 5.477624766433942e-05, - "iqr": 9.035100811161101e-05, - "noise_ratio": 0.020475548121190835, + "stddev": 0.00014904563254411823, + "iqr": 0.000144802441354841, + "noise_ratio": 0.03437583252827424, "noisy": false }, { @@ -42,35 +58,39 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "initialization", - "time": 0.00282091400003992, + "time": 0.0033804330741986632, "times": [ - 0.0026993060018867254, - 0.0034525980008766055, - 0.0028972849977435544, - 0.0028015130083076656, - 0.00282091400003992 + 0.003711733967065811, + 0.0034836909035220742, + 0.0033804330741986632, + 0.0033416320802643895, + 0.003401312045753002, + 0.0033778409706428647, + 0.0026543480344116688 ], - "stddev": 0.0002982087669368678, - "iqr": 9.577198943588883e-05, - "noise_ratio": 0.10571352651397657, - "noisy": true + "stddev": 0.0003254894320655094, + "iqr": 8.276494918391109e-05, + "noise_ratio": 0.09628631152316694, + "noisy": false }, { "driver": "sqlite", "library": "sqlalchemy", "library_key": "sqlalchemy", "scenario": "initialization", - "time": 0.0021299579966580495, + "time": 0.002610011026263237, "times": [ - 0.002090784997562878, - 0.002119087992468849, - 0.002180947005399503, - 0.0021560580062214285, - 0.0021299579966580495 + 0.002620298066176474, + 0.0024148650700226426, + 0.0023683670442551374, + 0.0028510469710454345, + 0.00272123992908746, + 0.002610011026263237, + 0.0024647440295666456 ], - "stddev": 3.459578361278528e-05, - "iqr": 3.697001375257969e-05, - "noise_ratio": 0.016242472230469717, + "stddev": 0.00017381458890211262, + "iqr": 0.00023096444783732295, + "noise_ratio": 0.06659534659168227, "noisy": false }, { @@ -78,17 +98,19 @@ "library": "raw", "library_key": "raw", "scenario": "write_heavy", - "time": 0.003833413007669151, + "time": 0.003948459983803332, "times": [ - 0.0040214440086856484, - 0.003827246997389011, - 0.0036687200044980273, - 0.003833413007669151, - 0.0038556829968001693 + 0.004474070039577782, + 0.004724171943962574, + 0.0036564769688993692, + 0.003934997948817909, + 0.003948459983803332, + 0.003882606979459524, + 0.003993142978288233 ], - "stddev": 0.0001252021174245056, - "iqr": 2.8435999411158264e-05, - "noise_ratio": 0.032660743096041424, + "stddev": 0.0003728363182771136, + "iqr": 0.00032480404479429126, + "noise_ratio": 0.09442575581530424, "noisy": false }, { @@ -96,17 +118,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "write_heavy", - "time": 0.004287199990358204, + "time": 0.004459739080630243, "times": [ - 0.004193825006950647, - 0.004254860003129579, - 0.004306560003897175, - 0.004420480006956495, - 0.004287199990358204 + 0.00433427793905139, + 0.004575498984195292, + 0.004495795001275837, + 0.004459739080630243, + 0.0044588060118258, + 0.004284303984604776, + 0.004543433082289994 ], - "stddev": 8.330387668522975e-05, - "iqr": 5.1700000767596066e-05, - "noise_ratio": 0.01943083524738242, + "stddev": 0.00010614109116045084, + "iqr": 0.00012307206634432077, + "noise_ratio": 0.02379984327366774, "noisy": false }, { @@ -114,17 +138,19 @@ "library": "sqlalchemy", "library_key": "sqlalchemy", "scenario": "write_heavy", - "time": 0.005627367994748056, + "time": 0.006842036033049226, "times": [ - 0.005526088003534824, - 0.005484861001605168, - 0.005846761996508576, - 0.005900725998799317, - 0.005627367994748056 + 0.006842036033049226, + 0.006421920959837735, + 0.006295768078416586, + 0.007126844022423029, + 0.007017450989224017, + 0.006796189001761377, + 0.007123388000763953 ], - "stddev": 0.00018776813475699234, - "iqr": 0.0003206739929737523, - "noise_ratio": 0.033366955019155264, + "stddev": 0.000331055721346827, + "iqr": 0.0004613645141944289, + "noise_ratio": 0.048385556543070196, "noisy": false }, { @@ -132,17 +158,19 @@ "library": "raw", "library_key": "raw", "scenario": "read_heavy", - "time": 0.004371547998744063, + "time": 0.004743056022562087, "times": [ - 0.004326966998632997, - 0.004294172991649248, - 0.004471345993806608, - 0.004371547998744063, - 0.0044424379884731025 + 0.004835583968088031, + 0.004662645980715752, + 0.004783147014677525, + 0.004743056022562087, + 0.004844794049859047, + 0.004677701974287629, + 0.004706606967374682 ], - "stddev": 7.497403988753363e-05, - "iqr": 0.00011547098984010518, - "noise_ratio": 0.01715045560727539, + "stddev": 7.325340630535809e-05, + "iqr": 0.00011721102055162191, + "noise_ratio": 0.015444347685732865, "noisy": false }, { @@ -150,17 +178,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "read_heavy", - "time": 0.005011429006117396, + "time": 0.0039920329581946135, "times": [ - 0.005175757993129082, - 0.005084271993837319, - 0.005011429006117396, - 0.0049597659963183105, - 0.004917177007882856 + 0.004133931011892855, + 0.0039920329581946135, + 0.003750842995941639, + 0.004003889043815434, + 0.0036798290675505996, + 0.0040249929297715425, + 0.003724480979144573 ], - "stddev": 0.00010270960981180456, - "iqr": 0.00012450599751900882, - "noise_ratio": 0.02049507429645877, + "stddev": 0.0001784776572647114, + "iqr": 0.0002767789992503822, + "noise_ratio": 0.04470846286435157, "noisy": false }, { @@ -168,17 +198,19 @@ "library": "sqlalchemy", "library_key": "sqlalchemy", "scenario": "read_heavy", - "time": 0.006095218996051699, + "time": 0.007906956016086042, "times": [ - 0.006095218996051699, - 0.006120414996985346, - 0.006083756001316942, - 0.006076361009036191, - 0.006195608002599329 + 0.00785231706686318, + 0.007870655041188002, + 0.008143718936480582, + 0.007724882918410003, + 0.008235729997977614, + 0.007906956016086042, + 0.008214759989641607 ], - "stddev": 4.843750639106733e-05, - "iqr": 3.6658995668403804e-05, - "noise_ratio": 0.007946803293276862, + "stddev": 0.00020201538550842105, + "iqr": 0.00031775340903550386, + "noise_ratio": 0.02554907161459323, "noisy": false }, { @@ -186,17 +218,19 @@ "library": "raw", "library_key": "raw", "scenario": "iterative_inserts", - "time": 0.004181689990218729, + "time": 0.004499407019466162, "times": [ - 0.004181689990218729, - 0.00413410400506109, - 0.004149139000219293, - 0.004257650987710804, - 0.004246014999807812 + 0.005027178907766938, + 0.004652772913686931, + 0.00465235800947994, + 0.004444435006007552, + 0.004410458961501718, + 0.0039364820113405585, + 0.004499407019466162 ], - "stddev": 5.592001368855386e-05, - "iqr": 9.687599958851933e-05, - "noise_ratio": 0.013372587116537753, + "stddev": 0.0003293841523900993, + "iqr": 0.00022511847782880068, + "noise_ratio": 0.07320612493269825, "noisy": false }, { @@ -204,17 +238,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "iterative_inserts", - "time": 0.009428682998986915, + "time": 0.005593776004388928, "times": [ - 0.009358828014228493, - 0.00932132700108923, - 0.009531204996164888, - 0.009428682998986915, - 0.009545498003717512 + 0.005593776004388928, + 0.005415856023319066, + 0.005871367990039289, + 0.00596916803624481, + 0.005209057009778917, + 0.005651762941852212, + 0.005503928987309337 ], - "stddev": 0.00010025793499290344, - "iqr": 0.00017237698193639517, - "noise_ratio": 0.010633291521591709, + "stddev": 0.00026095562918076586, + "iqr": 0.00030167296063154936, + "noise_ratio": 0.04665106879074487, "noisy": false }, { @@ -222,17 +258,19 @@ "library": "sqlalchemy", "library_key": "sqlalchemy", "scenario": "iterative_inserts", - "time": 0.028197134990477934, + "time": 0.03164735098835081, "times": [ - 0.028197134990477934, - 0.02821979699365329, - 0.028157600012491457, - 0.02819776799879037, - 0.027938569997786544 + 0.03179049305617809, + 0.031633345992304385, + 0.0316315449308604, + 0.031615594984032214, + 0.03193962492514402, + 0.03203303902409971, + 0.03164735098835081 ], - "stddev": 0.00011600569371612893, - "iqr": 4.0167986298911273e-05, - "noise_ratio": 0.004114095058072517, + "stddev": 0.00017015170016161622, + "iqr": 0.0002326135290786624, + "noise_ratio": 0.005376491075801194, "noisy": false }, { @@ -240,17 +278,19 @@ "library": "raw", "library_key": "raw", "scenario": "repeated_queries", - "time": 0.10908412099524867, + "time": 0.08772899198811501, "times": [ - 0.10908412099524867, - 0.10999828499916475, - 0.10901488899253309, - 0.10906870799954049, - 0.10911709500942379 + 0.08772899198811501, + 0.08770388108678162, + 0.08883201202843338, + 0.0952128820354119, + 0.08916934498120099, + 0.08770332706626505, + 0.087422383017838 ], - "stddev": 0.00041624358869188675, - "iqr": 4.838700988329947e-05, - "noise_ratio": 0.0038158036650450425, + "stddev": 0.0027699909893786675, + "iqr": 0.001297074428293854, + "noise_ratio": 0.03157440803325232, "noisy": false }, { @@ -258,17 +298,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "repeated_queries", - "time": 0.11235622799722478, + "time": 0.08900431008078158, "times": [ - 0.1121271220035851, - 0.112780675990507, - 0.11174950200074818, - 0.11350953599321656, - 0.11235622799722478 + 0.09118032699916512, + 0.09026313503272831, + 0.08900431008078158, + 0.08839647995773703, + 0.08889857493340969, + 0.08961459808051586, + 0.08807339705526829 ], - "stddev": 0.0006746834775724083, - "iqr": 0.0006535539869219065, - "noise_ratio": 0.006004860519072188, + "stddev": 0.0010890419164794767, + "iqr": 0.0012913391110487282, + "noise_ratio": 0.012235833472458208, "noisy": false }, { @@ -276,17 +318,19 @@ "library": "sqlalchemy", "library_key": "sqlalchemy", "scenario": "repeated_queries", - "time": 0.16552351499558426, + "time": 0.14078482799232006, "times": [ - 0.1641665240022121, - 0.16491104500892106, - 0.16918576699390542, - 0.17900026100687683, - 0.16552351499558426 + 0.140766387921758, + 0.14093793800566345, + 0.14028679195325822, + 0.14455940492916852, + 0.14299192698672414, + 0.140768563025631, + 0.14078482799232006 ], - "stddev": 0.006148746743329162, - "iqr": 0.004274721984984353, - "noise_ratio": 0.037147270244310575, + "stddev": 0.0015762577937641852, + "iqr": 0.001197457022499293, + "noise_ratio": 0.01119621919664647, "noisy": false }, { @@ -294,17 +338,19 @@ "library": "raw", "library_key": "raw", "scenario": "dict_key_transform", - "time": 0.009565733998897485, + "time": 0.009764551999978721, "times": [ - 0.009551441995427012, - 0.009336190996691585, - 0.009565733998897485, - 0.009962566997273825, - 0.009776612991117872 + 0.009175091981887817, + 0.009810257004573941, + 0.010031813057139516, + 0.009868148947134614, + 0.009764551999978721, + 0.009543713997118175, + 0.009712141007184982 ], - "stddev": 0.0002389347908709586, - "iqr": 0.00022517099569085985, - "noise_ratio": 0.024978197271479367, + "stddev": 0.00027531618687105965, + "iqr": 0.00021127547370269895, + "noise_ratio": 0.02819547551916971, "noisy": false }, { @@ -312,17 +358,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "dict_key_transform", - "time": 0.01544887199997902, + "time": 0.010424409061670303, "times": [ - 0.015325487009249628, - 0.01544887199997902, - 0.015402306002215482, - 0.015676584996981546, - 0.015492282997001894 + 0.010216107009910047, + 0.010424409061670303, + 0.010470677050761878, + 0.010552800027653575, + 0.010108203045092523, + 0.010718515026383102, + 0.010357138002291322 ], - "stddev": 0.0001314180127890617, - "iqr": 8.997699478641152e-05, - "noise_ratio": 0.008506641312662839, + "stddev": 0.00020447611331867764, + "iqr": 0.0002251160331070423, + "noise_ratio": 0.01961512754430556, "noisy": false }, { @@ -330,17 +378,19 @@ "library": "raw", "library_key": "raw", "scenario": "schema_mapping", - "time": 0.00957132400071714, + "time": 0.010748577071353793, "times": [ - 0.009600805002264678, - 0.00957132400071714, - 0.00926685800368432, - 0.00946096501138527, - 0.010403750988189131 + 0.009710537968203425, + 0.010970640927553177, + 0.011252216063439846, + 0.010748577071353793, + 0.009751014993526042, + 0.010219092946499586, + 0.010840734001249075 ], - "stddev": 0.00043551860653332256, - "iqr": 0.00013983999087940902, - "noise_ratio": 0.04550244109390623, + "stddev": 0.0006094194428678835, + "iqr": 0.0009206334943883121, + "noise_ratio": 0.05669768554686714, "noisy": false }, { @@ -348,17 +398,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "schema_mapping", - "time": 0.013286437999340706, + "time": 0.008897320018149912, "times": [ - 0.013902839011279866, - 0.01326507500198204, - 0.013467296987073496, - 0.013106648999382742, - 0.013286437999340706 + 0.009421866969205439, + 0.009213151992298663, + 0.008850419078953564, + 0.008430824032984674, + 0.008897320018149912, + 0.00892789603676647, + 0.008623883943073452 ], - "stddev": 0.0003059296050691119, - "iqr": 0.00020222198509145528, - "noise_ratio": 0.023025705240508603, + "stddev": 0.0003340181615090357, + "iqr": 0.00033337250351905823, + "noise_ratio": 0.03754143504197466, "noisy": false }, { @@ -366,17 +418,19 @@ "library": "raw", "library_key": "raw", "scenario": "complex_parameters", - "time": 0.00826842499373015, + "time": 0.009788242052309215, "times": [ - 0.008301143010612577, - 0.008119215999613516, - 0.008257767010945827, - 0.00826842499373015, - 0.00859405800292734 + 0.009973446023650467, + 0.009371003019623458, + 0.009632189990952611, + 0.00909399800002575, + 0.009788242052309215, + 0.009807934984564781, + 0.0098552800482139 ], - "stddev": 0.00017435958410293485, - "iqr": 4.337599966675043e-05, - "noise_ratio": 0.021087399865772464, + "stddev": 0.00031037478418095133, + "iqr": 0.0003300110111013055, + "noise_ratio": 0.031708940433050344, "noisy": false }, { @@ -384,17 +438,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "complex_parameters", - "time": 0.009813410986680537, + "time": 0.0068059819750487804, "times": [ - 0.00970755000889767, - 0.00992931799555663, - 0.009813410986680537, - 0.009962814001482911, - 0.009700522001367062 + 0.007320009055547416, + 0.0068059819750487804, + 0.006712261936627328, + 0.006802598014473915, + 0.007377100992016494, + 0.006354287965223193, + 0.00749253504909575 ], - "stddev": 0.00012172982103064316, - "iqr": 0.00022176798665896058, - "noise_ratio": 0.012404435236215375, + "stddev": 0.0004205031180166003, + "iqr": 0.0005911250482313335, + "noise_ratio": 0.06178434200357788, "noisy": false }, { @@ -402,17 +458,19 @@ "library": "raw", "library_key": "raw", "scenario": "thin_path_stress", - "time": 0.018350698010181077, + "time": 0.016130015021190047, "times": [ - 0.018361185997491702, - 0.018097220003255643, - 0.018584149991511367, - 0.018350698010181077, - 0.01830038399202749 + 0.01574800501111895, + 0.015771799022331834, + 0.016212303075008094, + 0.016130015021190047, + 0.016865947050973773, + 0.016172299976460636, + 0.015976262977346778 ], - "stddev": 0.0001736922365496957, - "iqr": 6.0802005464211106e-05, - "noise_ratio": 0.009465156935901305, + "stddev": 0.0003759178259244083, + "iqr": 0.0003182705258950591, + "noise_ratio": 0.023305485173483347, "noisy": false }, { @@ -420,17 +478,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "thin_path_stress", - "time": 0.0683942119940184, + "time": 0.03441405401099473, "times": [ - 0.0683942119940184, - 0.06859845398867037, - 0.06788588200288359, - 0.06728459399892017, - 0.07255751600314397 + 0.03328438405878842, + 0.034208326949737966, + 0.03611500596161932, + 0.03425449400674552, + 0.03441405401099473, + 0.03472177102230489, + 0.03446457698009908 ], - "stddev": 0.0020828111225034587, - "iqr": 0.0007125719857867807, - "noise_ratio": 0.030453031942024807, + "stddev": 0.0008454719529346822, + "iqr": 0.0003617635229602456, + "noise_ratio": 0.02456763602057949, "noisy": false }, { @@ -438,17 +498,19 @@ "library": "raw", "library_key": "raw", "scenario": "schema_type_numpy", - "time": 0.051206131000071764, + "time": 0.017359091085381806, "times": [ - 0.050969773990800604, - 0.05114704099833034, - 0.051770587990176864, - 0.051206131000071764, - 0.051988815001095645 + 0.017514660023152828, + 0.01737155602313578, + 0.017895041964948177, + 0.017087160027585924, + 0.017075509997084737, + 0.017359091085381806, + 0.01713676901999861 ], - "stddev": 0.0004385630392862737, - "iqr": 0.0006235469918465242, - "noise_ratio": 0.008564658776615228, + "stddev": 0.00029286320553590896, + "iqr": 0.0003311434993520379, + "noise_ratio": 0.01687088362492267, "noisy": false }, { @@ -456,17 +518,59 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "schema_type_numpy", - "time": 0.03937012600363232, + "time": 0.008491109940223396, "times": [ - 0.039747274000546895, - 0.03891997299797367, - 0.039939324007718824, - 0.03937012600363232, - 0.039315785994403996 + 0.008491109940223396, + 0.008352005039341748, + 0.008421472972258925, + 0.008437461918219924, + 0.008627098053693771, + 0.008568616001866758, + 0.008534990018233657 ], - "stddev": 0.00039773443787016765, - "iqr": 0.0004314880061428994, - "noise_ratio": 0.010102442593998107, + "stddev": 9.442758694826448e-05, + "iqr": 0.00012233556481078267, + "noise_ratio": 0.01112075896002121, + "noisy": false + }, + { + "driver": "sqlite", + "library": "raw", + "library_key": "raw", + "scenario": "declared_parameters", + "time": 0.5594844609731808, + "times": [ + 0.5595156580675393, + 0.5553837389452383, + 0.5815954350400716, + 0.5594844609731808, + 0.565795739996247, + 0.5570314479991794, + 0.5575527199544013 + ], + "stddev": 0.009113480936061203, + "iqr": 0.005363615055102855, + "noise_ratio": 0.016289068904986197, + "noisy": false + }, + { + "driver": "sqlite", + "library": "sqlspec", + "library_key": "sqlspec", + "scenario": "declared_parameters", + "time": 0.6029251200379804, + "times": [ + 0.5977484850445762, + 0.5951057759812102, + 0.6174034300493076, + 0.6029344130074605, + 0.6012199140386656, + 0.6029251200379804, + 0.6314223599620163 + ], + "stddev": 0.012898842918920777, + "iqr": 0.01068472198676318, + "noise_ratio": 0.02139377260995235, "noisy": false }, { @@ -474,17 +578,19 @@ "library": "raw", "library_key": "raw", "scenario": "scalar_param_select", - "time": 0.0013580560043919832, + "time": 0.0010593840852379799, "times": [ - 0.0013478469918482006, - 0.0013622740079881623, - 0.0013550139992730692, - 0.0013580560043919832, - 0.0013764250034000725 + 0.0010830099927261472, + 0.0010596660431474447, + 0.001077515073120594, + 0.0010510559659451246, + 0.0010465170489624143, + 0.0010593840852379799, + 0.001048788893967867 ], - "stddev": 1.0621408665912961e-05, - "iqr": 7.260008715093136e-06, - "noise_ratio": 0.007821038772748024, + "stddev": 1.4254633417049019e-05, + "iqr": 1.8668128177523613e-05, + "noise_ratio": 0.013455585765050323, "noisy": false }, { @@ -492,17 +598,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "scalar_param_select", - "time": 0.007731876001344062, + "time": 0.004632564028725028, "times": [ - 0.007662418996915221, - 0.007734676008112729, - 0.007731876001344062, - 0.00781970500247553, - 0.007689688005484641 + 0.004594160011038184, + 0.004839429981075227, + 0.004640675964765251, + 0.0047266019973903894, + 0.004632564028725028, + 0.004579934990033507, + 0.00461426901165396 ], - "stddev": 5.966058487248296e-05, - "iqr": 4.4988002628088e-05, - "noise_ratio": 0.007716184902876342, + "stddev": 9.181993114552221e-05, + "iqr": 7.94244697317481e-05, + "noise_ratio": 0.01982054226907099, "noisy": false }, { @@ -510,17 +618,19 @@ "library": "raw", "library_key": "raw", "scenario": "no_param_select", - "time": 0.0013393950066529214, + "time": 0.0010081400396302342, "times": [ - 0.001320764989941381, - 0.0013500049972208217, - 0.0013393950066529214, - 0.0013467960088746622, - 0.001333893000264652 + 0.0010081400396302342, + 0.0010372439865022898, + 0.001042937976308167, + 0.001000502030365169, + 0.0009958629962056875, + 0.001036742003634572, + 0.0010031139245256782 ], - "stddev": 1.1588819270037595e-05, - "iqr": 1.2903008610010147e-05, - "noise_ratio": 0.008652278986015822, + "stddev": 2.0241183417992388e-05, + "iqr": 3.51850176230073e-05, + "noise_ratio": 0.02007774973942752, "noisy": false }, { @@ -528,17 +638,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "no_param_select", - "time": 0.0053518780041486025, + "time": 0.0033864170545712113, "times": [ - 0.005385716998716816, - 0.0053518780041486025, - 0.005310234002536163, - 0.005422768997959793, - 0.005300614007865079 + 0.0033593899570405483, + 0.0033701759530231357, + 0.0033864170545712113, + 0.0033908940386027098, + 0.0036088910419493914, + 0.0034030129900202155, + 0.003369480953551829 ], - "stddev": 5.1246655623517794e-05, - "iqr": 7.548299618065357e-05, - "noise_ratio": 0.009575452875381883, + "stddev": 8.781527899776844e-05, + "iqr": 2.712506102398038e-05, + "noise_ratio": 0.02593161963888338, "noisy": false }, { @@ -546,17 +658,19 @@ "library": "raw", "library_key": "raw", "scenario": "complex_query", - "time": 0.0030574999982491136, + "time": 0.002532416023313999, "times": [ - 0.003050535000511445, - 0.0030785599956288934, - 0.0030124610057100654, - 0.003082666007685475, - 0.0030574999982491136 + 0.002506213029846549, + 0.0025418290169909596, + 0.0026709779631346464, + 0.002532416023313999, + 0.0025243479758501053, + 0.0025319099659100175, + 0.002543956972658634 ], - "stddev": 2.804994003750898e-05, - "iqr": 2.802499511744827e-05, - "noise_ratio": 0.009174142290620396, + "stddev": 5.46966360927739e-05, + "iqr": 1.4764023944735527e-05, + "noise_ratio": 0.02159859817234775, "noisy": false }, { @@ -564,17 +678,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "complex_query", - "time": 0.006443192993174307, + "time": 0.0053096660412848, "times": [ - 0.0064240339997923, - 0.006305467002675869, - 0.006445741004426964, - 0.006443192993174307, - 0.006471240005339496 + 0.0053096660412848, + 0.005284088896587491, + 0.005471133976243436, + 0.005274645052850246, + 0.005360667943023145, + 0.005315801012329757, + 0.005291476962156594 ], - "stddev": 6.507459549662343e-05, - "iqr": 2.1707004634663463e-05, - "noise_ratio": 0.010099743336193899, + "stddev": 6.847082086487153e-05, + "iqr": 5.045154830440879e-05, + "noise_ratio": 0.012895504224273848, "noisy": false }, { @@ -582,17 +698,19 @@ "library": "raw", "library_key": "raw", "scenario": "session_per_request", - "time": 0.010022154005127959, + "time": 0.009761010995134711, "times": [ - 0.010022154005127959, - 0.010339030995965004, - 0.01021236200176645, - 0.009826769004575908, - 0.009880614001303911 + 0.010327321011573076, + 0.009761010995134711, + 0.009475197992287576, + 0.009393716929480433, + 0.010132038034498692, + 0.009874555049464107, + 0.00946239905897528 ], - "stddev": 0.00021737530705022898, - "iqr": 0.0003317480004625395, - "noise_ratio": 0.021689479820306713, + "stddev": 0.00035954107570837333, + "iqr": 0.0005344980163499713, + "noise_ratio": 0.03683440945692853, "noisy": false }, { @@ -600,17 +718,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "session_per_request", - "time": 0.02374547498766333, + "time": 0.02312270808033645, "times": [ - 0.023479077994124964, - 0.02374547498766333, - 0.0237338339939015, - 0.027480500008095987, - 0.024171199009288102 + 0.02316584892105311, + 0.022735937032848597, + 0.021989678032696247, + 0.023261870024725795, + 0.02312270808033645, + 0.022884926991537213, + 0.023358808015473187 ], - "stddev": 0.0016723984789667789, - "iqr": 0.00043736501538660377, - "noise_ratio": 0.07043019690427979, + "stddev": 0.0004673790222771951, + "iqr": 0.0004034274606965482, + "noise_ratio": 0.020212988057166807, "noisy": false }, { @@ -618,17 +738,19 @@ "library": "raw", "library_key": "raw", "scenario": "observability_select", - "time": 0.017145584992249496, + "time": 0.016415206948295236, "times": [ - 0.017244325004867278, - 0.017175964007037692, - 0.017145584992249496, - 0.01694815600058064, - 0.017087613989133388 + 0.0161333050346002, + 0.016415206948295236, + 0.015968695981428027, + 0.016874741995707154, + 0.016531227971427143, + 0.016707770992070436, + 0.015678408090025187 ], - "stddev": 0.00011160414647370428, - "iqr": 8.835001790430397e-05, - "noise_ratio": 0.006509206103154481, + "stddev": 0.00042446223746293047, + "iqr": 0.0005684989737346768, + "noise_ratio": 0.025857866964449815, "noisy": false }, { @@ -636,17 +758,19 @@ "library": "sqlspec", "library_key": "sqlspec", "scenario": "observability_select", - "time": 0.035870573003194295, - "times": [ - 0.03587337300996296, - 0.035870573003194295, - 0.03573423300986178, - 0.036352101000375114, - 0.03557527500379365 - ], - "stddev": 0.0002903064358924952, - "iqr": 0.00013914000010117888, - "noise_ratio": 0.008093164161794774, + "time": 0.029671795084141195, + "times": [ + 0.029671795084141195, + 0.029835711931809783, + 0.030337580014020205, + 0.02892933296971023, + 0.029016419081017375, + 0.02880434098187834, + 0.02968790999148041 + ], + "stddev": 0.0005650660600362936, + "iqr": 0.0007889349362812936, + "noise_ratio": 0.01904387848574442, "noisy": false } ] diff --git a/tools/perf_baselines/subsystems.json b/tools/perf_baselines/subsystems.json index 832e37c05..012eacfa6 100644 --- a/tools/perf_baselines/subsystems.json +++ b/tools/perf_baselines/subsystems.json @@ -12,6 +12,8 @@ "prepare_driver_parameters (tuple)", "prepare_driver_parameters (dict)", "_driver_parameters (3 params)", + "_check_declared_parameters (2 declared)", + "_check_declared_parameters (none declared)", "SQLResult.__init__() - DML", "SQLResult.__init__() - SELECT (100 rows)", "StorageRegistry.get() - cached alias", @@ -40,7 +42,7 @@ "metadata": { "iterations": 5000, "profile": false, - "timestamp": "2026-10-13T21:39:22+0000", + "timestamp": "2026-11-17T16:13:19+0000", "warmup": 100 }, "results": [ @@ -48,289 +50,305 @@ "description": "Time SQL('INSERT INTO test (value) VALUES (?)')", "iterations": 5000, "name": "SQL() construction (no params)", - "ops_per_sec": 913719.3186319757, - "time_per_op_us": 1.0944279929390177, - "total_time": 0.005437102998257615 + "ops_per_sec": 1201103.2044950407, + "time_per_op_us": 0.832567922770977, + "total_time": 0.00663077284116298 }, { "description": "Time SQL('INSERT ...', ('hello',))", "iterations": 5000, "name": "SQL() construction (with params)", - "ops_per_sec": 626649.6571693261, - "time_per_op_us": 1.5957879950292408, - "total_time": 0.008684266998898238 + "ops_per_sec": 778292.4687715697, + "time_per_op_us": 1.2848640326410532, + "total_time": 0.006465306039899588 }, { "description": "Time SQL('SELECT ... WHERE ...', param)", "iterations": 5000, "name": "SQL() construction (SELECT + param)", - "ops_per_sec": 637129.3471429019, - "time_per_op_us": 1.5695400070399046, - "total_time": 0.007883617014158517 + "ops_per_sec": 767238.8989332351, + "time_per_op_us": 1.3033750001341105, + "total_time": 0.006807590136304498 }, { "description": "Convert same-shape UPDATE builders while parameter values vary", "iterations": 5000, "name": "QueryBuilder.to_statement() - same shape varying values", - "ops_per_sec": 4120.919565822979, - "time_per_op_us": 242.66428500413895, - "total_time": 1.2694333770195954 + "ops_per_sec": 6152.492263345013, + "time_per_op_us": 162.53575903829187, + "total_time": 0.8162630000151694 }, { "description": "Compile simple INSERT statement", "iterations": 5000, "name": "SQL.compile() - INSERT", - "ops_per_sec": 161767.66763378118, - "time_per_op_us": 6.1817050009267405, - "total_time": 0.030903736987966113 + "ops_per_sec": 184180.85385350234, + "time_per_op_us": 5.4294459987431765, + "total_time": 0.02760164102073759 }, { "description": "Compile simple SELECT statement", "iterations": 5000, "name": "SQL.compile() - SELECT", - "ops_per_sec": 162352.42156694084, - "time_per_op_us": 6.159440003102645, - "total_time": 0.03080166499421466 + "ops_per_sec": 179368.7864529638, + "time_per_op_us": 5.575106013566256, + "total_time": 0.028748748125508428 }, { "description": "Compile complex JOIN with multiple params", "iterations": 5000, "name": "SQL.compile() - complex JOIN", - "ops_per_sec": 147818.3926397272, - "time_per_op_us": 6.76505800220184, - "total_time": 0.033716250996803865 + "ops_per_sec": 164567.60761802475, + "time_per_op_us": 6.076529971323907, + "total_time": 0.030427350779064 }, { "description": "Cache lookup with cache miss (early return, no execution)", "iterations": 5000, "name": "QC _cached_execution() - cache miss", - "ops_per_sec": 3329193.936325096, - "time_per_op_us": 0.3003730089403689, - "total_time": 0.0015094660047907382 + "ops_per_sec": 6178902.227299015, + "time_per_op_us": 0.1618410460650921, + "total_time": 0.0008515319786965847 }, { "description": "Full stmt_cache lookup -> prepare -> execute cycle", "iterations": 5000, "name": "QC _cached_execution() - full cycle", - "ops_per_sec": 48080.33261572494, - "time_per_op_us": 20.79852500173729, - "total_time": 0.10446592500375118 + "ops_per_sec": 47818.61077093518, + "time_per_op_us": 20.912359934300184, + "total_time": 0.1061803309712559 }, { "description": "Prepare a single positional parameter tuple", "iterations": 5000, "name": "prepare_driver_parameters (tuple)", - "ops_per_sec": 1000033.0009910618, - "time_per_op_us": 0.9999670000979677, - "total_time": 0.005002377001801506 + "ops_per_sec": 1239212.8287058696, + "time_per_op_us": 0.8069638861343265, + "total_time": 0.004041154868900776 }, { "description": "Prepare a single named parameter dict", "iterations": 5000, "name": "prepare_driver_parameters (dict)", - "ops_per_sec": 854326.9969933727, - "time_per_op_us": 1.170511997770518, - "total_time": 0.005851471985806711 + "ops_per_sec": 943560.0265824522, + "time_per_op_us": 1.0598159860819578, + "total_time": 0.005572059890255332 }, { "description": "Format a 3-element positional parameter set", "iterations": 5000, "name": "_driver_parameters (3 params)", - "ops_per_sec": 635211.826417988, - "time_per_op_us": 1.574277994222939, - "total_time": 0.007899189979070798 + "ops_per_sec": 791355.2935024629, + "time_per_op_us": 1.263654907234013, + "total_time": 0.006361614912748337 + }, + { + "description": "Validate a named statement carrying two parameter declarations", + "iterations": 5000, + "name": "_check_declared_parameters (2 declared)", + "ops_per_sec": 797466.5984745527, + "time_per_op_us": 1.2539710151031613, + "total_time": 0.00630593579262495 + }, + { + "description": "Short-circuit declared-parameter validation when no declarations exist", + "iterations": 5000, + "name": "_check_declared_parameters (none declared)", + "ops_per_sec": 10363765.625094257, + "time_per_op_us": 0.09649002458900213, + "total_time": 0.0004958219360560179 }, { "description": "Construct SQLResult for an INSERT operation", "iterations": 5000, "name": "SQLResult.__init__() - DML", - "ops_per_sec": 1198160.0977602866, - "time_per_op_us": 0.834613005281426, - "total_time": 0.004188569990219548 + "ops_per_sec": 1479640.2521023024, + "time_per_op_us": 0.6758399540558457, + "total_time": 0.00339864706620574 }, { "description": "Construct SQLResult for a SELECT with 100 rows", "iterations": 5000, "name": "SQLResult.__init__() - SELECT (100 rows)", - "ops_per_sec": 1097976.8669275937, - "time_per_op_us": 0.910766000743024, - "total_time": 0.004547191012534313 + "ops_per_sec": 1445101.2280089331, + "time_per_op_us": 0.6919930456206203, + "total_time": 0.003463471308350563 }, { "description": "Resolve a cached storage alias through the registry hot path", "iterations": 5000, "name": "StorageRegistry.get() - cached alias", - "ops_per_sec": 980211.4838925864, - "time_per_op_us": 1.020188006805256, - "total_time": 0.005175599013455212 + "ops_per_sec": 1088629.7889144707, + "time_per_op_us": 0.9185859235003591, + "total_time": 0.004609520896337926 }, { "description": "Encode rows and route them through the sync local storage pipeline", "iterations": 5000, "name": "SyncStoragePipeline.write_rows() - local jsonl", - "ops_per_sec": 5791.079632721365, - "time_per_op_us": 172.6793730049394, - "total_time": 0.8718435269984184 + "ops_per_sec": 7153.684450326693, + "time_per_op_us": 139.7881059674546, + "total_time": 0.7009224889334291 }, { "description": "Decode JSONL bytes through the Arrow boundary helper", "iterations": 5000, "name": "_decode_arrow_payload() - jsonl", - "ops_per_sec": 6869.321496958173, - "time_per_op_us": 145.57478499773424, - "total_time": 0.7582230610132683 + "ops_per_sec": 8751.714444368226, + "time_per_op_us": 114.26332592964172, + "total_time": 0.5687030248809606 }, { "description": "Enter/exit SqliteCursor context manager", "iterations": 5000, "name": "SqliteCursor context manager", - "ops_per_sec": 1128520.4166203288, - "time_per_op_us": 0.8861160022206604, - "total_time": 0.004822624992812052 + "ops_per_sec": 2409714.1653278484, + "time_per_op_us": 0.41498697828501463, + "total_time": 0.002076625940389931 }, { "description": "Raw sqlite3 cursor() + close() for comparison", "iterations": 5000, "name": "raw sqlite3 cursor create+close", - "ops_per_sec": 5109861.88069036, - "time_per_op_us": 0.19570000586099923, - "total_time": 0.0009786640148377046 + "ops_per_sec": 6100870.813254403, + "time_per_op_us": 0.16391102690249681, + "total_time": 0.000823984039016068 }, { "description": "Full sqlspec execute() including QC, dispatch, result", "iterations": 5000, "name": "session.execute() - full path", - "ops_per_sec": 46879.57367445652, - "time_per_op_us": 21.331252006348222, - "total_time": 0.10676726199744735 + "ops_per_sec": 37107.59496044093, + "time_per_op_us": 26.94866107776761, + "total_time": 0.13498832704499364 }, { "description": "Raw sqlite3 execute for comparison baseline", "iterations": 5000, "name": "raw sqlite3 conn.execute()", - "ops_per_sec": 62133.80282324967, - "time_per_op_us": 16.09429898962844, - "total_time": 0.0820974859962007 + "ops_per_sec": 7125.084335325993, + "time_per_op_us": 140.3492159442976, + "total_time": 0.7125933900242671 }, { "description": "Pure-Python driver overhead on warm fast-path SELECT with 1-tuple param", "iterations": 5000, "name": "stub driver: fast-path execute(SELECT, tuple)", - "ops_per_sec": 199214.29866762075, - "time_per_op_us": 5.019720003474504, - "total_time": 0.02853145000699442 + "ops_per_sec": 251303.07317035782, + "time_per_op_us": 3.9792589377611876, + "total_time": 0.02121697086840868 }, { "description": "Pure-Python driver overhead on warm fast-path INSERT with 1-tuple param", "iterations": 5000, "name": "stub driver: fast-path execute(INSERT, tuple)", - "ops_per_sec": 266848.62190703605, - "time_per_op_us": 3.747442999156192, - "total_time": 0.021824130992172286 + "ops_per_sec": 531526.7099155732, + "time_per_op_us": 1.8813729984685779, + "total_time": 0.00944000098388642 }, { "description": "Pure-Python driver overhead on warm fetch_one_or_none with 1-tuple param", "iterations": 5000, "name": "stub driver: fast-path fetch_one_or_none(SELECT, tuple)", - "ops_per_sec": 152632.9798998559, - "time_per_op_us": 6.551664002472535, - "total_time": 0.03708145199925639 + "ops_per_sec": 183708.5070005023, + "time_per_op_us": 5.443406058475375, + "total_time": 0.028725587879307568 }, { "description": "Pure-Python driver overhead on parameterless SELECT 1", "iterations": 5000, "name": "stub driver: parameterless execute(SELECT)", - "ops_per_sec": 238372.71592313284, - "time_per_op_us": 4.195110988803208, - "total_time": 0.021016020997194573 + "ops_per_sec": 286821.0051005778, + "time_per_op_us": 3.4864949993789196, + "total_time": 0.01737948413938284 }, { "description": "Pure-Python driver overhead on SELECT with scalar positional param", "iterations": 5000, "name": "stub driver: scalar-param execute(SELECT, int)", - "ops_per_sec": 191849.39328270138, - "time_per_op_us": 5.21242200920824, - "total_time": 0.026079890012624674 + "ops_per_sec": 246536.53261549614, + "time_per_op_us": 4.0561939822509885, + "total_time": 0.021314883837476373 }, { "description": "Pure-Python driver overhead on CTE + 2-join query with scalar param", "iterations": 2000, "name": "stub driver: complex CTE+2-join execute(SELECT, int)", - "ops_per_sec": 190476.18934846655, - "time_per_op_us": 5.250000031082891, - "total_time": 0.0105269009800395 + "ops_per_sec": 234381.81134180585, + "time_per_op_us": 4.266542673576623, + "total_time": 0.009350505075417459 }, { "description": "New driver instance per request executing simple SELECT", "iterations": 2000, "name": "stub driver: session-per-request simple SELECT", - "ops_per_sec": 150011.68233821707, - "time_per_op_us": 6.666147492069285, - "total_time": 0.013450521000777371 + "ops_per_sec": 160224.76015532407, + "time_per_op_us": 6.241232622414827, + "total_time": 0.012901977053843439 }, { "description": "New driver instance per request executing CTE + 2-join SELECT", "iterations": 2000, "name": "stub driver: session-per-request complex CTE SELECT", - "ops_per_sec": 148770.50532131383, - "time_per_op_us": 6.721762474626303, - "total_time": 0.013813848985591903 + "ops_per_sec": 156776.73685627396, + "time_per_op_us": 6.378497346304357, + "total_time": 0.012717136065475643 }, { "description": "Pure-Python driver overhead on execute_many with 100 parameter rows", "iterations": 2000, "name": "stub driver: execute_many 100 rows", - "ops_per_sec": 7149.427723435246, - "time_per_op_us": 139.87133497721516, - "total_time": 0.2836982590088155 + "ops_per_sec": 10038.25303758506, + "time_per_op_us": 99.61892734281719, + "total_time": 0.19985014689154923 }, { "description": "Current list-join placeholder assembly shape from ParameterConverter", "iterations": 5000, "name": "librt ParameterConverter baseline", - "ops_per_sec": 1007553.6268856531, - "time_per_op_us": 0.9925030026352033, - "total_time": 0.00521571101853624 + "ops_per_sec": 1247609.2765902565, + "time_per_op_us": 0.8015329949557781, + "total_time": 0.003999022883363068 }, { "description": "librt.strings.StringWriter candidate for ParameterConverter assembly", "iterations": 5000, "name": "librt ParameterConverter StringWriter", - "ops_per_sec": 1062117.9553103049, - "time_per_op_us": 0.941515012527816, - "total_time": 0.004716872004792094 + "ops_per_sec": 1249486.105579616, + "time_per_op_us": 0.800329027697444, + "total_time": 0.004018869949504733 }, { "description": "Current list-join statement assembly shape from StatementSplitter", "iterations": 5000, "name": "librt splitter join baseline", - "ops_per_sec": 341651.7555122614, - "time_per_op_us": 2.9269570077303797, - "total_time": 0.014837033028015867 + "ops_per_sec": 375775.9665247996, + "time_per_op_us": 2.661160076968372, + "total_time": 0.013330832007341087 }, { "description": "librt.strings.StringWriter candidate for StatementSplitter assembly", "iterations": 5000, "name": "librt splitter StringWriter", - "ops_per_sec": 302831.35208128975, - "time_per_op_us": 3.302167999208905, - "total_time": 0.020603441007551737 + "ops_per_sec": 303409.4757514113, + "time_per_op_us": 3.2958759693428874, + "total_time": 0.0173936088103801 }, { "description": "Current StringIO row assembly shape from psqlpy binary copy encoding", "iterations": 5000, "name": "librt psqlpy copy baseline", - "ops_per_sec": 60645.925178129015, - "time_per_op_us": 16.48915400437545, - "total_time": 0.08254189301806036 + "ops_per_sec": 66307.21955802142, + "time_per_op_us": 15.08131402079016, + "total_time": 0.07701656490098685 }, { "description": "librt.strings.StringWriter candidate for psqlpy binary copy encoding", "iterations": 5000, "name": "librt psqlpy copy StringWriter", - "ops_per_sec": 90935.46218894077, - "time_per_op_us": 10.99681000050623, - "total_time": 0.05711644601251464 + "ops_per_sec": 86773.67135845075, + "time_per_op_us": 11.524232919327915, + "total_time": 0.05768734193406999 } ] } diff --git a/tools/scripts/bench_bigquery_storage.py b/tools/scripts/bench_bigquery_storage.py index f51039260..587442238 100644 --- a/tools/scripts/bench_bigquery_storage.py +++ b/tools/scripts/bench_bigquery_storage.py @@ -11,7 +11,7 @@ import json import sys import tempfile -from collections.abc import Callable, Iterator, Sequence +from collections.abc import Callable, Generator, Sequence from contextlib import AbstractContextManager, contextmanager from pathlib import Path from time import perf_counter @@ -154,7 +154,7 @@ def main(argv: Sequence[str] | None = None) -> int: local = urlparse(args.endpoint).hostname in {"localhost", "127.0.0.1", "::1"} @contextmanager - def factory(native: bool) -> Iterator[BigQueryDriver]: + def factory(native: bool) -> Generator[BigQueryDriver, None, None]: with Client( project=args.project, client_options={"api_endpoint": args.endpoint}, diff --git a/tools/scripts/bench_imports.py b/tools/scripts/bench_imports.py index 4f2843e24..48a9e0f7e 100644 --- a/tools/scripts/bench_imports.py +++ b/tools/scripts/bench_imports.py @@ -27,7 +27,7 @@ SCHEMA_VERSION = 1 CONFIG = "from sqlspec.adapters.sqlite import SqliteConfig; config = SqliteConfig(connection_config={'database': ':memory:'})" QUERY = "with config.provide_session() as session:\n assert session.execute('SELECT 1 AS value').data[0][0] == 1" -SERIALIZER = "from sqlspec.utils.serializers._json import encode_json" +SERIALIZER = "from sqlspec.utils.serializers._json import to_json" SCENARIOS: dict[str, list[tuple[str, str]]] = { "bare": [("import", "import sqlspec")], "metadata": [("metadata", "import sqlspec; version = sqlspec.__version__")], @@ -60,12 +60,12 @@ "from sqlspec import sql\nstatement = sql.select('1 AS value')\nwith config.provide_session() as session:\n assert session.execute(statement).data[0][0] == 1", ), ], - "primitive-json": [("import", SERIALIZER), ("first_encode", "encoded = encode_json({'value': 1})")], + "primitive-json": [("import", SERIALIZER), ("first_encode", "encoded = to_json({'value': 1})")], "asyncpg-uuid": [ ("import", SERIALIZER), ( "first_encode", - "from asyncpg.pgproto.pgproto import UUID\nencoded = encode_json(UUID('12345678-1234-5678-1234-567812345678'))", + "from asyncpg.pgproto.pgproto import UUID\nencoded = to_json(UUID('12345678-1234-5678-1234-567812345678'))", ), ], "migration-loader": [("config", CONFIG), ("first_loader", "loader = config.get_migration_loader()")], @@ -105,7 +105,7 @@ "migration-tracker-import": [("import", "from sqlspec.migrations.tracker import SyncMigrationTracker")], "event-hints-import": [("import", "from sqlspec.extensions.events import EventRuntimeHints")], "warm-query": [("config", CONFIG), ("first_query", QUERY)], - "warm-json": [("import", SERIALIZER), ("first_encode", "encoded = encode_json({'value': 1})")], + "warm-json": [("import", SERIALIZER), ("first_encode", "encoded = to_json({'value': 1})")], } OPTIONAL = { "aiosqlite-query": "aiosqlite", @@ -163,7 +163,7 @@ def child(scenario: str, root: str | None) -> dict[str, Any]: "preloaded_modules": sorted(before), } if scenario.startswith("warm-"): - code = compile(QUERY if scenario == "warm-query" else "encode_json({'value': 1})", "", "exec") + code = compile(QUERY if scenario == "warm-query" else "to_json({'value': 1})", "", "exec") warm_start = time.perf_counter_ns() for _ in range(1000): exec(code, namespace) # noqa: S102 diff --git a/tools/scripts/export_hashed_requirements.py b/tools/scripts/export_hashed_requirements.py index ffe851563..bfcaccd3e 100644 --- a/tools/scripts/export_hashed_requirements.py +++ b/tools/scripts/export_hashed_requirements.py @@ -34,19 +34,40 @@ "runtime": ("typing-extensions", "sqlglot", "mypy-extensions", "rich-click"), "build": ("hatch-mypyc", "hatchling", "typing-extensions", "sqlglot", "mypy-extensions", "rich-click"), "pgo": ("anyio", "pyarrow", "aiosqlite", "duckdb", "adbc-driver-sqlite", "adbc-driver-manager"), - "pgo-validate": ( - "anyio", - "pyarrow", - "oracledb", - "asyncpg", + "test": ("aiosqlite", "pymssql", "cloud-sql-python-connector", "google-cloud-alloydb-connector"), + "smoke": ("obstore", "pyarrow"), + "drivers": ( + "adbc-driver-manager", + "adbc-driver-postgresql", + "adbc-driver-sqlite", + "aiomysql", "aiosqlite", + "asyncmy", + "asyncpg", "duckdb", - "adbc-driver-sqlite", - "adbc-driver-manager", + "google-cloud-bigquery", + "google-cloud-spanner", + "mssql-python", + "mysql-connector-python", + "oracledb", + "psqlpy", + "psycopg", + "psycopg-binary", + "psycopg-pool", + "pyarrow", + "pymssql", + "pymysql", ), - "test": ("aiosqlite", "pymssql", "cloud-sql-python-connector", "google-cloud-alloydb-connector"), - "smoke": ("obstore", "pyarrow"), + "drivers-db2": ("ibm-db",), + "drivers-sqlanywhere": ("sqlanydb",), + "drivers-sqlanywhere-build": ("setuptools",), } +"""Root distributions exported per requirement profile. + +``drivers`` holds SDKs with a binary wheel for every Linux wheel-test target. ``ibm-db`` ships Linux +wheels for x86_64 only, so it has its own profile. ``sqlanydb`` ships as a source distribution, so it +has its own profile and ``drivers-sqlanywhere-build`` pins the backend that builds it. +""" @dataclass(frozen=True, slots=True) diff --git a/tools/scripts/mypyc_inventory.py b/tools/scripts/mypyc_inventory.py index ec8bb8bd8..6a0ac0b50 100644 --- a/tools/scripts/mypyc_inventory.py +++ b/tools/scripts/mypyc_inventory.py @@ -28,16 +28,16 @@ HOT_SURFACE_CLASSIFICATIONS: dict[str, dict[str, str]] = { "sqlspec/driver/_routing.py": { - "classification": "keep_interpreted", - "reason": "Generic routing facades retain typed Python context and streaming boundaries.", + "classification": "compile_now", + "reason": "SyncRoutingSession and AsyncRoutingSession compile natively with slotted async context helpers and allow_interpreted_subclasses=True.", }, "sqlspec/core/routing.py": { "classification": "compile_now", "reason": "Weighted selectors and passive health breaker compile with core.", }, "sqlspec/utils/routing.py": { - "classification": "keep_interpreted", - "reason": "Pure ASGI middleware and request routing contexts preserve Python callable boundaries.", + "classification": "compile_now", + "reason": "Pure stdlib routing directives, ASGI middleware, and HMAC cookie verification compile without framework imports.", }, "sqlspec/driver/_deferred.py": { "classification": "compile_now", @@ -68,8 +68,20 @@ "reason": "Teradata output and parameter coercion helpers.", }, "sqlspec/adapters/teradata/driver.py": { - "classification": "prove_separately", - "reason": "Public DB-API driver remains interpreted alongside sibling adapters.", + "classification": "compile_now", + "reason": "Public DB-API driver compiles cleanly with allow_interpreted_subclasses=True and direct submodule imports.", + }, + "sqlspec/adapters/sybase/driver.py": { + "classification": "compile_now", + "reason": "Sybase driver compiles cleanly with allow_interpreted_subclasses=True and direct submodule imports.", + }, + "sqlspec/adapters/sqlanywhere/driver.py": { + "classification": "compile_now", + "reason": "SQL Anywhere driver compiles cleanly with allow_interpreted_subclasses=True and direct submodule imports.", + }, + "sqlspec/driver/_session.py": { + "classification": "compile_now", + "reason": "Shared sync/async driver session lifecycle context managers compile natively with allow_interpreted_subclasses=True.", }, "sqlspec/service/_base.py": { "classification": "keep_interpreted", @@ -241,6 +253,10 @@ "classification": "compile_now", "reason": "Sybase and SQL Anywhere render helper functions.", }, + "sqlspec/extensions/events/_buffer.py": { + "classification": "compile_now", + "reason": "Typed event message buffer helper has no optional runtime imports.", + }, "sqlspec/extensions/events/_models.py": { "classification": "compile_now", "reason": "EventMessage has concrete datetime annotations and slot dataclass layout compatible with mypyc.", @@ -269,6 +285,10 @@ "classification": "keep_interpreted", "reason": "Base event queue store remains interpreted because adapter-specific event stores subclass it.", }, + "sqlspec/extensions/adk/_table_utils.py": { + "classification": "compile_now", + "reason": "Pure ADK table schema validation helper has no optional ADK runtime import.", + }, "sqlspec/extensions/adk/_types.py": { "classification": "compile_now", "reason": "ADK session/event record TypedDict definitions have no optional ADK runtime import.", @@ -285,6 +305,38 @@ "classification": "keep_interpreted", "reason": "Imports Google ADK models at module import time and keeps Pydantic model reconstruction interpreted.", }, + "sqlspec/extensions/starlette/_state.py": { + "classification": "compile_now", + "reason": "Unquoted Starlette extension state dataclass compiles without framework imports.", + }, + "sqlspec/extensions/starlette/_utils.py": { + "classification": "compile_now", + "reason": "Per-request Starlette session and tenant helpers compile without top-level framework imports.", + }, + "sqlspec/extensions/flask/_state.py": { + "classification": "compile_now", + "reason": "Unquoted Flask extension state dataclass compiles without framework imports.", + }, + "sqlspec/extensions/flask/_utils.py": { + "classification": "compile_now", + "reason": "Per-request Flask session and tenant helpers compile without top-level framework imports.", + }, + "sqlspec/extensions/flask/_portal.py": { + "classification": "compile_now", + "reason": "Slotted blocking portal session view compiles without top-level Flask imports.", + }, + "sqlspec/extensions/litestar/_utils.py": { + "classification": "compile_now", + "reason": "Per-request Litestar ASGI scope state and session lease helpers compile without runtime framework imports.", + }, + "sqlspec/extensions/sanic/_state.py": { + "classification": "compile_now", + "reason": "Unquoted Sanic extension state dataclass compiles without framework imports.", + }, + "sqlspec/extensions/sanic/_utils.py": { + "classification": "compile_now", + "reason": "Per-request Sanic session and tenant helpers compile without top-level framework imports.", + }, "sqlspec/extensions/fastapi/providers.py": { "classification": "compile_now", "reason": ( @@ -344,8 +396,8 @@ "reason": "Snowflake parameter and result conversion helpers.", }, "sqlspec/adapters/snowflake/driver.py": { - "classification": "prove_separately", - "reason": "Optional Snowflake driver subclasses and deferred exception handler stay interpreted.", + "classification": "compile_now", + "reason": "Snowflake driver compiles cleanly with allow_interpreted_subclasses=True and deferred pyarrow helpers.", }, "sqlspec/adapters/snowflake/config.py": { "classification": "helper_split_first", @@ -356,8 +408,12 @@ "reason": "Cross-module data dictionary inheritance stays interpreted.", }, "sqlspec/adapters/snowflake/migrations.py": { - "classification": "keep_interpreted", - "reason": "Snowflake migration tracker extends the interpreted migration tracker facade.", + "classification": "compile_now", + "reason": "Snowflake single-inheritance migration tracker compiles with allow_interpreted_subclasses=True.", + }, + "sqlspec/adapters/teradata/migrations.py": { + "classification": "compile_now", + "reason": "Teradata single-inheritance migration tracker compiles with allow_interpreted_subclasses=True.", }, "sqlspec/adapters/sqlanywhere/pool.py": { "classification": "compile_now", @@ -376,16 +432,16 @@ "reason": "Async pool runtime compiles without adapter driver exception-handler subclasses.", }, "sqlspec/adapters/sqlite/driver.py": { - "classification": "prove_separately", - "reason": "Native Arrow ingestion bypasses interpreted parameter-conversion overrides and breaks rollback on rejected batches.", + "classification": "compile_now", + "reason": "SQLite driver compiles cleanly with allow_interpreted_subclasses=True and interpreted subclass overrides.", }, "sqlspec/adapters/aiosqlite/driver.py": { - "classification": "prove_separately", - "reason": "Grouped code generation cannot resolve the inherited pending_exception slot through the interpreted driver facade.", + "classification": "compile_now", + "reason": "AIOSQLite driver compiles cleanly with direct submodule imports and awaited exception cleanup outside handlers.", }, "sqlspec/adapters/asyncpg/driver.py": { - "classification": "prove_separately", - "reason": "Grouped code generation cannot resolve the inherited pending_exception slot through the interpreted driver facade.", + "classification": "compile_now", + "reason": "AsyncPG driver compiles cleanly with direct submodule imports, slotted savepoints, and awaited cleanup outside handlers.", }, **{ f"sqlspec/adapters/{adapter}/tenant.py": { @@ -509,7 +565,6 @@ "sqlspec/extensions/adk/converters.py", "sqlspec/config.py", "sqlspec/core/_pagination.py", - "sqlspec/driver/_routing.py", }) @@ -619,9 +674,9 @@ def build_inventory(root: Path | None = None) -> dict[str, Any]: "adapter_driver_shells": { "count": len(adapter_drivers), "modules": adapter_drivers, - "status": "blocked", - "classification": "prove_separately", - "reason": "Driver shells need separate layout and dispatch verification: grouped builds lose inherited fields, and compiled SQLite bypasses an interpreted Arrow conversion override.", + "status": "compiled", + "classification": "compile_now", + "reason": "All 24 adapter driver shells compile directly with single inheritance, allow_interpreted_subclasses=True, and direct submodule imports.", }, "adapter_pool_runtimes": { "count": len(adapter_pools), diff --git a/tools/scripts/mypyc_smoke.py b/tools/scripts/mypyc_smoke.py index 5efc5d62b..ba2618da4 100644 --- a/tools/scripts/mypyc_smoke.py +++ b/tools/scripts/mypyc_smoke.py @@ -7,6 +7,7 @@ import importlib.util import inspect import json +import os import subprocess import sys import tempfile @@ -15,10 +16,119 @@ from pathlib import Path from typing import Any, NamedTuple -__all__ = ("SMOKE_IMPORTS", "SmokeImport", "is_compiled_module", "main", "run_construction_checks", "run_smoke") +__all__ = ( + "SDK_PROFILE_DEPENDENCIES", + "SMOKE_IMPORTS", + "SmokeImport", + "is_compiled_module", + "main", + "run_construction_checks", + "run_smoke", +) COMPILED_SUFFIXES: tuple[str, ...] = tuple(dict.fromkeys((*importlib.machinery.EXTENSION_SUFFIXES, ".so", ".pyd"))) +SDK_PROFILE_DEPENDENCIES: tuple[str, ...] = ( + "adbc_driver_manager", + "aiomysql", + "aiosqlite", + "asyncmy", + "asyncpg", + "duckdb", + "google.cloud.bigquery", + "google.cloud.spanner_v1", + "mssql_python", + "mysql.connector", + "oracledb", + "psqlpy", + "psycopg", + "psycopg_pool", + "pyarrow", + "pymssql", + "pymysql", + "sqlanydb", +) +"""Importable SDKs installed by the ``drivers`` and ``drivers-sqlanywhere`` requirement profiles. + +Every Linux wheel-test job installs these. SDKs without a wheel for every Linux target, such as +``ibm_db``, are required per job through ``--require-sdk``. +""" + +_BEHAVIOUR_PROBE_TIMEOUT_SECONDS = 30 + +_BEHAVIOUR_PROBE_PRELUDE = """ +import asyncio +import json +import os +import sys +from importlib.machinery import EXTENSION_SUFFIXES + +SCENARIO_TIMEOUT_SECONDS = 5.0 +failures = [] +loop_errors = [] + + +class ScenarioHung(Exception): + pass + + +def record_loop_error(loop, context): + loop_errors.append(repr(context.get("exception") or context.get("message"))) + + +async def settle(awaitable, label, *, cancel_when=None): + asyncio.get_running_loop().set_exception_handler(record_loop_error) + task = asyncio.ensure_future(awaitable) + if cancel_when is not None: + started = asyncio.ensure_future(cancel_when.wait()) + await asyncio.wait({task, started}, timeout=SCENARIO_TIMEOUT_SECONDS, return_when=asyncio.FIRST_COMPLETED) + started.cancel() + if task.done(): + return task + if not cancel_when.is_set(): + failures.append(f"{label}: statement did not start within {SCENARIO_TIMEOUT_SECONDS}s") + raise ScenarioHung(label) + task.cancel() + done, _ = await asyncio.wait({task}, timeout=SCENARIO_TIMEOUT_SECONDS) + if not done: + detail = f" after event loop error {loop_errors[-1]}" if loop_errors else "" + failures.append(f"{label}: task still pending after {SCENARIO_TIMEOUT_SECONDS}s{detail}") + raise ScenarioHung(label) + return task + + +def expect(task, label, expected): + if task.cancelled(): + actual = "CancelledError" + elif task.exception() is not None: + actual = type(task.exception()).__name__ + else: + actual = "result" + if actual != expected: + detail = "" if task.cancelled() or task.exception() is None else f": {task.exception()}" + failures.append(f"{label}: expected {expected}, got {actual}{detail}") + + +def report(*module_names): + compiled = all(sys.modules[name].__file__.endswith(tuple(EXTENSION_SUFFIXES)) for name in module_names) + print(json.dumps({"compiled": compiled, "failures": failures})) + sys.stdout.flush() + os._exit(1 if failures else 0) + + +def run_probe(main, *module_names): + async def guarded(): + try: + await main() + except ScenarioHung: + pass + except Exception as exc: + failures.append(f"probe raised {type(exc).__name__}: {exc}") + report(*module_names) + + asyncio.run(guarded()) +""" + class SmokeImport(NamedTuple): """A module and optional attribute required by the compiled wheel smoke.""" @@ -45,8 +155,8 @@ class SmokeImport(NamedTuple): SmokeImport("prometheus_observer", "sqlspec.extensions.prometheus._observer", "PrometheusStatementObserver", True), SmokeImport("async_bridge", "sqlspec.utils.sync_tools", "async_", True), SmokeImport("core_routing", "sqlspec.core.routing", "ReplicaSpec", True), - SmokeImport("routing_context", "sqlspec.utils.routing", "RoutingContext"), - SmokeImport("driver_routing", "sqlspec.driver._routing", "SyncRoutingSession"), + SmokeImport("routing_context", "sqlspec.utils.routing", "RoutingContext", True), + SmokeImport("driver_routing", "sqlspec.driver._routing", "SyncRoutingSession", True), SmokeImport("deferred_session", "sqlspec.driver._deferred", "DeferredAsyncSession", True), SmokeImport("core_statement", "sqlspec.core.statement", "SQL", True), SmokeImport("builder_select", "sqlspec.builder._select", "Select", True), @@ -111,6 +221,62 @@ class SmokeImport(NamedTuple): for adapter in ("asyncpg", "oracledb", "psqlpy", "psycopg") ), SmokeImport("adk_record_types", "sqlspec.extensions.adk._types", "StoredSession", True, "google.adk"), + SmokeImport("adk_table_utils", "sqlspec.extensions.adk._table_utils", None, True), + SmokeImport("events_buffer", "sqlspec.extensions.events._buffer", None, True), + SmokeImport("starlette_state", "sqlspec.extensions.starlette._state", "SQLSpecConfigState", True, "starlette"), + SmokeImport( + "starlette_utils", "sqlspec.extensions.starlette._utils", "get_or_create_db_session", True, "starlette" + ), + SmokeImport("flask_state", "sqlspec.extensions.flask._state", "FlaskConfigState", True, "flask"), + SmokeImport("flask_utils", "sqlspec.extensions.flask._utils", "get_or_create_db_session", True, "flask"), + SmokeImport("flask_portal", "sqlspec.extensions.flask._portal", "PortalSession", True, "flask"), + SmokeImport("litestar_utils", "sqlspec.extensions.litestar._utils", "request_session_scope", True, "litestar"), + SmokeImport("sanic_state", "sqlspec.extensions.sanic._state", "SanicConfigState", True, "sanic"), + SmokeImport("sanic_utils", "sqlspec.extensions.sanic._utils", "get_or_create_db_session", True, "sanic"), + SmokeImport( + "snowflake_migrations", + "sqlspec.adapters.snowflake.migrations", + "SnowflakeSyncMigrationTracker", + True, + "snowflake.connector", + ), + SmokeImport( + "teradata_migrations", + "sqlspec.adapters.teradata.migrations", + "TeradataSyncMigrationTracker", + True, + "teradatasql", + ), + SmokeImport("driver_session", "sqlspec.driver._session", "SyncDriverSessionContext", True), + *( + SmokeImport(f"{adapter}_driver", f"sqlspec.adapters.{adapter}.driver", None, True, dependency) + for adapter, dependency in ( + ("adbc", "adbc_driver_manager"), + ("aiomysql", "aiomysql"), + ("aiosqlite", "aiosqlite"), + ("arrow_odbc", "arrow_odbc"), + ("asyncmy", "asyncmy"), + ("asyncpg", "asyncpg"), + ("bigquery", "google"), + ("cockroach_asyncpg", "asyncpg"), + ("cockroach_psycopg", "psycopg"), + ("db2", "ibm_db"), + ("duckdb", "duckdb"), + ("mssql_python", "mssql_python"), + ("mysqlconnector", "mysql.connector"), + ("oracledb", "oracledb"), + ("psqlpy", "psqlpy"), + ("psycopg", "psycopg"), + ("pymssql", "pymssql"), + ("pymysql", "pymysql"), + ("snowflake", "snowflake.connector"), + ("spanner", "google"), + ("sqlanywhere", "sqlanydb"), + ("sqlite", None), + ("sybase", "pymssql"), + ("teradata", "teradatasql"), + ) + ), SmokeImport("migration_runner", "sqlspec.migrations.runner", "SyncMigrationRunner", True), SmokeImport("sqlite_type_converter", "sqlspec.adapters.sqlite.type_converter", "register_type_handlers", True), ) @@ -122,8 +288,18 @@ def is_compiled_module(module: Any) -> bool: return module_file.endswith(COMPILED_SUFFIXES) -def run_smoke(*, require_compiled: bool = False) -> list[dict[str, Any]]: - """Import the compiled-wheel smoke matrix and return per-entry results.""" +def run_smoke( + *, require_compiled: bool = False, require_sdk_profile: bool = False, required_sdks: Sequence[str] = () +) -> list[dict[str, Any]]: + """Import the compiled-wheel smoke matrix and return per-entry results. + + Args: + require_compiled: Fail when a compiled-surface module imports from Python source. + require_sdk_profile: Fail instead of skipping when a missing dependency is one of + ``SDK_PROFILE_DEPENDENCIES``. + required_sdks: Further SDK modules whose absence fails instead of skipping. + """ + required = _required_sdks(require_sdk_profile=require_sdk_profile, required_sdks=required_sdks) results: list[dict[str, Any]] = [] for entry in SMOKE_IMPORTS: result: dict[str, Any] = { @@ -151,6 +327,8 @@ def run_smoke(*, require_compiled: bool = False) -> list[dict[str, Any]]: if _is_missing_optional_dependency(missing_name, optional_dependency): if require_compiled and entry.require_compiled and not _has_compiled_module_file(entry.module): result["error"] = "compiled extension is missing from the installed package" + elif _is_required_sdk(optional_dependency, required): + result["error"] = f"required SDK profile dependency is missing: {missing_name}" else: result["skipped"] = True result["skip_reason"] = f"optional dependency missing: {optional_dependency}" @@ -162,8 +340,18 @@ def run_smoke(*, require_compiled: bool = False) -> list[dict[str, Any]]: return results -def run_construction_checks(*, require_compiled: bool = False) -> list[dict[str, Any]]: - """Run construction-time smoke checks for provider classes.""" +def run_construction_checks( + *, require_compiled: bool = False, require_sdk_profile: bool = False, required_sdks: Sequence[str] = () +) -> list[dict[str, Any]]: + """Run construction-time smoke checks for provider classes. + + Args: + require_compiled: Fail when a compiled-surface module imports from Python source. + require_sdk_profile: Fail instead of skipping an adapter whose missing dependency is one of + ``SDK_PROFILE_DEPENDENCIES``. + required_sdks: Further SDK modules whose absence fails instead of skipping. + """ + required = _required_sdks(require_sdk_profile=require_sdk_profile, required_sdks=required_sdks) return [ *[ _check_public_import_order(order, require_compiled=require_compiled) @@ -178,9 +366,11 @@ def run_construction_checks(*, require_compiled: bool = False) -> list[dict[str, _check_tenant_lease_boundary(require_compiled=require_compiled), _check_session_routing_boundary(require_compiled=require_compiled), _check_oracle_26ai_boundary(require_compiled=require_compiled), - _check_adapter_config_construction(), + _check_adapter_config_construction(required_sdks=required), + _check_adapter_driver_construction(require_compiled=require_compiled, required_sdks=required), _check_statement_sentinel_identity(require_compiled=require_compiled), _check_statement_cache_rebind(require_compiled=require_compiled), + _check_exception_causes(require_compiled=require_compiled), _check_obstore_arrow_reads(require_compiled=require_compiled), _check_pipeline_arrow_streaming(require_compiled=require_compiled), _check_service_subclasses(require_compiled=require_compiled), @@ -188,6 +378,20 @@ def run_construction_checks(*, require_compiled: bool = False) -> list[dict[str, _check_aiosqlite_exception_mapping(require_compiled=require_compiled), _check_fastapi_filter_construction(require_compiled=require_compiled), _check_litestar_filter_construction(require_compiled=require_compiled), + _check_sqlite_migration_lock(require_compiled=require_compiled), + _check_duckdb_migration_lock(require_compiled=require_compiled), + _check_deferred_session_dialect(require_compiled=require_compiled), + _check_aiosqlite_load_failure(require_compiled=require_compiled), + _check_asyncpg_execute_stack(require_compiled=require_compiled), + _check_psycopg_execute_stack(require_compiled=require_compiled), + *[ + _check_mysql_observed_tuple_rows(adapter, driver_name, require_compiled=require_compiled) + for adapter, driver_name in ( + ("pymysql", "PyMysqlDriver"), + ("aiomysql", "AiomysqlDriver"), + ("asyncmy", "AsyncmyDriver"), + ) + ], ] @@ -197,11 +401,36 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument( "--require-compiled", action="store_true", help="Fail when compiled-surface modules import from Python source." ) + parser.add_argument( + "--require-sdk-profile", + action="store_true", + help="Fail when an SDK installed by the Linux wheel-test driver profile is missing instead of skipping.", + ) + parser.add_argument( + "--require-sdk", + action="append", + default=[], + metavar="MODULE", + help="Fail when this SDK module is missing instead of skipping (repeatable).", + ) parser.add_argument("--json", action="store_true", help="Print machine-readable JSON output.") args = parser.parse_args(argv) - results = run_smoke(require_compiled=args.require_compiled) - results.extend(run_construction_checks(require_compiled=args.require_compiled)) + required = _required_sdks(require_sdk_profile=args.require_sdk_profile, required_sdks=args.require_sdk) + results = run_smoke( + require_compiled=args.require_compiled, + require_sdk_profile=args.require_sdk_profile, + required_sdks=args.require_sdk, + ) + if required: + results.append(_check_sdk_profile(required)) + results.extend( + run_construction_checks( + require_compiled=args.require_compiled, + require_sdk_profile=args.require_sdk_profile, + required_sdks=args.require_sdk, + ) + ) if args.json: json.dump({"results": results}, sys.stdout, indent=2, sort_keys=True) sys.stdout.write("\n") @@ -239,13 +468,33 @@ def _has_compiled_module_file(module_name: str) -> bool: ) +_OPTIONAL_COMPANIONS: dict[str, tuple[str, ...]] = { + "aiomysql": ("pymysql",), + "ibm_db": ("ibm_db_dbi",), + "psycopg": ("psycopg_pool",), +} + + +def _required_sdks(*, require_sdk_profile: bool, required_sdks: Sequence[str]) -> tuple[str, ...]: + """Return every SDK module whose absence is a failure for this run.""" + profile = SDK_PROFILE_DEPENDENCIES if require_sdk_profile else () + return tuple(dict.fromkeys((*profile, *required_sdks))) + + +def _is_required_sdk(dependency: str | None, required: Sequence[str]) -> bool: + """Return whether a dependency, or a submodule of it, is a required SDK module.""" + if not dependency: + return False + return any(sdk == dependency or sdk.startswith(f"{dependency}.") for sdk in required) + + def _is_missing_optional_dependency(missing_name: str, optional_dependency: str | None) -> bool: if optional_dependency is None or not missing_name: return False - return ( - missing_name == optional_dependency - or missing_name.startswith(f"{optional_dependency}.") - or optional_dependency.startswith(f"{missing_name}.") + candidates = (optional_dependency, *_OPTIONAL_COMPANIONS.get(optional_dependency, ())) + return any( + missing_name == dep or missing_name.startswith(f"{dep}.") or dep.startswith(f"{missing_name}.") + for dep in candidates ) @@ -309,6 +558,57 @@ def _check_statement_sentinel_identity(*, require_compiled: bool = False) -> dic return result +def _check_exception_causes(*, require_compiled: bool = False) -> dict[str, Any]: + """Check that an error raised while handling another keeps its cause, or its suppression.""" + result = _new_smoke_result( + name="exception_causes", + module="sqlspec.utils.module_loader", + attribute="module_to_os_path", + compiled_required=require_compiled, + ) + try: + module_loader = importlib.import_module("sqlspec.utils.module_loader") + env_module = importlib.import_module("sqlspec.utils.env") + except Exception as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + return result + + result["imported"] = True + result["compiled"] = is_compiled_module(module_loader) and is_compiled_module(env_module) + if require_compiled and not result["compiled"]: + result["error"] = "module was imported from Python source, not a compiled extension" + return result + + failures: list[str] = [] + try: + module_loader.module_to_os_path("sqlspec_smoke_missing_package.child") + except ModuleNotFoundError as exc: + if not isinstance(exc.__cause__, ModuleNotFoundError): + failures.append("ModuleNotFoundError for a missing parent package has no direct cause") + else: + failures.append("resolving a module in a missing package did not raise") + + variable = "SQLSPEC_SMOKE_EXCEPTION_CAUSE" + previous = os.environ.get(variable) + os.environ[variable] = "not-a-number" + try: + env_module.get_env(variable, 0)() + except ValueError as exc: + if exc.__cause__ is not None or not exc.__suppress_context__: + failures.append("ValueError for an unparsable environment value still chains the conversion error") + else: + failures.append("an unparsable integer environment value did not raise") + finally: + if previous is None: + os.environ.pop(variable, None) + else: + os.environ[variable] = previous + + if failures: + result["error"] = "; ".join(failures) + return result + + def _check_statement_cache_rebind(*, require_compiled: bool = False) -> dict[str, Any]: """Exercise a cache-hit copy, parameter rebind, and expression snapshot.""" result = _new_smoke_result( @@ -1142,12 +1442,18 @@ def _check_fastapi_filter_construction(*, require_compiled: bool = False) -> dic return result -def _discover_adapter_config_classes(*, skipped: "list[str] | None" = None) -> "list[tuple[str, type[Any]]]": +def _discover_adapter_config_classes( + *, skipped: "list[str] | None" = None, required_sdks: Sequence[str] = () +) -> "list[tuple[str, type[Any]]]": """Return every database config class defined by an adapter ``config`` module. A class qualifies when it is defined in ``sqlspec.adapters..config`` and carries ``migration_tracker_type``, which excludes the pool, connection and extension helper classes that share those modules. + + Args: + skipped: Collects adapters whose optional SDK is missing; without it a missing SDK raises. + required_sdks: SDK modules whose absence raises instead of being collected. """ adapters_package = importlib.import_module("sqlspec.adapters") package_root = Path(next(iter(adapters_package.__path__))) @@ -1157,7 +1463,10 @@ def _discover_adapter_config_classes(*, skipped: "list[str] | None" = None) -> " try: module = importlib.import_module(module_name) except ModuleNotFoundError as exc: - if skipped is None or (exc.name or "").split(".")[0] not in { + missing_root = (exc.name or "").split(".")[0] + if _is_required_sdk(missing_root, required_sdks): + raise + if skipped is None or missing_root not in { "adbc_driver_manager", "aiomysql", "aiosqlite", @@ -1201,14 +1510,14 @@ def _adapter_config_construction_failure(qualified_name: str, config_cls: "type[ return None -def _check_adapter_config_construction() -> dict[str, Any]: +def _check_adapter_config_construction(*, required_sdks: Sequence[str] = ()) -> dict[str, Any]: """Construct every adapter database config with no arguments.""" result = _new_smoke_result( name="adapter_config_construction", module="sqlspec.adapters", attribute="migration_tracker_type" ) try: skipped: list[str] = [] - discovered = _discover_adapter_config_classes(skipped=skipped) + discovered = _discover_adapter_config_classes(skipped=skipped, required_sdks=required_sdks) result["skipped_adapters"] = skipped except Exception as exc: result["error"] = f"{type(exc).__name__}: {exc}" @@ -1228,6 +1537,84 @@ def _check_adapter_config_construction() -> dict[str, Any]: return result +class _PlaceholderConnection: + """Connection stand-in for constructing adapter drivers without a database.""" + + def adbc_get_info(self) -> dict[str, str]: + """Return driver metadata that resolves the ADBC dialect.""" + return {"vendor_name": "sqlite", "driver_name": "placeholder"} + + +def _adapter_driver_construction_failures(module: Any, driver_bases: "tuple[type[Any], ...]") -> list[str]: + """Return a failure description for each driver class in a module that cannot be constructed.""" + module_name = module.__name__ + driver_classes: list[type[Any]] = [ + candidate + for candidate in vars(module).values() + if inspect.isclass(candidate) and candidate.__module__ == module_name and issubclass(candidate, driver_bases) + ] + if not driver_classes: + return [f"{module_name}: no driver class found"] + failures: list[str] = [] + for driver_cls in driver_classes: + connection = _PlaceholderConnection() + try: + driver = driver_cls(connection=connection) + except Exception as exc: + failures.append(f"{module_name}.{driver_cls.__name__}: {exc!r}") + continue + if driver.connection is not connection: + failures.append(f"{module_name}.{driver_cls.__name__}: constructed driver lost its connection") + return failures + + +def _check_adapter_driver_construction( + *, require_compiled: bool = False, required_sdks: Sequence[str] = () +) -> dict[str, Any]: + """Construct every adapter driver class around a placeholder connection.""" + result = _new_smoke_result( + name="adapter_driver_construction", + module="sqlspec.adapters", + attribute="driver", + compiled_required=require_compiled, + ) + skipped: list[str] = [] + failures: list[str] = [] + compiled = True + try: + driver_bases = ( + importlib.import_module("sqlspec.driver._sync").SyncDriverAdapterBase, + importlib.import_module("sqlspec.driver._async").AsyncDriverAdapterBase, + ) + for entry in SMOKE_IMPORTS: + if not (entry.module.startswith("sqlspec.adapters.") and entry.module.endswith(".driver")): + continue + try: + module = importlib.import_module(entry.module) + except ModuleNotFoundError as exc: + optional_dependency = entry.optional_dependency + if not _is_missing_optional_dependency(exc.name or "", optional_dependency) or _is_required_sdk( + optional_dependency, required_sdks + ): + raise + skipped.append(f"{entry.module}: optional dependency missing: {optional_dependency}") + continue + compiled = compiled and is_compiled_module(module) + failures.extend(_adapter_driver_construction_failures(module, driver_bases)) + except Exception as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + return result + + result["imported"] = True + result["compiled"] = compiled + result["skipped_adapters"] = skipped + if failures: + result["error"] = "; ".join(failures) + elif require_compiled and not compiled: + result["error"] = "module was imported from Python source, not a compiled extension" + return result + + def _check_service_subclasses(*, require_compiled: bool = False) -> dict[str, Any]: """Exercise Python service subclasses against the compiled query and transaction runtime.""" result = _new_smoke_result( @@ -1434,6 +1821,7 @@ def _check_public_import_order(order: str, *, require_compiled: bool) -> dict[st imports[order] + """ import importlib +import pickle import sqlspec import sqlspec.builder as builder @@ -1451,6 +1839,13 @@ def _check_public_import_order(order: str, *, require_compiled: bool) -> dict[st assert isinstance(sql.select("value"), QueryBuilder) assert isinstance(sql.select("value"), Select) +class _AppSelect(Select): + pass + +assert issubclass(_AppSelect, Select) +assert isinstance(_AppSelect().select("1"), Select) +assert pickle.loads(pickle.dumps(Select)) is Select + from typing import get_type_hints from sqlspec.extensions.events import BaseEventQueueStore from sqlspec.migrations import SchemaEnsureResult @@ -2113,8 +2508,9 @@ def _check_session_routing_boundary(*, require_compiled: bool = False) -> dict[s if require_compiled: assert is_compiled_module(core) assert is_compiled_module(errors) - for module_name in ("sqlspec.utils.routing", "sqlspec.driver._routing", "sqlspec.service._base"): - assert not is_compiled_module(importlib.import_module(module_name)), module_name + assert is_compiled_module(context) + assert is_compiled_module(importlib.import_module("sqlspec.driver._routing")) + assert not is_compiled_module(importlib.import_module("sqlspec.service._base")), "sqlspec.service._base" assert statements.SQL("SELECT 1").is_read_only() assert not statements.SQL("SELECT nextval('sequence')", dialect="postgres").is_read_only() with tempfile.TemporaryDirectory() as directory: @@ -2174,11 +2570,679 @@ async def run_async() -> None: await config.close_pool() asyncio.run(run_async()) + if importlib.util.find_spec("asyncpg") is not None: + asyncpg_driver_module = importlib.import_module("sqlspec.adapters.asyncpg.driver") + + class _FakeAsyncpgConn: + async def fetch(self, sql: str, *args: Any) -> list[Any]: + return [] + + async def execute(self, sql: str, *args: Any) -> str: + return "UPDATE 1" + + async def executemany(self, sql: str, sets: Any) -> None: + return None + + def is_in_transaction(self) -> bool: + return False + + async def run_asyncpg_param_check() -> None: + drv = asyncpg_driver_module.AsyncpgDriver(connection=_FakeAsyncpgConn()) + assert await drv.select("SELECT $1::int", 1) == [] + exec_res = await drv.execute("UPDATE t SET x = $1", (1,)) + assert exec_res.num_rows == 1 + many_res = await drv.execute_many("INSERT INTO t VALUES ($1)", ((1,), (2,))) + assert many_res.num_rows == expected_count + + asyncio.run(run_asyncpg_param_check()) + if importlib.util.find_spec("duckdb") is not None: + duckdb_module = importlib.import_module("sqlspec.adapters.duckdb") + duck_cfg = duckdb_module.DuckDBConfig() + try: + with duck_cfg.provide_session() as duck_session: + duck_session.execute("CREATE TABLE items (id INTEGER)") + duck_res = duck_session.execute_many("INSERT INTO items VALUES (?)", ((1,), (2,))) + assert duck_res.num_rows == expected_count + assert duck_session.select_value("SELECT COUNT(*) FROM items") == expected_count + finally: + duck_cfg.close_pool() except Exception as exc: result["error"] = f"{type(exc).__name__}: {exc}" return result +def _check_sdk_profile(required_sdks: Sequence[str]) -> dict[str, Any]: + """Import every SDK module this run requires.""" + result = _new_smoke_result(name="sdk_profile", module="sqlspec.adapters", attribute=None) + result["imported"] = True + missing: list[str] = [] + for dependency in required_sdks: + try: + importlib.import_module(dependency) + except Exception as exc: + missing.append(f"{dependency} ({type(exc).__name__}: {exc})") + if missing: + result["error"] = f"sdk_profile: required SDKs are not importable: {'; '.join(missing)}" + return result + + +def _run_behaviour_probe( + name: str, + module: str, + script: str, + *, + require_compiled: bool, + dependencies: tuple[str, ...] = (), + scenarios: tuple[str, ...] = ("",), + arguments: tuple[str, ...] = (), +) -> dict[str, Any]: + """Run a behavioural probe script once per scenario, each in a fresh interpreter under a time limit. + + The script runs after ``_BEHAVIOUR_PROBE_PRELUDE`` with the scenario name as its first argument + and ends by calling ``report`` or ``run_probe``, which print the failures and exit without waiting + for unfinished tasks. Each scenario starts with clean interpreter state. + """ + result = _new_smoke_result(name=name, module=module, attribute=None, compiled_required=require_compiled) + for dependency in dependencies: + if importlib.util.find_spec(dependency) is None: + result["skipped"] = True + result["skip_reason"] = f"optional dependency missing: {dependency}" + return result + child_script = _BEHAVIOUR_PROBE_PRELUDE + script + if not require_compiled: + source_file = importlib.import_module("sqlspec").__file__ + if source_file is None: + result["error"] = f"{name}: SQLSpec package origin is unavailable" + return result + source_root = str(Path(source_file).resolve().parent.parent) + child_script = f"import sys; sys.path.insert(0, {source_root!r})\n" + child_script + failures: list[str] = [] + compiled = True + for scenario in scenarios: + try: + completed = subprocess.run( + [sys.executable, "-I", "-c", child_script, scenario, *arguments], + capture_output=True, + check=False, + text=True, + timeout=_BEHAVIOUR_PROBE_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + failures.append(f"{scenario or 'probe'} did not finish within {_BEHAVIOUR_PROBE_TIMEOUT_SECONDS}s") + break + try: + payload = json.loads(completed.stdout.splitlines()[-1]) + failures.extend(str(failure) for failure in payload["failures"]) + compiled = compiled and bool(payload["compiled"]) + except (ValueError, IndexError, KeyError, TypeError): + failures.append( + f"{scenario or 'probe'} exited with return code {completed.returncode}; " + f"stdout={completed.stdout!r}; stderr={completed.stderr!r}" + ) + result["imported"] = True + result["compiled"] = compiled + if failures: + result["error"] = f"{name}: {'; '.join(failures)}" + elif require_compiled and not compiled: + result["error"] = "module was imported from Python source, not a compiled extension" + return result + + +def _check_sqlite_migration_lock(*, require_compiled: bool = False) -> dict[str, Any]: + """Resolve the SQLite migration lock backend and hold the lock against a second migrator.""" + script = """ +import tempfile +from pathlib import Path + +from sqlspec.adapters.sqlite import SqliteConfig +from sqlspec.exceptions import SQLSpecError +from sqlspec.migrations.base import BaseMigrationTracker +from sqlspec.migrations.tracker import SyncMigrationTracker + +with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as root: + database = str(Path(root) / "lock.db") + first_config = SqliteConfig(connection_config={"database": database}) + second_config = SqliteConfig(connection_config={"database": database}) + try: + with first_config.provide_session() as first, second_config.provide_session() as second: + second.execute("PRAGMA busy_timeout = 200") + backend = BaseMigrationTracker._resolve_lock_backend(first) + if backend != "sqlite": + failures.append(f"SQLite session resolved lock backend {backend!r}, expected 'sqlite'") + with SyncMigrationTracker().acquire_lock(first): + try: + with SyncMigrationTracker().acquire_lock(second, timeout_seconds=1): + failures.append("second migrator acquired the SQLite lock while the first held it") + except SQLSpecError: + pass + try: + with SyncMigrationTracker().acquire_lock(second, timeout_seconds=1): + pass + except SQLSpecError as exc: + failures.append(f"released SQLite lock could not be acquired: {type(exc).__name__}: {exc}") + finally: + first_config.close_pool() + second_config.close_pool() + +report("sqlspec.migrations.base", "sqlspec.migrations.tracker", "sqlspec.adapters.sqlite.driver") +""" + return _run_behaviour_probe( + "sqlite_migration_lock", "sqlspec.migrations.tracker", script, require_compiled=require_compiled + ) + + +def _check_duckdb_migration_lock(*, require_compiled: bool = False) -> dict[str, Any]: + """Resolve the DuckDB migration lock backend from an open session.""" + script = """ +from sqlspec.adapters.duckdb import DuckDBConfig +from sqlspec.migrations.base import BaseMigrationTracker + +config = DuckDBConfig(connection_config={"database": ":memory:"}) +try: + with config.provide_session() as session: + backend = BaseMigrationTracker._resolve_lock_backend(session) + if backend != "lease": + failures.append(f"DuckDB session resolved lock backend {backend!r}, expected 'lease'") +finally: + config.close_pool() + +report("sqlspec.migrations.base", "sqlspec.adapters.duckdb.driver") +""" + return _run_behaviour_probe( + "duckdb_migration_lock", + "sqlspec.migrations.base", + script, + require_compiled=require_compiled, + dependencies=("duckdb",), + ) + + +def _check_deferred_session_dialect(*, require_compiled: bool = False) -> dict[str, Any]: + """Read ``dialect`` from a deferred session before its connection is opened.""" + script = """ +from sqlspec.adapters.aiosqlite import AiosqliteConfig +from sqlspec.driver._deferred import DeferredAsyncSession + +opened = [] + + +async def open_session(): + opened.append(True) + raise AssertionError("the request connection must stay closed") + + +session = DeferredAsyncSession( + AiosqliteConfig(connection_config={"database": ":memory:"}), open_session, lambda: None, "db_session" +) +try: + dialect = session.dialect +except Exception as exc: + failures.append(f"dialect before the connection opens raised {type(exc).__name__}: {exc}") +else: + if dialect != "sqlite": + failures.append(f"dialect before the connection opens was {dialect!r}, expected 'sqlite'") +if opened: + failures.append("reading dialect opened the request connection") + +report("sqlspec.driver._deferred", "sqlspec.adapters.aiosqlite.driver") +""" + return _run_behaviour_probe( + "deferred_session_dialect", + "sqlspec.driver._deferred", + script, + require_compiled=require_compiled, + dependencies=("aiosqlite",), + ) + + +def _check_aiosqlite_load_failure(*, require_compiled: bool = False) -> dict[str, Any]: + """Fail and cancel aiosqlite Arrow loads that run inside a cursor context.""" + script = """ +import tempfile +import time +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +from sqlspec.adapters.aiosqlite import AiosqliteConfig + +scenario = sys.argv[1] +operation, _, outcome = scenario.partition(" ") + + +async def main(): + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as directory: + rows = pa.table({"id": [1, 2] if outcome == "cancelled" else [1, 1], "name": ["a", "b"]}) + source = Path(directory) / "rows.parquet" + pq.write_table(rows, source) + config = AiosqliteConfig(connection_config={"database": str(Path(directory) / "load.db")}) + context = config.provide_session() + session = await context.__aenter__() + await session.execute_script("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)") + started = None + if outcome == "cancelled": + loop = asyncio.get_running_loop() + started = asyncio.Event() + seen = [] + + def mark_insert(): + if not seen: + seen.append(True) + loop.call_soon_threadsafe(started.set) + time.sleep(0.2) + return 1 + + await session.connection.create_function("probe_mark_insert", 0, mark_insert) + await session.execute_script( + "CREATE TRIGGER items_probe AFTER INSERT ON items BEGIN SELECT probe_mark_insert(); END" + ) + if operation == "load_from_storage": + load = session.load_from_storage("items", source, file_format="parquet") + else: + load = session.load_from_arrow("items", rows, drop_indexes=True) + task = await settle(load, scenario, cancel_when=started) + expect(task, scenario, "CancelledError" if outcome == "cancelled" else "UniqueViolationError") + await context.__aexit__(None, None, None) + await config.close_pool() + + +run_probe(main, "sqlspec.adapters.aiosqlite.driver", "sqlspec.storage._ingest", "sqlspec.driver._async") +""" + return _run_behaviour_probe( + "aiosqlite_load_failure", + "sqlspec.adapters.aiosqlite.driver", + script, + require_compiled=require_compiled, + dependencies=("aiosqlite", "pyarrow"), + scenarios=( + "load_from_storage duplicate-key", + "load_from_arrow duplicate-key", + "load_from_storage cancelled", + "load_from_arrow cancelled", + ), + ) + + +def _check_asyncpg_execute_stack(*, require_compiled: bool = False) -> dict[str, Any]: + """Fail and cancel asyncpg ``execute_stack`` inside transaction and savepoint scopes whose exits suspend.""" + script = """ +import sqlspec.adapters.asyncpg.driver as asyncpg_driver +from sqlspec.core.stack import StatementStack + +scenario = sys.argv[1] +scope, _, mode = scenario.partition(" ") + + +class FakeTransaction: + def __init__(self, connection): + self.connection = connection + + async def __aenter__(self): + self.connection.in_transaction = True + return self + + async def __aexit__(self, exc_type, exc, traceback): + await asyncio.sleep(0) + await asyncio.sleep(0.01) + self.connection.in_transaction = False + self.connection.log.append("COMMIT" if exc_type is None else "ROLLBACK") + + +class FakeConnection: + def __init__(self): + self.in_transaction = scope == "savepoint" + self.log = [] + self.started = asyncio.Event() + + def is_in_transaction(self): + return self.in_transaction + + def transaction(self): + return FakeTransaction(self) + + async def execute(self, sql, *args, **kwargs): + if sql.startswith(("SAVEPOINT", "ROLLBACK", "RELEASE")): + await asyncio.sleep(0) + await asyncio.sleep(0.01) + self.log.append(sql.split()[0]) + return "OK" + loop = asyncio.get_running_loop() + future = loop.create_future() + if mode == "committed": + loop.call_later(0.01, future.set_result, "INSERT 0 1") + elif mode == "failing": + loop.call_later(0.01, future.set_exception, RuntimeError("statement failed through a future")) + self.started.set() + return await future + + fetch = execute + + +async def main(): + connection = FakeConnection() + driver = asyncpg_driver.AsyncpgDriver(connection) + stack = StatementStack().push_execute("INSERT INTO items VALUES ($1)", (1,)) + task = await settle( + driver.execute_stack(stack, continue_on_error=scope == "savepoint"), + scenario, + cancel_when=connection.started if mode == "cancelled" else None, + ) + if mode == "cancelled": + expect(task, scenario, "CancelledError") + elif scope == "savepoint": + expect(task, scenario, "result") + if task.exception() is None: + errors = [type(entry.error).__name__ for entry in task.result()] + if errors != ["StackExecutionError"] or connection.log != ["SAVEPOINT", "ROLLBACK", "RELEASE"]: + failures.append(f"{scenario}: operation errors were {errors}, statements were {connection.log}") + else: + expect(task, scenario, "result" if mode == "committed" else "StackExecutionError") + expected_log = ["COMMIT"] if mode == "committed" else ["ROLLBACK"] + if connection.log != expected_log: + failures.append(f"{scenario}: transaction ended with {connection.log}, expected {expected_log}") + + +run_probe(main, "sqlspec.adapters.asyncpg.driver", "sqlspec.driver._async") +""" + return _run_behaviour_probe( + "asyncpg_execute_stack", + "sqlspec.adapters.asyncpg.driver", + script, + require_compiled=require_compiled, + dependencies=("asyncpg",), + scenarios=( + "transaction committed", + "transaction failing", + "transaction cancelled", + "savepoint failing", + "savepoint cancelled", + ), + ) + + +def _check_psycopg_execute_stack(*, require_compiled: bool = False) -> dict[str, Any]: + """Prepare psycopg stack parameters and fail or cancel a pipeline whose exit suspends.""" + script = """ +import contextlib + +import sqlspec.adapters.psycopg.driver as psycopg_driver +from sqlspec.core.stack import StatementStack + +scenario = sys.argv[1] + + +class FakeInfo: + transaction_status = 0 + + +class FakeCursor: + description = None + rowcount = 1 + statusmessage = "INSERT 0 1" + + def __init__(self, connection): + self.connection = connection + + def execute(self, sql, params=None): + self.connection.sent.append(params) + return self + + def fetchall(self): + return [] + + def close(self): + pass + + +class FakeAsyncCursor(FakeCursor): + async def execute(self, sql, params=None): + self.connection.sent.append(params) + if scenario == "parameters": + return self + loop = asyncio.get_running_loop() + future = loop.create_future() + if scenario == "pipeline failing": + loop.call_later(0.01, future.set_exception, RuntimeError("statement failed through a future")) + self.connection.started.set() + return await future + + async def fetchall(self): + return [] + + async def close(self): + pass + + +class FakePipeline: + def sync(self): + pass + + +class FakeAsyncPipeline: + async def sync(self): + pass + + +class FakeConnection: + def __init__(self): + self.autocommit = True + self.info = FakeInfo() + self.sent = [] + self.events = [] + + def cursor(self): + return FakeCursor(self) + + @contextlib.contextmanager + def pipeline(self): + yield FakePipeline() + + def commit(self): + self.events.append("commit") + + def rollback(self): + self.events.append("rollback") + + +class FakeAsyncConnection: + def __init__(self): + self.autocommit = True + self.info = FakeInfo() + self.sent = [] + self.events = [] + self.started = asyncio.Event() + + def cursor(self): + return FakeAsyncCursor(self) + + @contextlib.asynccontextmanager + async def pipeline(self): + try: + yield FakeAsyncPipeline() + finally: + await asyncio.sleep(0) + await asyncio.sleep(0.01) + self.events.append("pipeline-exit") + + async def commit(self): + self.events.append("commit") + + async def rollback(self): + self.events.append("rollback") + + async def set_autocommit(self, value): + self.autocommit = value + + +PARAMETER_CASES = ( + ("tuple parameters", "INSERT INTO items (id) VALUES (%s)", ((1,),), {}), + ("list parameters", "INSERT INTO items (id) VALUES (%s)", ([1],), {}), + ("scalar parameter", "INSERT INTO items (id) VALUES (%s)", (1,), {}), + ("named parameters", "INSERT INTO items (id) VALUES (:id)", (), {"id": 1}), + ("no parameters", "INSERT INTO items (id) VALUES (1)", (), {}), +) + + +def check_rows(label, results, connection): + rows = [entry.rows_affected for entry in results] + if rows != [1] or len(connection.sent) != 1 or connection.events[-1:] != ["commit"]: + failures.append(f"{label}: rows={rows}, statements sent={len(connection.sent)}, events={connection.events}") + + +async def check_parameters(): + for case, sql, args, kwargs in PARAMETER_CASES: + label = f"sync stack with {case}" + connection = FakeConnection() + try: + results = psycopg_driver.PsycopgSyncDriver(connection).execute_stack( + StatementStack().push_execute(sql, *args, **kwargs) + ) + except Exception as exc: + failures.append(f"{label}: {type(exc).__name__}: {exc.__cause__ or exc}") + else: + check_rows(label, results, connection) + for case, sql, args, kwargs in PARAMETER_CASES: + label = f"async stack with {case}" + connection = FakeAsyncConnection() + task = await settle( + psycopg_driver.PsycopgAsyncDriver(connection).execute_stack( + StatementStack().push_execute(sql, *args, **kwargs) + ), + label, + ) + if task.cancelled() or task.exception() is not None: + exc = None if task.cancelled() else task.exception() + failures.append(f"{label}: {type(exc).__name__}: {exc.__cause__ or exc}" if exc else f"{label}: cancelled") + else: + check_rows(label, task.result(), connection) + + +async def check_pipeline_exit(): + connection = FakeAsyncConnection() + task = await settle( + psycopg_driver.PsycopgAsyncDriver(connection).execute_stack( + StatementStack().push_execute("INSERT INTO items (id) VALUES (:id)", id=1) + ), + scenario, + cancel_when=connection.started if scenario == "pipeline cancelled" else None, + ) + expect(task, scenario, "CancelledError" if scenario == "pipeline cancelled" else "StackExecutionError") + if "pipeline-exit" not in connection.events: + failures.append(f"{scenario}: pipeline did not exit, events were {connection.events}") + + +run_probe( + check_parameters if scenario == "parameters" else check_pipeline_exit, + "sqlspec.adapters.psycopg.driver", + "sqlspec.adapters.psycopg.core", +) +""" + return _run_behaviour_probe( + "psycopg_execute_stack", + "sqlspec.adapters.psycopg.driver", + script, + require_compiled=require_compiled, + dependencies=("psycopg", "psycopg_pool"), + scenarios=("parameters", "pipeline failing", "pipeline cancelled"), + ) + + +def _check_mysql_observed_tuple_rows( + adapter: str, driver_name: str, *, require_compiled: bool = False +) -> dict[str, Any]: + """Repeat an observed SELECT on a MySQL driver whose cursor returns a tuple from ``fetchall``.""" + script = """ +import importlib + +from sqlspec.observability import ObservabilityConfig, ObservabilityRuntime + +adapter, driver_name = sys.argv[2:4] +driver_module = importlib.import_module(f"sqlspec.adapters.{adapter}.driver") +asynchronous = adapter != "pymysql" + + +class FakeCursor: + description = (("a", 3, None, None, None, None, None),) + rowcount = 2 + lastrowid = 0 + + def execute(self, sql, params=None): + return 2 + + def fetchall(self): + return ((1,), (2,)) + + def close(self): + pass + + +class FakeAsyncCursor(FakeCursor): + async def execute(self, sql, params=None): + return 2 + + async def fetchall(self): + return ((1,), (2,)) + + async def close(self): + pass + + +class FakeConnection: + server_status = 0 + + def cursor(self, *args, **kwargs): + return FakeAsyncCursor() if asynchronous else FakeCursor() + + def get_autocommit(self): + return True + + def get_transaction_status(self): + return False + + +class FakeAiomysqlConnection(FakeConnection): + async def cursor(self, *args, **kwargs): + return FakeAsyncCursor() + + +async def main(): + connection = FakeAiomysqlConnection() if adapter == "aiomysql" else FakeConnection() + driver = getattr(driver_module, driver_name)(connection=connection) + observed = [] + driver.attach_observability( + ObservabilityRuntime(ObservabilityConfig(statement_observers=(observed.append,)), config_name="smoke") + ) + for attempt in ("first", "second", "third"): + label = f"{attempt} observed SELECT" + try: + result = driver.execute("SELECT a FROM items WHERE id = %s", (1,)) + if asynchronous: + result = (await settle(result, label)).result() + rows = result.get_data() + except ScenarioHung: + raise + except Exception as exc: + failures.append(f"{label}: {type(exc).__name__}: {exc}") + continue + if rows != [{"a": 1}, {"a": 2}]: + failures.append(f"{label}: rows were {rows}") + if not observed: + failures.append("statement observer was not called") + + +run_probe(main, driver_module.__name__, "sqlspec.driver._async" if asynchronous else "sqlspec.driver._sync") +""" + return _run_behaviour_probe( + f"{adapter}_observed_tuple_rows", + f"sqlspec.adapters.{adapter}.driver", + script, + require_compiled=require_compiled, + dependencies=(adapter, "pymysql") if adapter == "aiomysql" else (adapter,), + arguments=(adapter, driver_name), + ) + + def _failed_results(results: list[dict[str, Any]]) -> list[dict[str, Any]]: return [ result diff --git a/tools/scripts/pgo_training.py b/tools/scripts/pgo_training.py index 9e37d13b3..c7eca66d5 100644 --- a/tools/scripts/pgo_training.py +++ b/tools/scripts/pgo_training.py @@ -4,16 +4,42 @@ This script runs against compiled modules and should not be packaged with the library. Run as: python tools/scripts/pgo_training.py + +Pass ``--strict`` when the environment is expected to provide every optional SDK the workloads use, +so that a skipped workload fails the run. """ +import argparse import sys import tempfile import time +from dataclasses import dataclass from datetime import UTC from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Sequence __all__ = ("main",) +_OPTIONAL_SDK_MODULES: frozenset[str] = frozenset({ + "adbc_driver_manager", + "adbc_driver_sqlite", + "aiosqlite", + "anyio", + "duckdb", + "pyarrow", +}) + + +@dataclass(slots=True) +class _TrainingRow: + """Dataclass row schema for exercising schema-mapped reads during PGO training.""" + + id: int + value: str + def _train_text_transforms() -> None: """Exercise text transformation hot paths.""" @@ -163,12 +189,7 @@ def _train_serialization() -> None: """Exercise JSON encode/decode paths.""" from datetime import datetime - from sqlspec.utils.serializers._json import ( - convert_date_to_iso, - convert_datetime_to_gmt_iso, - decode_json, - encode_json, - ) + from sqlspec.utils.serializers._json import convert_date_to_iso, convert_datetime_to_gmt_iso, from_json, to_json small_obj = {"key": "value", "num": 42} medium_obj = {f"field_{i}": i * 1.5 for i in range(20)} @@ -177,12 +198,12 @@ def _train_serialization() -> None: today = datetime.now(tz=UTC).date() for _ in range(10000): - s1 = encode_json(small_obj) - decode_json(s1) - s2 = encode_json(medium_obj) - decode_json(s2) - s3 = encode_json(list_obj) - decode_json(s3) + s1 = to_json(small_obj) + from_json(s1) + s2 = to_json(medium_obj) + from_json(s2) + s3 = to_json(list_obj) + from_json(s3) convert_datetime_to_gmt_iso(now) convert_date_to_iso(today) @@ -204,14 +225,17 @@ def _train_parameter_fingerprinting() -> None: def _train_sqlite_sync() -> None: - """Exercise sync driver via file-backed SQLite.""" - from sqlspec import SQLSpec + """Exercise sync driver paths via file-backed SQLite.""" + from sqlspec import SQLSpec, StatementStack from sqlspec.adapters.sqlite.config import SqliteConfig - create_sql = "CREATE TABLE test (value TEXT);" - insert_sql = "INSERT INTO test (value) VALUES (?);" - select_all = "SELECT * FROM test;" - select_by = "SELECT * FROM test WHERE value = ?;" + create_sql = "CREATE TABLE test (id INTEGER PRIMARY KEY, value TEXT);" + insert_sql = "INSERT INTO test (id, value) VALUES (?, ?);" + select_all = "SELECT id, value FROM test;" + select_page = "SELECT id, value FROM test ORDER BY id LIMIT 50;" + select_by = "SELECT id, value FROM test WHERE id = ?;" + select_value_sql = "SELECT value FROM test WHERE id = ?;" + select_named = "SELECT id, value FROM test WHERE id = :id AND value = :val;" with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: tmp_name = tmp.name @@ -222,32 +246,40 @@ def _train_sqlite_sync() -> None: with spec.provide_session(config) as session: session.execute(create_sql) - data = [(f"value_{i}",) for i in range(2000)] + data = [(i, f"value_{i}") for i in range(2000)] session.execute_many(insert_sql, data) - session.fetch(select_all) - - for i in range(10000): - session.fetch_one_or_none(select_by, (f"value_{i % 100}",)) + session.select(select_all) + session.select(select_page, schema_type=_TrainingRow) + session.execute_script("CREATE INDEX IF NOT EXISTS idx_test_value ON test (value);") + + for i in range(2000): + session.select(select_page) + session.select(select_by, i % 100, schema_type=_TrainingRow) + session.select_one(select_by, (i % 100,)) + session.select_one(select_by, (i % 100,), schema_type=_TrainingRow) + session.select_value(select_value_sql, (i % 100,)) + session.select_value_or_none(select_value_sql, (i % 100,)) + session.select_with_total(select_page) + session.fetch_one_or_none(select_by, i % 100) + session.execute(select_by, i % 100).scalar() + session.execute(select_named, {"id": i % 100, "val": f"value_{i % 100}"}).all() + + stack = ( + StatementStack() + .push_execute(select_by, (1,)) + .push_execute(select_value_sql, (2,)) + .push_execute(select_named, {"id": 3, "val": "value_3"}) + ) + for _ in range(500): + session.execute_stack(stack) + + for i in range(300): + with spec.provide_session(config) as pooled_session: + pooled_session.select_one(select_by, i % 100) config.close_pool() finally: - Path(tmp_name).unlink() - - with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: - tmp_name = tmp.name - - try: - config2 = SqliteConfig(connection_config={"database": tmp_name}) - with spec.provide_session(config2) as session: - session.execute(create_sql) - for i in range(500): - session.execute(insert_sql, (f"val_{i}",)) - for i in range(50000): - session.fetch_one_or_none(select_by, (f"val_{i % 500}",)) - - config2.close_pool() - finally: - Path(tmp_name).unlink() + Path(tmp_name).unlink(missing_ok=True) def _train_duckdb_sync() -> None: @@ -267,25 +299,31 @@ def _train_duckdb_sync() -> None: data = [(i, f"value_{i}") for i in range(2000)] session.execute_many(insert_sql, data) session.fetch(select_all) - for i in range(10000): + for i in range(5000): session.fetch_one_or_none(select_by, (i % 100,)) + for i in range(1000): + session.select_one(select_by, (i % 100,), schema_type=_TrainingRow) + session.select_value("SELECT value FROM test WHERE id = ?", i % 100) config.close_pool() def _train_aiosqlite_async() -> None: - """Exercise async driver via file-backed aiosqlite.""" + """Exercise async driver paths via file-backed aiosqlite.""" import asyncio import anyio async def _run() -> None: - from sqlspec import SQLSpec + from sqlspec import SQLSpec, StatementStack from sqlspec.adapters.aiosqlite.config import AiosqliteConfig - create_sql = "CREATE TABLE test (id INTEGER, value TEXT);" + create_sql = "CREATE TABLE test (id INTEGER PRIMARY KEY, value TEXT);" insert_sql = "INSERT INTO test (id, value) VALUES (?, ?);" - select_all = "SELECT * FROM test;" - select_by = "SELECT * FROM test WHERE id = ?;" + select_all = "SELECT id, value FROM test;" + select_page = "SELECT id, value FROM test ORDER BY id LIMIT 50;" + select_by = "SELECT id, value FROM test WHERE id = ?;" + select_value_sql = "SELECT value FROM test WHERE id = ?;" + select_named = "SELECT id, value FROM test WHERE id = :id AND value = :val;" with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: tmp_name = tmp.name @@ -297,16 +335,146 @@ async def _run() -> None: await session.execute(create_sql) data = [(i, f"value_{i}") for i in range(1000)] await session.execute_many(insert_sql, data) - await session.fetch(select_all) - for i in range(5000): - await session.fetch_one_or_none(select_by, (i % 100,)) + await session.select(select_all) + await session.execute_script("CREATE INDEX IF NOT EXISTS idx_async_value ON test (value);") + for i in range(1000): + await session.select(select_page) + await session.select(select_by, i % 100, schema_type=_TrainingRow) + await session.select_one(select_by, (i % 100,)) + await session.select_one(select_by, (i % 100,), schema_type=_TrainingRow) + await session.select_value(select_value_sql, (i % 100,)) + await session.select_value_or_none(select_value_sql, (i % 100,)) + await session.select_with_total(select_page) + await session.fetch_one_or_none(select_by, i % 100) + await session.execute(select_named, {"id": i % 100, "val": f"value_{i % 100}"}) + stack = ( + StatementStack() + .push_execute(select_by, (1,)) + .push_execute(select_value_sql, (2,)) + .push_execute(select_named, {"id": 3, "val": "value_3"}) + ) + for _ in range(300): + await session.execute_stack(stack) + for i in range(200): + async with spec.provide_session(config) as session: + _ = await session.select_value(select_value_sql, i % 100) await config.close_pool() finally: - await anyio.Path(tmp_name).unlink() + await anyio.Path(tmp_name).unlink(missing_ok=True) asyncio.run(_run()) +def _train_routing_and_loader() -> None: + """Exercise routing sessions, HMAC routing hints, file loader, and numeric Postgres compilation.""" + import asyncio + import sqlite3 + + from sqlspec.adapters.aiosqlite.config import AiosqliteConfig + from sqlspec.adapters.sqlite.config import SqliteConfig + from sqlspec.core.parameters import ParameterStyle, ParameterStyleConfig + from sqlspec.core.statement import SQL, StatementConfig + from sqlspec.driver import AsyncRoutingSession, SyncRoutingSession + from sqlspec.loader import SQLFileLoader + from sqlspec.utils.routing import ( + RoutingContext, + RoutingContextPayload, + apply_request_routing_hints, + sign_primary_pin, + ) + + pg_config = StatementConfig( + dialect="postgres", + parameter_config=ParameterStyleConfig( + default_parameter_style=ParameterStyle.NUMERIC, + supported_parameter_styles={ParameterStyle.NUMERIC, ParameterStyle.NAMED_COLON, ParameterStyle.QMARK}, + default_execution_parameter_style=ParameterStyle.NUMERIC, + supported_execution_parameter_styles={ParameterStyle.NUMERIC}, + ), + ) + for i in range(5000): + SQL( + "SELECT id, value FROM test WHERE id = :id AND value = :val", + {"id": i, "val": "item"}, + statement_config=pg_config, + ).compile() + + with tempfile.TemporaryDirectory() as tmp_dir: + base = Path(tmp_dir) + sql_file = base / "queries.sql" + sql_file.write_text( + "-- name: by_id\n-- slot: table\nSELECT id, value FROM /* slot: table */ WHERE id = :id;\n", + encoding="utf-8", + ) + loader = SQLFileLoader() + loader.add_fragment("common_cols", "id, value") + loader.add_named_sql("by_frag", "SELECT /* include: common_cols */ FROM test WHERE id = :id") + loader.load_sql(sql_file) + for _ in range(1000): + loader.get_sql("by_id", table="test").compile() + loader.get_sql("by_frag").compile() + + secret = b"pgo-training-secret-key" + causal_val = "lsn-1" + token = sign_primary_pin(secret, ttl_seconds=60.0, bind_key="default", causal_token=causal_val) + base_payload = RoutingContextPayload() + for _ in range(1000): + apply_request_routing_hints( + base_payload, + bind_key="default", + routing_header_value="replica", + cookie_header=f"sqlspec_pin={token}", + primary_pin_cookie="sqlspec_pin", + primary_pin_secret=secret, + ) + + primary_db = str(base / "primary.db") + replica_db = str(base / "replica.db") + for db_path, origin in ((primary_db, "primary"), (replica_db, "replica")): + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE notes (id INTEGER PRIMARY KEY, origin TEXT)") + conn.executemany("INSERT INTO notes VALUES (?, ?)", [(i, origin) for i in range(10)]) + conn.commit() + + sync_cfg = SqliteConfig(connection_config={"database": primary_db}, replicas=[{"database": replica_db}]) + try: + with SyncRoutingSession(sync_cfg) as rs: + for i in range(500): + rs.select("SELECT id, origin FROM notes WHERE id < ?", (5,)) + rs.select_one("SELECT id, origin FROM notes WHERE id = ?", (i % 10,)) + rs.select_one_or_none("SELECT id, origin FROM notes WHERE id = ?", (i % 10,)) + rs.select_value("SELECT origin FROM notes WHERE id = ?", (i % 10,)) + rs.fetch("SELECT id, origin FROM notes WHERE id < ?", (5,)) + rs.fetch_one("SELECT id, origin FROM notes WHERE id = ?", (i % 10,)) + with RoutingContext.primary(), rs.transaction(): + rs.execute("UPDATE notes SET origin = ? WHERE id = ?", ("updated", 1)) + with RoutingContext.replica(): + rs.select_value("SELECT origin FROM notes WHERE id = ?", (2,)) + with RoutingContext.auto(): + rs.select_value("SELECT origin FROM notes WHERE id = ?", (3,)) + finally: + sync_cfg.close_pool() + + async def _run_async_routing() -> None: + async_cfg = AiosqliteConfig(connection_config={"database": primary_db}, replicas=[{"database": replica_db}]) + try: + async with AsyncRoutingSession(async_cfg) as rs: + for i in range(300): + await rs.select("SELECT id, origin FROM notes WHERE id < ?", (5,)) + await rs.select_one("SELECT id, origin FROM notes WHERE id = ?", (i % 10,)) + await rs.select_one_or_none("SELECT id, origin FROM notes WHERE id = ?", (i % 10,)) + await rs.select_value("SELECT origin FROM notes WHERE id = ?", (i % 10,)) + await rs.fetch("SELECT id, origin FROM notes WHERE id < ?", (5,)) + await rs.fetch_one("SELECT id, origin FROM notes WHERE id = ?", (i % 10,)) + with RoutingContext.primary(): + async with rs.transaction(): + await rs.execute("UPDATE notes SET origin = ? WHERE id = ?", ("async-upd", 1)) + finally: + await async_cfg.close_pool() + + asyncio.run(_run_async_routing()) + + def _train_builder_and_where() -> None: """Exercise SQL builder construction, WHERE clauses, and compilation.""" from sqlspec.builder import Delete, Insert, Select, Update @@ -391,8 +559,34 @@ def _train_adbc_sqlite() -> None: Path(tmp_name).unlink() -def main() -> None: - """Run all PGO training workloads.""" +def _missing_optional_sdk(exc: ImportError) -> str | None: + """Return the missing module name when an import failed because an optional SDK is absent. + + Any other import failure, including one raised for a ``sqlspec`` module or by an installed SDK, + returns ``None``. + """ + if not isinstance(exc, ModuleNotFoundError) or not exc.name: + return None + return exc.name if exc.name.partition(".")[0] in _OPTIONAL_SDK_MODULES else None + + +def main(argv: "Sequence[str] | None" = None) -> None: + """Run all PGO training workloads. + + A workload whose optional SDK is not installed is skipped, or fails the run with ``--strict``. + Every other import failure propagates. + + Args: + argv: Command-line arguments, defaulting to ``sys.argv[1:]``. + + Raises: + RuntimeError: If ``--strict`` is set and a workload's optional SDK is not installed. + """ + parser = argparse.ArgumentParser(description="Run the PGO training workloads.") + parser.add_argument( + "--strict", action="store_true", help="Fail instead of skipping a workload whose optional SDK is not installed." + ) + args = parser.parse_args(argv) start = time.perf_counter() workloads = [ @@ -407,6 +601,7 @@ def main() -> None: ("sqlite_sync", _train_sqlite_sync), ("duckdb_sync", _train_duckdb_sync), ("aiosqlite_async", _train_aiosqlite_async), + ("routing_and_loader", _train_routing_and_loader), ("adbc_sqlite", _train_adbc_sqlite), ] @@ -415,8 +610,13 @@ def main() -> None: try: fn() except ImportError as exc: - elapsed = time.perf_counter() - t0 - print(f" {name}: skipped ({exc.name} not installed)") # noqa: T201 + missing = _missing_optional_sdk(exc) + if missing is None: + raise + if args.strict: + msg = f"workload {name} requires {missing}, which is not installed" + raise RuntimeError(msg) from exc + print(f" {name}: skipped ({missing} not installed)") # noqa: T201 continue elapsed = time.perf_counter() - t0 print(f" {name}: {elapsed:.2f}s") # noqa: T201