Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
b96534a
fix(security): harden migration guards, config discovery, multi-state…
cofin Nov 16, 2026
2fd5b67
perf(core): optimize statement caching, positional fast path, and res…
cofin Nov 16, 2026
695f546
perf(driver): streamline session lifecycle, async execution, and cros…
cofin Nov 16, 2026
5e21165
perf(mypyc): compile adapter drivers, session contexts, routing, and …
cofin Nov 17, 2026
0582cf9
perf(build): harden Linux PGO/LTO compiler flags, expand training cov…
cofin Nov 17, 2026
f0434f5
fix(mypyc): handle list and tuple parameter sequences in compiled asy…
cofin Nov 17, 2026
26f4cd3
fix(typing): restore session context driver types and harden mypyc sm…
cofin Nov 17, 2026
9b08960
fix(migrations): resolve the lock backend and deferred session dialec…
cofin Nov 17, 2026
c3c735f
fix(adapters): keep compiled async drivers from hanging on failed or …
cofin Nov 17, 2026
fdc5854
fix(sqlite,duckdb): fix failed aiosqlite loads, the rowid cache key, …
cofin Nov 17, 2026
2568f93
fix(driver): accept tuple fetch results, keep exception causes, and r…
cofin Nov 17, 2026
49b3be9
fix(storage,config): treat fsspec keys literally and discover config …
cofin Nov 17, 2026
8a7ad88
fix(observability,extensions): fire lifecycle hooks registered at run…
cofin Nov 17, 2026
35babf4
ci(build): exercise compiled drivers and compiled-only behaviour in t…
cofin Nov 17, 2026
b3ecc7d
docs(changelog): describe the BigQuery commit mode, Litestar lookup, …
cofin Nov 17, 2026
f512861
fix(migrations,storage): keep sticky shared directories off the impor…
cofin Nov 17, 2026
114c5ca
perf(sqlite): keep the SQL-keyed rowid eligibility lookup for stateme…
cofin Nov 17, 2026
b888495
perf(driver): cache the observability check per driver and refresh it…
cofin Nov 17, 2026
fe2dba0
perf(sqlite): check for a trailing comment only when the rowid lookup…
cofin Nov 17, 2026
7d11b13
fix(config): apply the ownership test to the opened pyproject.toml be…
cofin Nov 17, 2026
9a5340c
fix(migrations): log the directory that ends project root discovery a…
cofin Nov 17, 2026
f6ce198
fix: keep exception causes at raise sites in compiled modules
cofin Nov 17, 2026
d0618c0
perf(sqlite): stop assigning the read-only in_transaction flag on beg…
cofin Nov 17, 2026
66c1eeb
test(storage): use a fresh key for each create-only streaming check
cofin Nov 17, 2026
2ce3f53
perf(sqlite): close the statement cursor without a suppress context m…
cofin Nov 17, 2026
18b4c58
test(core): time the Snowflake splitter scaling check as the fastest …
cofin Nov 17, 2026
1b53225
ci: cover more adapter drivers and exception causes in the compiled w…
cofin Nov 17, 2026
7927ee5
chore(bench): refresh the SQLite and subsystem performance baselines
cofin Nov 17, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
16 changes: 8 additions & 8 deletions .github/workflows/pgo-validate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
enable-cache: false

- name: Export hashed requirements from uv.lock
run: python tools/scripts/export_hashed_requirements.py --profile cli --profile runtime --profile build --profile pgo-validate --output-dir .build-requirements
run: python tools/scripts/export_hashed_requirements.py --profile cli --profile runtime --profile build --profile pgo --profile test --output-dir .build-requirements

- name: Install cibuildwheel
run: python -m pip install --require-hashes -r .build-requirements/cli.txt
Expand All @@ -46,23 +46,23 @@ jobs:
PGO_DIR=/tmp/sqlspec-pgo &&
BUILD_DIR=/tmp/sqlspec-mypyc-build &&
mkdir -p "$PGO_DIR" "$BUILD_DIR" &&
CFLAGS="-fprofile-generate=$PGO_DIR"
CFLAGS="-DNDEBUG -fno-strict-overflow -fno-semantic-interposition -fprofile-generate=$PGO_DIR"
LDFLAGS="-Wl,-Bsymbolic-functions"
HATCH_BUILD_HOOKS_ENABLE=1
HATCH_MYPYC_BUILD_DIR="$BUILD_DIR"
MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1
pip install --no-build-isolation --no-deps "{package}" &&
pip install --require-hashes -r "{project}/.build-requirements/pgo-validate.txt" &&
python "{project}/tools/scripts/pgo_training.py" &&
pip install --require-hashes -r "{project}/.build-requirements/pgo.txt" &&
PYTHONSAFEPATH=1 python "{project}/tools/scripts/pgo_training.py" --strict &&
pip uninstall -y sqlspec &&
rm -rf "$BUILD_DIR/build" "$BUILD_DIR/tmp"

CIBW_ENVIRONMENT_LINUX: >-
HATCH_BUILD_HOOKS_ENABLE=1
HATCH_MYPYC_BUILD_DIR=/tmp/sqlspec-mypyc-build
MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1
CFLAGS="-fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch"
CFLAGS="-DNDEBUG -fno-strict-overflow -fno-semantic-interposition -flto=auto -fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch"
LDFLAGS="-Wl,-Bsymbolic-functions -flto=auto"

CIBW_BEFORE_TEST: 'pip install --require-hashes -r "{project}/.build-requirements/runtime.txt"'
CIBW_BEFORE_TEST: 'pip install --require-hashes -r "{project}/.build-requirements/runtime.txt" -r "{project}/.build-requirements/test.txt"'
CIBW_TEST_COMMAND: 'python "{project}/tools/scripts/mypyc_smoke.py" --require-compiled'

- name: List built wheels
Expand Down
37 changes: 27 additions & 10 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -143,32 +143,30 @@ jobs:
PGO_DIR=/tmp/sqlspec-pgo &&
BUILD_DIR=/tmp/sqlspec-mypyc-build &&
mkdir -p "$PGO_DIR" "$BUILD_DIR" &&
CFLAGS="-fprofile-generate=$PGO_DIR"
CFLAGS="-DNDEBUG -fno-strict-overflow -fno-semantic-interposition -fprofile-generate=$PGO_DIR"
LDFLAGS="-Wl,-Bsymbolic-functions"
HATCH_BUILD_HOOKS_ENABLE=1
HATCH_MYPYC_BUILD_DIR="$BUILD_DIR"
MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1
pip install --no-build-isolation --no-deps "{package}" &&
pip install --require-hashes -r "{project}/.build-requirements/pgo.txt" &&
python "{project}/tools/scripts/pgo_training.py" &&
PYTHONSAFEPATH=1 python "{project}/tools/scripts/pgo_training.py" --strict &&
pip uninstall -y sqlspec &&
rm -rf "$BUILD_DIR/build" "$BUILD_DIR/tmp"

CIBW_ENVIRONMENT_LINUX: >-
HATCH_BUILD_HOOKS_ENABLE=1
HATCH_MYPYC_BUILD_DIR=/tmp/sqlspec-mypyc-build
MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1
CFLAGS="-fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch"
CFLAGS="-DNDEBUG -fno-strict-overflow -fno-semantic-interposition -flto=auto -fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch"
LDFLAGS="-Wl,-Bsymbolic-functions -flto=auto"

# ── macOS (Clang): plain mypyc, no PGO ──
CIBW_ENVIRONMENT_MACOS: >-
HATCH_BUILD_HOOKS_ENABLE=1
MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1

# ── Windows (MSVC): plain mypyc, no PGO ──
CIBW_ENVIRONMENT_WINDOWS: >-
HATCH_BUILD_HOOKS_ENABLE=1
HATCH_MYPYC_BUILD_DIR=C:/b
MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1

- name: Upload wheel artifacts
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
Expand All @@ -184,7 +182,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest]
python-version: ["3.11", "3.13", "3.14", "3.15"]
python-version: ["3.11", "3.12", "3.13", "3.14", "3.15"]

runs-on: ${{ matrix.os }}
steps:
Expand Down Expand Up @@ -223,7 +221,7 @@ jobs:
shell: bash
env:
PYTHON_VERSION: ${{ matrix.python-version }}
run: uv run --no-project --python "$PYTHON_VERSION" python tools/scripts/export_hashed_requirements.py --profile runtime --profile smoke --output-dir .build-requirements
run: uv run --no-project --python "$PYTHON_VERSION" python tools/scripts/export_hashed_requirements.py --profile runtime --profile smoke --profile drivers --profile drivers-db2 --profile drivers-sqlanywhere --profile drivers-sqlanywhere-build --output-dir .build-requirements

- name: Download standard wheel artifacts
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
Expand All @@ -242,6 +240,10 @@ jobs:
# missing host wheel fails loudly instead of silently grabbing a stale
# PyPI release. Core and smoke dependencies use hash-pinned requirements.
# The version assert catches drift between pyproject.toml and the wheel.
# Linux also installs the hash-pinned driver SDK profiles so the smoke run
# imports those compiled adapter drivers instead of skipping them. sqlanydb
# ships only as a source distribution, so it builds with a hash-pinned
# backend; ibm_db ships Linux wheels for x86_64 only.
- name: Test standard wheel installation
shell: bash
env:
Expand All @@ -268,7 +270,22 @@ jobs:
else
echo "::warning::obstore or pyarrow binary wheel unavailable for Python ${PYTHON_VERSION}; the obstore smoke check will be skipped"
fi
uv run --no-project --python test-mypyc python -I tools/scripts/mypyc_smoke.py --require-compiled
smoke_args=(--require-compiled)
if [ "$RUNNER_OS" = "Linux" ]; then
uv pip install --no-config --python test-mypyc --require-hashes --only-binary :all: -r .build-requirements/drivers.txt
uv pip install --no-config --python test-mypyc --require-hashes --build-constraints .build-requirements/drivers-sqlanywhere-build.txt -r .build-requirements/drivers-sqlanywhere.txt
smoke_args+=(--require-sdk-profile)
if [ "$RUNNER_ARCH" = "X64" ]; then
grep -v '^[[:space:]]*--hash=' .build-requirements/drivers-db2.txt | sed -e 's/[[:space:]]*\\$//' > .build-requirements/drivers-db2-specs.txt
if uv pip install --no-config --python test-mypyc --dry-run --only-binary :all: -r .build-requirements/drivers-db2-specs.txt; then
uv pip install --no-config --python test-mypyc --require-hashes --only-binary :all: -r .build-requirements/drivers-db2.txt
smoke_args+=(--require-sdk ibm_db --require-sdk ibm_db_dbi)
else
echo "::warning::ibm_db binary wheel unavailable for Python ${PYTHON_VERSION}; the Db2 driver smoke check will be skipped"
fi
fi
fi
uv run --no-project --python test-mypyc python -I tools/scripts/mypyc_smoke.py "${smoke_args[@]}"

publish-release:
name: Publish to PyPI
Expand Down
31 changes: 22 additions & 9 deletions .github/workflows/test-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -313,32 +313,30 @@ jobs:
PGO_DIR=/tmp/sqlspec-pgo &&
BUILD_DIR=/tmp/sqlspec-mypyc-build &&
mkdir -p "$PGO_DIR" "$BUILD_DIR" &&
CFLAGS="-fprofile-generate=$PGO_DIR"
CFLAGS="-DNDEBUG -fno-strict-overflow -fno-semantic-interposition -fprofile-generate=$PGO_DIR"
LDFLAGS="-Wl,-Bsymbolic-functions"
HATCH_BUILD_HOOKS_ENABLE=1
HATCH_MYPYC_BUILD_DIR="$BUILD_DIR"
MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1
pip install --no-build-isolation --no-deps "{package}" &&
pip install --require-hashes -r "{project}/.build-requirements/pgo.txt" &&
python "{project}/tools/scripts/pgo_training.py" &&
PYTHONSAFEPATH=1 python "{project}/tools/scripts/pgo_training.py" --strict &&
pip uninstall -y sqlspec &&
rm -rf "$BUILD_DIR/build" "$BUILD_DIR/tmp"

CIBW_ENVIRONMENT_LINUX: >-
HATCH_BUILD_HOOKS_ENABLE=1
HATCH_MYPYC_BUILD_DIR=/tmp/sqlspec-mypyc-build
MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1
CFLAGS="-fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch"
CFLAGS="-DNDEBUG -fno-strict-overflow -fno-semantic-interposition -flto=auto -fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch"
LDFLAGS="-Wl,-Bsymbolic-functions -flto=auto"

# macOS + Windows: plain mypyc, no PGO. macOS PGO needs filter
# deps absent from BEFORE_BUILD; MSVC PGO isn't validated.
CIBW_ENVIRONMENT_MACOS: >-
HATCH_BUILD_HOOKS_ENABLE=1
MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1

CIBW_ENVIRONMENT_WINDOWS: >-
HATCH_BUILD_HOOKS_ENABLE=1
HATCH_MYPYC_BUILD_DIR=C:/b
MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1

- name: Upload wheel artifacts
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
Expand Down Expand Up @@ -389,7 +387,7 @@ jobs:
shell: bash
env:
PYTHON_VERSION: ${{ matrix.python-version }}
run: uv run --no-project --python "$PYTHON_VERSION" python tools/scripts/export_hashed_requirements.py --profile runtime --profile smoke --output-dir .build-requirements
run: uv run --no-project --python "$PYTHON_VERSION" python tools/scripts/export_hashed_requirements.py --profile runtime --profile smoke --profile drivers --profile drivers-db2 --profile drivers-sqlanywhere --profile drivers-sqlanywhere-build --output-dir .build-requirements

- name: Download standard wheel artifacts
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
Expand Down Expand Up @@ -430,7 +428,22 @@ jobs:
else
echo "::warning::obstore or pyarrow binary wheel unavailable for Python ${PYTHON_VERSION}; the obstore smoke check will be skipped"
fi
uv run --no-project --python test-mypyc python -I tools/scripts/mypyc_smoke.py --require-compiled
smoke_args=(--require-compiled)
if [ "$RUNNER_OS" = "Linux" ]; then
uv pip install --no-config --python test-mypyc --require-hashes --only-binary :all: -r .build-requirements/drivers.txt
uv pip install --no-config --python test-mypyc --require-hashes --build-constraints .build-requirements/drivers-sqlanywhere-build.txt -r .build-requirements/drivers-sqlanywhere.txt
smoke_args+=(--require-sdk-profile)
if [ "$RUNNER_ARCH" = "X64" ]; then
grep -v '^[[:space:]]*--hash=' .build-requirements/drivers-db2.txt | sed -e 's/[[:space:]]*\\$//' > .build-requirements/drivers-db2-specs.txt
if uv pip install --no-config --python test-mypyc --dry-run --only-binary :all: -r .build-requirements/drivers-db2-specs.txt; then
uv pip install --no-config --python test-mypyc --require-hashes --only-binary :all: -r .build-requirements/drivers-db2.txt
smoke_args+=(--require-sdk ibm_db --require-sdk ibm_db_dbi)
else
echo "::warning::ibm_db binary wheel unavailable for Python ${PYTHON_VERSION}; the Db2 driver smoke check will be skipped"
fi
fi
fi
uv run --no-project --python test-mypyc python -I tools/scripts/mypyc_smoke.py "${smoke_args[@]}"

verify-packages:
name: Verify package integrity
Expand Down
16 changes: 11 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -454,17 +454,21 @@ bench-analyze: ## Analyze saved profile dat

PGO_DIR := /tmp/sqlspec-pgo
PGO_BUILD_DIR := /tmp/sqlspec-mypyc-build
PGO_BASE_CFLAGS := -DNDEBUG -fno-strict-overflow -fno-semantic-interposition
PGO_BASE_LDFLAGS := -Wl,-Bsymbolic-functions

.PHONY: pgo-verify
pgo-verify: ## Verify CFLAGS reach the C compiler through hatch-mypyc
@echo "${INFO} Testing CFLAGS passthrough to C compiler..."
@CFLAGS="-DPGO_CANARY=1" uv run python -c "\
@CFLAGS="-DPGO_CANARY=1" LDFLAGS="-Wl,-Bsymbolic-functions" uv run --no-sync python -c "\
from distutils.sysconfig import customize_compiler; \
from distutils.ccompiler import new_compiler; \
cc = new_compiler(); customize_compiler(cc); \
cmd = ' '.join(cc.compiler_so); \
ld_cmd = ' '.join(cc.linker_so); \
assert 'PGO_CANARY' in cmd, f'CFLAGS not found in: {cmd}'; \
print('CFLAGS reach compiler: ' + cmd)" 2>/dev/null
assert '-Wl,-Bsymbolic-functions' in ld_cmd, f'LDFLAGS not found in: {ld_cmd}'; \
print('CFLAGS reach compiler: ' + cmd + ' | LDFLAGS: ' + ld_cmd)" 2>/dev/null
@echo "${OK} CFLAGS passthrough verified"

.PHONY: pgo-local
Expand All @@ -473,17 +477,19 @@ pgo-local: ## Run full three-stage PGO
@rm -rf $(PGO_DIR) $(PGO_BUILD_DIR) dist/
@mkdir -p $(PGO_DIR) $(PGO_BUILD_DIR)
@echo "${INFO} Stage 1: Building instrumented wheel..."
@CFLAGS="-fprofile-generate=$(PGO_DIR)" \
@CFLAGS="$(PGO_BASE_CFLAGS) -fprofile-generate=$(PGO_DIR)" \
LDFLAGS="$(PGO_BASE_LDFLAGS)" \
HATCH_MYPYC_BUILD_DIR=$(PGO_BUILD_DIR) \
HATCH_BUILD_HOOKS_ENABLE=1 uv build --wheel >/dev/null 2>&1
@echo "${OK} Instrumented wheel built"
@echo "${INFO} Stage 2: Running training workload..."
@uv pip install dist/*.whl --force-reinstall --no-deps >/dev/null 2>&1
@.venv/bin/python tools/scripts/pgo_training.py
@PYTHONSAFEPATH=1 .venv/bin/python tools/scripts/pgo_training.py
@echo "${OK} Training complete"
@rm -rf dist/ $(PGO_BUILD_DIR)/build $(PGO_BUILD_DIR)/tmp
@echo "${INFO} Stage 3: Building PGO-optimized wheel..."
@CFLAGS="-fprofile-use=$(PGO_DIR) -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch" \
@CFLAGS="$(PGO_BASE_CFLAGS) -flto=auto -fprofile-use=$(PGO_DIR) -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch" \
LDFLAGS="$(PGO_BASE_LDFLAGS) -flto=auto" \
HATCH_MYPYC_BUILD_DIR=$(PGO_BUILD_DIR) \
HATCH_BUILD_HOOKS_ENABLE=1 uv build --wheel >/dev/null 2>&1
@echo "${OK} PGO-optimized wheel built: $$(ls dist/*.whl)"
Expand Down
Loading
Loading