From 700e31d1f9dac65bddc19b34e1b0da8381581ec6 Mon Sep 17 00:00:00 2001 From: Tim Date: Thu, 19 Nov 2026 19:18:21 -0800 Subject: [PATCH 1/2] Verify Python activation and gate releases on installed consumers Use process-local bootstrap state and verify that stdlib sqlite3 opened the Prolly engine. Exercise both import orders, child-process reopen, uv-managed Python rejection, and source installs before publishing. Co-Authored-By: OpenAI Codex --- .github/workflows/wheels.yml | 49 ++++++++++++++++-- src/doltlite/_loader.py | 51 ++++++++++++------- tests/smoke.py | 97 +++++++++++++++++++++++++++++------- 3 files changed, 158 insertions(+), 39 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index f7e23b3..6b33c23 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -71,8 +71,6 @@ jobs: # runs inside the manylinux container, so build deps go into # CIBW_BEFORE_ALL_LINUX below. CIBW_BEFORE_BUILD: bash scripts/build-libdoltlite.sh - # Smoke test: install the wheel into a clean venv and run a - # script that verifies dolt_* functions are wired up. CIBW_TEST_COMMAND: "python {project}/tests/smoke.py" # cibuildwheel's macOS Python (python.org-style framework # build) statically links SQLite into _sqlite3.so — same case @@ -161,9 +159,54 @@ jobs: venv/bin/python tests/smoke.py venv/bin/python tests/macos_shim_smoke.py + smoke_linux: + name: Linux installed wheel and sdist + needs: [build_wheels, build_sdist] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v6 + with: + version: "0.9.15" + + - uses: actions/download-artifact@v4 + with: + name: wheels-ubuntu-latest-x86_64 + path: wheelhouse + + - uses: actions/download-artifact@v4 + with: + name: sdist + path: dist + + - name: Test the wheel with a uv venv using distro Python + run: | + venv="$RUNNER_TEMP/shared-sqlite" + uv venv --no-managed-python --python /usr/bin/python3 "$venv" + uv pip install --python "$venv/bin/python" wheelhouse/*.whl + "$venv/bin/python" tests/smoke.py + + - name: Test the wheel with uv's statically linked Python + run: | + venv="$RUNNER_TEMP/standalone" + uv python install 3.12.12 + uv venv --managed-python --python 3.12.12 "$venv" + uv pip install --python "$venv/bin/python" wheelhouse/*.whl + "$venv/bin/python" tests/smoke.py --reject-standalone + + - name: Install the sdist with an explicit matching native library + run: | + library="$RUNNER_TEMP/native-wheel" + /usr/bin/python3 -m zipfile -e wheelhouse/*.whl "$library" + venv="$RUNNER_TEMP/from-sdist" + uv venv --no-managed-python --python /usr/bin/python3 "$venv" + uv pip install --python "$venv/bin/python" dist/*.tar.gz + DOLTLITE_LIB="$library/doltlite/_lib/libdoltlite.so" "$venv/bin/python" tests/smoke.py + publish: name: Publish to PyPI - needs: [build_wheels, build_sdist, smoke_macos] + needs: [build_wheels, build_sdist, smoke_macos, smoke_linux] if: >- (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && inputs.publish) diff --git a/src/doltlite/_loader.py b/src/doltlite/_loader.py index 617f6d2..55f2e8b 100644 --- a/src/doltlite/_loader.py +++ b/src/doltlite/_loader.py @@ -24,6 +24,7 @@ """ from __future__ import annotations +from contextlib import closing import ctypes import hashlib import os @@ -36,6 +37,11 @@ _BOOTSTRAP_MARKER = "_DOLTLITE_BOOTSTRAPPED" +_SHARED_SQLITE_HINT = ( + "DoltLite needs a Python interpreter that dynamically links libsqlite3. " + "With uv, select one explicitly using uv venv --python /path/to/python." +) +_bootstrapped = False _PKG_LIB_DIR = Path(__file__).resolve().parent / "_lib" @@ -75,10 +81,6 @@ def libdoltlite_path() -> str: ) -def _is_bootstrapped() -> bool: - return os.environ.get(_BOOTSTRAP_MARKER) == "1" - - def _sqlite3_already_imported() -> bool: return "sqlite3" in sys.modules or "_sqlite3" in sys.modules @@ -89,32 +91,47 @@ def bootstrap() -> None: Idempotent: subsequent calls (and re-entrant calls after a re-exec) are no-ops. Raises `DoltliteLoadError` if the library can't be found or if a re-exec is needed but `sys.argv` does not name a script we - can replay (interactive shell, `python -c …`, REPL, notebook). + can replay (interactive shell, `python -c …`, REPL, notebook), or + when Python still resolves sqlite3 to another engine. """ - if _is_bootstrapped(): + global _bootstrapped + if _bootstrapped: return - lib = libdoltlite_path() + if os.environ.pop(_BOOTSTRAP_MARKER, None) != "1": + lib = libdoltlite_path() + if sys.platform != "darwin" and not _sqlite3_already_imported(): + ctypes.CDLL(lib, mode=ctypes.RTLD_GLOBAL) + else: + _reexec(lib) - if sys.platform != "darwin" and not _sqlite3_already_imported(): - ctypes.CDLL(lib, mode=ctypes.RTLD_GLOBAL) - os.environ[_BOOTSTRAP_MARKER] = "1" - return - - _require_replayable_argv() + _verify_engine() + _bootstrapped = True - env = dict(os.environ) - env[_BOOTSTRAP_MARKER] = "1" +def _reexec(lib: str) -> None: + _require_replayable_argv() + env = dict(os.environ, **{_BOOTSTRAP_MARKER: "1"}) if sys.platform == "darwin": env["DYLD_INSERT_LIBRARIES"] = _build_macos_shim(lib) else: existing = env.get("LD_PRELOAD", "").strip() env["LD_PRELOAD"] = f"{lib} {existing}".strip() - os.execvpe(sys.executable, [sys.executable, *sys.argv], env) +def _verify_engine() -> None: + import sqlite3 + + message = f"Could not activate DoltLite as Python's sqlite3 engine. {_SHARED_SQLITE_HINT}" + try: + with closing(sqlite3.connect(":memory:")) as db: + engine = db.execute("SELECT doltlite_engine()").fetchone() + except sqlite3.Error as error: + raise DoltliteLoadError(message) from error + if engine != ("prolly",): + raise DoltliteLoadError(message) + def _require_replayable_argv() -> None: """Bail out clearly if we can't safely re-exec the current process. @@ -171,7 +188,7 @@ def _detect_sqlite3_install_name() -> str: raise DoltliteLoadError( "Could not find a libsqlite3 dependency in " - f"{_sqlite3.__file__}. Output of otool -L:\n{out}" + f"{_sqlite3.__file__}. {_SHARED_SQLITE_HINT} Output of otool -L:\n{out}" ) diff --git a/tests/smoke.py b/tests/smoke.py index 3b8b0cf..88c2670 100644 --- a/tests/smoke.py +++ b/tests/smoke.py @@ -1,29 +1,88 @@ -"""Smoke test: after `import doltlite`, the stdlib sqlite3 module must -expose libdoltlite's SQL functions and virtual tables.""" -import doltlite -import sqlite3 +"""Exercise the installed package through stdlib sqlite3 in fresh processes.""" +from contextlib import closing +from importlib.metadata import version +import os +from pathlib import Path +import subprocess import sys +import tempfile -def main() -> int: - conn = sqlite3.connect(":memory:") +SCRIPT = Path(__file__).resolve() +ROWS = [(1, "café", b"\x00\xff", None), (2, "second", b"\x01", 2**63 - 1)] +MESSAGE = "file-backed smoke" - version = conn.execute("SELECT dolt_version()").fetchone() - print(f"dolt_version: {version}") - assert version and version[0].startswith("v"), f"unexpected dolt_version: {version}" - conn.execute("CREATE TABLE t(x INT)") - conn.execute("INSERT INTO t VALUES (1), (2), (3)") - conn.execute("SELECT dolt_commit('-A', '-m', 'smoke')") +def child(*args, env=None, cwd=None): + subprocess.run([sys.executable, str(SCRIPT), "--worker", *args], + env=env, cwd=cwd, check=True, timeout=30) - log = list(conn.execute("SELECT commit_hash, message FROM dolt_log")) - assert any(msg == "smoke" for _, msg in log), f"smoke commit missing from log: {log}" - print(f"dolt_log entries: {len(log)}") - print(f"libdoltlite path: {doltlite.libdoltlite_path()}") - print("OK") - return 0 +def consumer(mode, order, filename=None, commit=None): + if order == "sqlite3-first": + import sqlite3 + + try: + import doltlite + except RuntimeError as error: + if mode != "unsupported": + raise + assert type(error).__name__ == "DoltliteLoadError", repr(error) + assert "dynamically links libsqlite3" in str(error), str(error) + assert "uv venv --python" in str(error), str(error) + return + + if mode == "unsupported": + raise AssertionError(f"{order}: standalone Python unexpectedly accepted DoltLite") + + import sqlite3 + + package_version = version("doltlite") + expected_engine_version = "v" + package_version.split(".post", 1)[0] + assert doltlite.__version__ == package_version + if override := os.environ.get("DOLTLITE_LIB"): + assert Path(doltlite.libdoltlite_path()).resolve() == Path(override).resolve() + + if mode == "reopen": + assert Path(filename).is_file(), filename + with closing(sqlite3.connect(filename)) as db: + assert db.execute("SELECT doltlite_engine(), dolt_version()").fetchone() == ( + "prolly", expected_engine_version) + if mode == "write": + db.execute("CREATE TABLE t(id INTEGER PRIMARY KEY, label TEXT, blob BLOB, value INTEGER)") + db.executemany("INSERT INTO t VALUES (?, ?, ?, ?)", ROWS) + db.commit() + db.execute("BEGIN") + db.execute("UPDATE t SET label = 'rolled back'") + db.rollback() + (commit,) = db.execute("SELECT dolt_commit('-A', '-m', ?)", (MESSAGE,)).fetchone() + assert isinstance(commit, str) and commit, commit + + assert db.execute("SELECT * FROM t ORDER BY id").fetchall() == ROWS + assert db.execute("SELECT commit_hash FROM dolt_log WHERE message = ?", + (MESSAGE,)).fetchall() == [(commit,)] + + if mode == "write": + child("reopen", order, filename, commit) + + +def main(unsupported=False): + env = dict(os.environ) + for name in ("_DOLTLITE_BOOTSTRAPPED", "LD_PRELOAD", "DYLD_INSERT_LIBRARIES", "PYTHONPATH"): + env.pop(name, None) + with tempfile.TemporaryDirectory() as directory: + for order in ("doltlite-first", "sqlite3-first"): + if unsupported: + child("unsupported", order, env=env, cwd=directory) + else: + filename = str(Path(directory, f"{order}.db")) + child("write", order, filename, env=env, cwd=directory) + print(f"{order}: {'unsupported interpreter rejected' if unsupported else 'rows and history survived a child process'}") if __name__ == "__main__": - sys.exit(main()) + if sys.argv[1:2] == ["--worker"]: + consumer(*sys.argv[2:]) + else: + assert sys.argv[1:] in ([], ["--reject-standalone"]) + main(unsupported=bool(sys.argv[1:])) From ff2bc4617c2c5c2c3b044e15d13eb3c3434c6315 Mon Sep 17 00:00:00 2001 From: Tim Date: Thu, 19 Nov 2026 19:53:16 -0800 Subject: [PATCH 2/2] Keep the DoltLite shim out of macOS tool subprocesses Child consumers inherit DYLD_INSERT_LIBRARIES from their parents. Keep it out of otool and install_name_tool invocations so the loader can inspect and prepare a shim without injecting it into those tools. Co-Authored-By: OpenAI Codex --- src/doltlite/_loader.py | 23 +++++++++++++---------- tests/test_loader.py | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/doltlite/_loader.py b/src/doltlite/_loader.py index 55f2e8b..d82979d 100644 --- a/src/doltlite/_loader.py +++ b/src/doltlite/_loader.py @@ -88,8 +88,8 @@ def _sqlite3_already_imported() -> bool: def bootstrap() -> None: """Make libdoltlite the active SQLite engine for this interpreter. - Idempotent: subsequent calls (and re-entrant calls after a re-exec) - are no-ops. Raises `DoltliteLoadError` if the library can't be found + Subsequent calls in the same process are no-ops after verification. + Raises `DoltliteLoadError` if the library can't be found or if a re-exec is needed but `sys.argv` does not name a script we can replay (interactive shell, `python -c …`, REPL, notebook), or when Python still resolves sqlite3 to another engine. @@ -132,6 +132,7 @@ def _verify_engine() -> None: if engine != ("prolly",): raise DoltliteLoadError(message) + def _require_replayable_argv() -> None: """Bail out clearly if we can't safely re-exec the current process. @@ -165,21 +166,26 @@ def _require_replayable_argv() -> None: ) +def _run_macos_tool(*command: str) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env.pop("DYLD_INSERT_LIBRARIES", None) + return subprocess.run(command, env=env, check=True, capture_output=True, text=True) + + def _detect_sqlite3_install_name() -> str: """On macOS, return the libsqlite3 path that the active interpreter's `_sqlite3` extension was linked against.""" import _sqlite3 try: - out = subprocess.run( - ["otool", "-L", _sqlite3.__file__], - check=True, capture_output=True, text=True, - ).stdout + out = _run_macos_tool("otool", "-L", _sqlite3.__file__).stdout except FileNotFoundError as e: raise DoltliteLoadError( "otool is required on macOS to detect the active " "libsqlite3 path. Install Xcode Command Line Tools." ) from e + except subprocess.CalledProcessError as e: + raise DoltliteLoadError(f"otool failed for {_sqlite3.__file__}: {e} {e.stderr}") from e for line in out.splitlines(): token = line.strip().split(" ", 1)[0] @@ -212,10 +218,7 @@ def _build_macos_shim(lib: str) -> str: staged = Path(staging) / shim.name shutil.copyfile(lib, staged) try: - subprocess.run( - ["install_name_tool", "-id", install_name, str(staged)], - check=True, capture_output=True, text=True, - ) + _run_macos_tool("install_name_tool", "-id", install_name, str(staged)) except FileNotFoundError as e: raise DoltliteLoadError( "install_name_tool is required on macOS. Install Xcode " diff --git a/tests/test_loader.py b/tests/test_loader.py index b32c044..cc18b9a 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -122,6 +122,22 @@ def fail_copy(source, destination): self.assertEqual(list(self.cache.rglob("libsqlite3.dylib")), []) self.assertEqual(self.worker()[1], "library:patched") + def test_macos_tools_do_not_inherit_the_shim(self): + def run(command, *, env=None, **kwargs): + tool_env = os.environ if env is None else env + if "DYLD_INSERT_LIBRARIES" in tool_env: + raise subprocess.CalledProcessError(-6, command) + output = f"\t{INSTALL_NAME} (compatibility version 1.0.0)\n" + return subprocess.CompletedProcess(command, 0, stdout=output) + + with mock.patch.dict(os.environ, DYLD_INSERT_LIBRARIES=str(self.lib)), \ + mock.patch.object(loader.tempfile, "gettempdir", return_value=str(self.cache)), \ + mock.patch.object(loader.subprocess, "run", side_effect=run) as tools: + loader._build_macos_shim(str(self.lib)) + self.assertEqual(os.environ["DYLD_INSERT_LIBRARIES"], str(self.lib)) + self.assertEqual([call.args[0][0] for call in tools.call_args_list], + ["otool", "install_name_tool"]) + def test_concurrent_start_cannot_reuse_unfinished_shim(self): ready = self.root / "ready" release = self.root / "release"