Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 46 additions & 3 deletions .github/workflows/wheels.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,6 @@ jobs:
# runs inside the manylinux container, so build deps go into
# CIBW_BEFORE_ALL_LINUX below.
CIBW_BEFORE_BUILD: bash scripts/build-libdoltlite.sh
# Smoke test: install the wheel into a clean venv and run a
# script that verifies dolt_* functions are wired up.
CIBW_TEST_COMMAND: "python {project}/tests/smoke.py"
# cibuildwheel's macOS Python (python.org-style framework
# build) statically links SQLite into _sqlite3.so — same case
Expand Down Expand Up @@ -161,9 +159,54 @@ jobs:
venv/bin/python tests/smoke.py
venv/bin/python tests/macos_shim_smoke.py

smoke_linux:
name: Linux installed wheel and sdist
needs: [build_wheels, build_sdist]
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4

- uses: astral-sh/setup-uv@v6
with:
version: "0.9.15"

- uses: actions/download-artifact@v4
with:
name: wheels-ubuntu-latest-x86_64
path: wheelhouse

- uses: actions/download-artifact@v4
with:
name: sdist
path: dist

- name: Test the wheel with a uv venv using distro Python
run: |
venv="$RUNNER_TEMP/shared-sqlite"
uv venv --no-managed-python --python /usr/bin/python3 "$venv"
uv pip install --python "$venv/bin/python" wheelhouse/*.whl
"$venv/bin/python" tests/smoke.py

- name: Test the wheel with uv's statically linked Python
run: |
venv="$RUNNER_TEMP/standalone"
uv python install 3.12.12
uv venv --managed-python --python 3.12.12 "$venv"
uv pip install --python "$venv/bin/python" wheelhouse/*.whl
"$venv/bin/python" tests/smoke.py --reject-standalone

- name: Install the sdist with an explicit matching native library
run: |
library="$RUNNER_TEMP/native-wheel"
/usr/bin/python3 -m zipfile -e wheelhouse/*.whl "$library"
venv="$RUNNER_TEMP/from-sdist"
uv venv --no-managed-python --python /usr/bin/python3 "$venv"
uv pip install --python "$venv/bin/python" dist/*.tar.gz
DOLTLITE_LIB="$library/doltlite/_lib/libdoltlite.so" "$venv/bin/python" tests/smoke.py

publish:
name: Publish to PyPI
needs: [build_wheels, build_sdist, smoke_macos]
needs: [build_wheels, build_sdist, smoke_macos, smoke_linux]
if: >-
(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) ||
(github.event_name == 'workflow_dispatch' && inputs.publish)
Expand Down
74 changes: 47 additions & 27 deletions src/doltlite/_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
"""
from __future__ import annotations

from contextlib import closing
import ctypes
import hashlib
import os
Expand All @@ -36,6 +37,11 @@


_BOOTSTRAP_MARKER = "_DOLTLITE_BOOTSTRAPPED"
_SHARED_SQLITE_HINT = (
"DoltLite needs a Python interpreter that dynamically links libsqlite3. "
"With uv, select one explicitly using uv venv --python /path/to/python."
)
_bootstrapped = False
_PKG_LIB_DIR = Path(__file__).resolve().parent / "_lib"


Expand Down Expand Up @@ -75,46 +81,58 @@ def libdoltlite_path() -> str:
)


def _is_bootstrapped() -> bool:
return os.environ.get(_BOOTSTRAP_MARKER) == "1"


def _sqlite3_already_imported() -> bool:
return "sqlite3" in sys.modules or "_sqlite3" in sys.modules


def bootstrap() -> None:
"""Make libdoltlite the active SQLite engine for this interpreter.

Idempotent: subsequent calls (and re-entrant calls after a re-exec)
are no-ops. Raises `DoltliteLoadError` if the library can't be found
Subsequent calls in the same process are no-ops after verification.
Raises `DoltliteLoadError` if the library can't be found
or if a re-exec is needed but `sys.argv` does not name a script we
can replay (interactive shell, `python -c …`, REPL, notebook).
can replay (interactive shell, `python -c …`, REPL, notebook), or
when Python still resolves sqlite3 to another engine.
"""
if _is_bootstrapped():
global _bootstrapped
if _bootstrapped:
return

lib = libdoltlite_path()
if os.environ.pop(_BOOTSTRAP_MARKER, None) != "1":
lib = libdoltlite_path()
if sys.platform != "darwin" and not _sqlite3_already_imported():
ctypes.CDLL(lib, mode=ctypes.RTLD_GLOBAL)
else:
_reexec(lib)

if sys.platform != "darwin" and not _sqlite3_already_imported():
ctypes.CDLL(lib, mode=ctypes.RTLD_GLOBAL)
os.environ[_BOOTSTRAP_MARKER] = "1"
return
_verify_engine()
_bootstrapped = True

_require_replayable_argv()

env = dict(os.environ)
env[_BOOTSTRAP_MARKER] = "1"

def _reexec(lib: str) -> None:
_require_replayable_argv()
env = dict(os.environ, **{_BOOTSTRAP_MARKER: "1"})
if sys.platform == "darwin":
env["DYLD_INSERT_LIBRARIES"] = _build_macos_shim(lib)
else:
existing = env.get("LD_PRELOAD", "").strip()
env["LD_PRELOAD"] = f"{lib} {existing}".strip()

os.execvpe(sys.executable, [sys.executable, *sys.argv], env)


def _verify_engine() -> None:
import sqlite3

message = f"Could not activate DoltLite as Python's sqlite3 engine. {_SHARED_SQLITE_HINT}"
try:
with closing(sqlite3.connect(":memory:")) as db:
engine = db.execute("SELECT doltlite_engine()").fetchone()
except sqlite3.Error as error:
raise DoltliteLoadError(message) from error
if engine != ("prolly",):
raise DoltliteLoadError(message)


def _require_replayable_argv() -> None:
"""Bail out clearly if we can't safely re-exec the current process.

Expand Down Expand Up @@ -148,21 +166,26 @@ def _require_replayable_argv() -> None:
)


def _run_macos_tool(*command: str) -> subprocess.CompletedProcess[str]:
env = os.environ.copy()
env.pop("DYLD_INSERT_LIBRARIES", None)
return subprocess.run(command, env=env, check=True, capture_output=True, text=True)


def _detect_sqlite3_install_name() -> str:
"""On macOS, return the libsqlite3 path that the active interpreter's
`_sqlite3` extension was linked against."""
import _sqlite3

try:
out = subprocess.run(
["otool", "-L", _sqlite3.__file__],
check=True, capture_output=True, text=True,
).stdout
out = _run_macos_tool("otool", "-L", _sqlite3.__file__).stdout
except FileNotFoundError as e:
raise DoltliteLoadError(
"otool is required on macOS to detect the active "
"libsqlite3 path. Install Xcode Command Line Tools."
) from e
except subprocess.CalledProcessError as e:
raise DoltliteLoadError(f"otool failed for {_sqlite3.__file__}: {e} {e.stderr}") from e

for line in out.splitlines():
token = line.strip().split(" ", 1)[0]
Expand All @@ -171,7 +194,7 @@ def _detect_sqlite3_install_name() -> str:

raise DoltliteLoadError(
"Could not find a libsqlite3 dependency in "
f"{_sqlite3.__file__}. Output of otool -L:\n{out}"
f"{_sqlite3.__file__}. {_SHARED_SQLITE_HINT} Output of otool -L:\n{out}"
)


Expand All @@ -195,10 +218,7 @@ def _build_macos_shim(lib: str) -> str:
staged = Path(staging) / shim.name
shutil.copyfile(lib, staged)
try:
subprocess.run(
["install_name_tool", "-id", install_name, str(staged)],
check=True, capture_output=True, text=True,
)
_run_macos_tool("install_name_tool", "-id", install_name, str(staged))
except FileNotFoundError as e:
raise DoltliteLoadError(
"install_name_tool is required on macOS. Install Xcode "
Expand Down
97 changes: 78 additions & 19 deletions tests/smoke.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,88 @@
"""Smoke test: after `import doltlite`, the stdlib sqlite3 module must
expose libdoltlite's SQL functions and virtual tables."""
import doltlite
import sqlite3
"""Exercise the installed package through stdlib sqlite3 in fresh processes."""
from contextlib import closing
from importlib.metadata import version
import os
from pathlib import Path
import subprocess
import sys
import tempfile


def main() -> int:
conn = sqlite3.connect(":memory:")
SCRIPT = Path(__file__).resolve()
ROWS = [(1, "café", b"\x00\xff", None), (2, "second", b"\x01", 2**63 - 1)]
MESSAGE = "file-backed smoke"

version = conn.execute("SELECT dolt_version()").fetchone()
print(f"dolt_version: {version}")
assert version and version[0].startswith("v"), f"unexpected dolt_version: {version}"

conn.execute("CREATE TABLE t(x INT)")
conn.execute("INSERT INTO t VALUES (1), (2), (3)")
conn.execute("SELECT dolt_commit('-A', '-m', 'smoke')")
def child(*args, env=None, cwd=None):
subprocess.run([sys.executable, str(SCRIPT), "--worker", *args],
env=env, cwd=cwd, check=True, timeout=30)

log = list(conn.execute("SELECT commit_hash, message FROM dolt_log"))
assert any(msg == "smoke" for _, msg in log), f"smoke commit missing from log: {log}"
print(f"dolt_log entries: {len(log)}")

print(f"libdoltlite path: {doltlite.libdoltlite_path()}")
print("OK")
return 0
def consumer(mode, order, filename=None, commit=None):
if order == "sqlite3-first":
import sqlite3

try:
import doltlite
except RuntimeError as error:
if mode != "unsupported":
raise
assert type(error).__name__ == "DoltliteLoadError", repr(error)
assert "dynamically links libsqlite3" in str(error), str(error)
assert "uv venv --python" in str(error), str(error)
return

if mode == "unsupported":
raise AssertionError(f"{order}: standalone Python unexpectedly accepted DoltLite")

import sqlite3

package_version = version("doltlite")
expected_engine_version = "v" + package_version.split(".post", 1)[0]
assert doltlite.__version__ == package_version
if override := os.environ.get("DOLTLITE_LIB"):
assert Path(doltlite.libdoltlite_path()).resolve() == Path(override).resolve()

if mode == "reopen":
assert Path(filename).is_file(), filename
with closing(sqlite3.connect(filename)) as db:
assert db.execute("SELECT doltlite_engine(), dolt_version()").fetchone() == (
"prolly", expected_engine_version)
if mode == "write":
db.execute("CREATE TABLE t(id INTEGER PRIMARY KEY, label TEXT, blob BLOB, value INTEGER)")
db.executemany("INSERT INTO t VALUES (?, ?, ?, ?)", ROWS)
db.commit()
db.execute("BEGIN")
db.execute("UPDATE t SET label = 'rolled back'")
db.rollback()
(commit,) = db.execute("SELECT dolt_commit('-A', '-m', ?)", (MESSAGE,)).fetchone()
assert isinstance(commit, str) and commit, commit

assert db.execute("SELECT * FROM t ORDER BY id").fetchall() == ROWS
assert db.execute("SELECT commit_hash FROM dolt_log WHERE message = ?",
(MESSAGE,)).fetchall() == [(commit,)]

if mode == "write":
child("reopen", order, filename, commit)


def main(unsupported=False):
env = dict(os.environ)
for name in ("_DOLTLITE_BOOTSTRAPPED", "LD_PRELOAD", "DYLD_INSERT_LIBRARIES", "PYTHONPATH"):
env.pop(name, None)
with tempfile.TemporaryDirectory() as directory:
for order in ("doltlite-first", "sqlite3-first"):
if unsupported:
child("unsupported", order, env=env, cwd=directory)
else:
filename = str(Path(directory, f"{order}.db"))
child("write", order, filename, env=env, cwd=directory)
print(f"{order}: {'unsupported interpreter rejected' if unsupported else 'rows and history survived a child process'}")


if __name__ == "__main__":
sys.exit(main())
if sys.argv[1:2] == ["--worker"]:
consumer(*sys.argv[2:])
else:
assert sys.argv[1:] in ([], ["--reject-standalone"])
main(unsupported=bool(sys.argv[1:]))
16 changes: 16 additions & 0 deletions tests/test_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,22 @@ def fail_copy(source, destination):
self.assertEqual(list(self.cache.rglob("libsqlite3.dylib")), [])
self.assertEqual(self.worker()[1], "library:patched")

def test_macos_tools_do_not_inherit_the_shim(self):
def run(command, *, env=None, **kwargs):
tool_env = os.environ if env is None else env
if "DYLD_INSERT_LIBRARIES" in tool_env:
raise subprocess.CalledProcessError(-6, command)
output = f"\t{INSTALL_NAME} (compatibility version 1.0.0)\n"
return subprocess.CompletedProcess(command, 0, stdout=output)

with mock.patch.dict(os.environ, DYLD_INSERT_LIBRARIES=str(self.lib)), \
mock.patch.object(loader.tempfile, "gettempdir", return_value=str(self.cache)), \
mock.patch.object(loader.subprocess, "run", side_effect=run) as tools:
loader._build_macos_shim(str(self.lib))
self.assertEqual(os.environ["DYLD_INSERT_LIBRARIES"], str(self.lib))
self.assertEqual([call.args[0][0] for call in tools.call_args_list],
["otool", "install_name_tool"])

def test_concurrent_start_cannot_reuse_unfinished_shim(self):
ready = self.root / "ready"
release = self.root / "release"
Expand Down
Loading