From 873dd855ac950d9933878101bc27cf4383b13b4c Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 2 Nov 2026 08:10:59 -0800 Subject: [PATCH] Reuse macOS shims across interpreter starts and publish them atomically Use a stable digest of the resolved library path, mtime, and SQLite install name. Patch a private staging copy and atomically replace the cached shim so failed or concurrent builds cannot expose an unfinished library. Cover cross-process reuse, invalidation, failure cleanup, and concurrent starts in CI. Co-Authored-By: OpenAI Codex --- .github/workflows/wheels.yml | 4 + src/doltlite/_loader.py | 38 +++++---- tests/macos_shim_smoke.py | 60 ++++++++++++++ tests/test_loader.py | 149 +++++++++++++++++++++++++++++++++++ 4 files changed, 235 insertions(+), 16 deletions(-) create mode 100644 tests/macos_shim_smoke.py create mode 100644 tests/test_loader.py diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 3229d12..f7e23b3 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -47,6 +47,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Test loader cache + run: python3 -m unittest discover -s tests -p 'test_*.py' + - uses: pypa/cibuildwheel@v2.21 env: CIBW_ARCHS: ${{ matrix.cibw_archs }} @@ -156,6 +159,7 @@ jobs: "$PY" -m venv venv venv/bin/pip install wheelhouse/*.whl venv/bin/python tests/smoke.py + venv/bin/python tests/macos_shim_smoke.py publish: name: Publish to PyPI diff --git a/src/doltlite/_loader.py b/src/doltlite/_loader.py index ad1e14f..617f6d2 100644 --- a/src/doltlite/_loader.py +++ b/src/doltlite/_loader.py @@ -25,6 +25,7 @@ from __future__ import annotations import ctypes +import hashlib import os import shutil import subprocess @@ -179,8 +180,10 @@ def _build_macos_shim(lib: str) -> str: `_sqlite3.so` was linked against. The shim is cached under $TMPDIR keyed by (lib path, mtime, target install_name).""" install_name = _detect_sqlite3_install_name() + lib = str(Path(lib).resolve()) src_stat = os.stat(lib) - key = f"{abs(hash((lib, src_stat.st_mtime_ns, install_name))):016x}" + identity = f"{lib}\0{src_stat.st_mtime_ns}\0{install_name}" + key = hashlib.sha256(os.fsencode(identity)).hexdigest()[:16] cache_dir = Path(tempfile.gettempdir()) / f"doltlite-shim-{key}" shim = cache_dir / "libsqlite3.dylib" @@ -188,20 +191,23 @@ def _build_macos_shim(lib: str) -> str: return str(shim) cache_dir.mkdir(parents=True, exist_ok=True) - shutil.copyfile(lib, shim) - try: - subprocess.run( - ["install_name_tool", "-id", install_name, str(shim)], - check=True, capture_output=True, text=True, - ) - except FileNotFoundError as e: - raise DoltliteLoadError( - "install_name_tool is required on macOS. Install Xcode " - "Command Line Tools (`xcode-select --install`)." - ) from e - except subprocess.CalledProcessError as e: - raise DoltliteLoadError( - f"install_name_tool failed for {shim}: {e.stderr}" - ) from e + with tempfile.TemporaryDirectory(prefix=".build-", dir=cache_dir) as staging: + staged = Path(staging) / shim.name + shutil.copyfile(lib, staged) + try: + subprocess.run( + ["install_name_tool", "-id", install_name, str(staged)], + check=True, capture_output=True, text=True, + ) + except FileNotFoundError as e: + raise DoltliteLoadError( + "install_name_tool is required on macOS. Install Xcode " + "Command Line Tools (`xcode-select --install`)." + ) from e + except subprocess.CalledProcessError as e: + raise DoltliteLoadError( + f"install_name_tool failed for {shim}: {e.stderr}" + ) from e + os.replace(staged, shim) return str(shim) diff --git a/tests/macos_shim_smoke.py b/tests/macos_shim_smoke.py new file mode 100644 index 0000000..fb7a4d7 --- /dev/null +++ b/tests/macos_shim_smoke.py @@ -0,0 +1,60 @@ +import concurrent.futures +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile + + +PROGRAM = """ +import doltlite +import json +import os +import sqlite3 + +conn = sqlite3.connect(':memory:') +conn.execute('SELECT count(*) FROM dolt_branches').fetchone() +print(json.dumps(os.environ['DYLD_INSERT_LIBRARIES'])) +""" + + +def main(): + if sys.platform != "darwin": + raise RuntimeError("this smoke test requires macOS") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + script = root / "query.py" + script.write_text(PROGRAM) + + def run(seed, cache): + env = dict(os.environ, PYTHONHASHSEED=str(seed), TMPDIR=str(cache)) + env.pop("_DOLTLITE_BOOTSTRAPPED", None) + env.pop("DYLD_INSERT_LIBRARIES", None) + env.pop("DOLTLITE_LIB", None) + result = subprocess.run([sys.executable, str(script)], env=env, + check=True, capture_output=True, text=True, + timeout=30) + return Path(json.loads(result.stdout)) + + sequential = root / "sequential" + sequential.mkdir() + first = run(1, sequential) + stat = first.stat() + second = run(2, sequential) + assert first == second, (first, second) + assert (stat.st_ino, stat.st_mtime_ns) == (second.stat().st_ino, second.stat().st_mtime_ns) + assert len(list(sequential.glob("doltlite-shim-*"))) == 1 + + concurrent_cache = root / "concurrent" + concurrent_cache.mkdir() + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + paths = list(pool.map(lambda seed: run(seed, concurrent_cache), range(4))) + assert len(set(paths)) == 1, paths + assert len(list(concurrent_cache.glob("doltlite-shim-*"))) == 1 + assert len(list(concurrent_cache.rglob("libsqlite3.dylib"))) == 1 + print("macOS shim reuse and concurrent imports: OK") + + +if __name__ == "__main__": + main() diff --git a/tests/test_loader.py b/tests/test_loader.py new file mode 100644 index 0000000..b32c044 --- /dev/null +++ b/tests/test_loader.py @@ -0,0 +1,149 @@ +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time +import unittest +from unittest import mock + + +LOADER_PATH = Path(__file__).resolve().parents[1] / "src/doltlite/_loader.py" +SPEC = importlib.util.spec_from_file_location("doltlite_loader_test", LOADER_PATH) +loader = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(loader) +INSTALL_NAME = "/test/libsqlite3.dylib" +WORKER = """ +import importlib.util +import json +from pathlib import Path +import sys +import time + +spec = importlib.util.spec_from_file_location('loader', sys.argv[1]) +loader = importlib.util.module_from_spec(spec) +spec.loader.exec_module(loader) +loader._detect_sqlite3_install_name = lambda: sys.argv[4] +loader.tempfile.gettempdir = lambda: sys.argv[3] +def install(command, **kwargs): + if len(sys.argv) > 5: + Path(sys.argv[5]).write_text('ready') + deadline = time.monotonic() + 15 + while not Path(sys.argv[6]).exists(): + if time.monotonic() > deadline: + raise RuntimeError('installer was not released') + time.sleep(0.01) + path = Path(command[-1]) + path.write_bytes(path.read_bytes() + b':patched') + with (Path(sys.argv[3]) / 'installs').open('a') as log: + log.write('installed\\n') +loader.subprocess.run = install +shim = loader._build_macos_shim(sys.argv[2]) +print(json.dumps([shim, Path(shim).read_text()])) +""" + + +class MacosShimTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.lib = self.root / "libdoltlite.dylib" + self.lib.write_bytes(b"library") + self.cache = self.root / "cache" + self.cache.mkdir() + + def command(self, lib=None, install_name=INSTALL_NAME): + return [sys.executable, "-c", WORKER, str(LOADER_PATH), + str(lib or self.lib), str(self.cache), install_name] + + def worker(self, seed="1", lib=None, install_name=INSTALL_NAME, cwd=None): + env = dict(os.environ, PYTHONHASHSEED=seed) + result = subprocess.run(self.command(lib, install_name), env=env, + cwd=cwd, check=True, capture_output=True, + text=True, timeout=20) + return json.loads(result.stdout) + + def test_reuses_shim_across_hash_seeds(self): + results = [self.worker(seed) for seed in ("1", "2", "random")] + self.assertEqual(results, [results[0]] * 3) + self.assertEqual(results[0][1], "library:patched") + self.assertEqual((self.cache / "installs").read_text(), "installed\n") + + def test_source_mtime_invalidates_shim(self): + first = self.worker() + stat = self.lib.stat() + os.utime(self.lib, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + self.assertNotEqual(first[0], self.worker()[0]) + + def test_install_name_invalidates_shim(self): + first = self.worker() + self.assertNotEqual(first[0], self.worker(install_name="/other/libsqlite3.dylib")[0]) + + def test_relative_source_uses_same_cache(self): + first = self.worker() + self.assertEqual(first, self.worker(lib=self.lib.name, cwd=self.lib.parent)) + + def test_different_sources_do_not_share_cache(self): + other = self.root / "other" / self.lib.name + other.parent.mkdir() + other.write_bytes(b"other library") + stat = self.lib.stat() + os.utime(other, ns=(stat.st_atime_ns, stat.st_mtime_ns)) + first = self.worker(lib=self.lib.name, cwd=self.lib.parent) + second = self.worker(lib=other.name, cwd=other.parent) + self.assertNotEqual(first[0], second[0]) + self.assertEqual(second[1], "other library:patched") + + def test_failure_does_not_publish_shim(self): + for error in (FileNotFoundError("missing tool"), + subprocess.CalledProcessError(1, "install_name_tool", stderr="bad library")): + with self.subTest(error=error), \ + mock.patch.object(loader, "_detect_sqlite3_install_name", return_value=INSTALL_NAME), \ + mock.patch.object(loader.tempfile, "gettempdir", return_value=str(self.cache)), \ + mock.patch.object(loader.subprocess, "run", side_effect=error): + with self.assertRaisesRegex(loader.DoltliteLoadError, "install_name_tool"): + loader._build_macos_shim(str(self.lib)) + self.assertEqual(list(self.cache.rglob("libsqlite3.dylib")), []) + self.assertEqual(self.worker()[1], "library:patched") + + def test_failed_copy_does_not_publish_shim(self): + def fail_copy(source, destination): + Path(destination).write_bytes(b"partial") + raise OSError("copy failed") + + with mock.patch.object(loader, "_detect_sqlite3_install_name", return_value=INSTALL_NAME), \ + mock.patch.object(loader.tempfile, "gettempdir", return_value=str(self.cache)), \ + mock.patch.object(loader.shutil, "copyfile", side_effect=fail_copy): + with self.assertRaisesRegex(OSError, "copy failed"): + loader._build_macos_shim(str(self.lib)) + self.assertEqual(list(self.cache.rglob("libsqlite3.dylib")), []) + self.assertEqual(self.worker()[1], "library:patched") + + def test_concurrent_start_cannot_reuse_unfinished_shim(self): + ready = self.root / "ready" + release = self.root / "release" + command = self.command() + [str(ready), str(release)] + first = subprocess.Popen(command, env=dict(os.environ, PYTHONHASHSEED="1"), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + deadline = time.monotonic() + 10 + while not ready.exists(): + if first.poll() is not None or time.monotonic() > deadline: + self.fail("first installer did not start") + time.sleep(0.01) + second = self.worker() + self.assertEqual(second[1], "library:patched") + finally: + release.write_text("release") + stdout, stderr = first.communicate(timeout=20) + self.assertEqual(first.returncode, 0, stderr) + self.assertEqual(json.loads(stdout), second) + self.assertEqual(len(list(self.cache.glob("doltlite-shim-*"))), 1) + self.assertEqual(len(list(self.cache.rglob("libsqlite3.dylib"))), 1) + + +if __name__ == "__main__": + unittest.main()