From 5c1bc800b18135adca4685eeb5b0b82cd55daa8c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 06:52:57 -0500 Subject: [PATCH 01/26] PYTHON-6048 Add sanitizer rebuild-and-test script --- .evergreen/scripts/run-sanitizer-tests.sh | 59 +++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100755 .evergreen/scripts/run-sanitizer-tests.sh diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh new file mode 100755 index 0000000000..d44d8a7a90 --- /dev/null +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Clean sanitizer rebuild of the C extensions, run the fixed BSON/client +# test files under the matching sanitizer runtime, and fail on any +# sanitizer diagnostic even if pytest itself exits 0. +set -eu + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." + +SANITIZER=${SANITIZER:?"SANITIZER must be set to 'asan' or 'tsan'"} +PYTHON_BIN=${PYTHON_BIN:-python3} +TEST_FILES=(test/test_bson.py test/test_raw_bson.py test/test_raw_bson_shared.py test/test_client.py) + +# A stale build/ can leave a .so linked against the wrong sanitizer's +# runtime without a build error, so always start clean. +rm -rf build +rm -f bson/*.so pymongo/*.so + +export CC=${CC:-clang} +export CXX=${CXX:-clang++} +export PYMONGO_C_EXT_MUST_BUILD=1 +# Route CPython's own allocations through the system allocator so ASan's +# redzones can see them; pymalloc otherwise hides real bugs and adds noise. +export PYTHONMALLOC=malloc + +case "$SANITIZER" in + asan) + export CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g -O0" + export LDFLAGS="-fsanitize=address,undefined" + RUNTIME_LIB=$("$CC" -print-file-name=libasan.so) + export ASAN_OPTIONS="detect_leaks=0" + export UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1" + ;; + tsan) + export CFLAGS="-fsanitize=thread -fno-omit-frame-pointer -g -O0" + export LDFLAGS="-fsanitize=thread" + RUNTIME_LIB=$("$CC" -print-file-name=libtsan.so) + export TSAN_OPTIONS="halt_on_error=1" + ;; + *) + echo "Unknown SANITIZER: $SANITIZER (expected 'asan' or 'tsan')" >&2 + exit 1 + ;; +esac + +"$PYTHON_BIN" -m pip install -e . --no-build-isolation --force-reinstall --no-deps +"$PYTHON_BIN" -m pip install pytest + +LOG_FILE=$(mktemp) +set +e +LD_PRELOAD="$RUNTIME_LIB" "$PYTHON_BIN" -m pytest -v "${TEST_FILES[@]}" 2>&1 | tee "$LOG_FILE" +PYTEST_STATUS=${PIPESTATUS[0]} +set -e + +if grep -qE "ERROR: (AddressSanitizer|LeakSanitizer)|runtime error:|WARNING: ThreadSanitizer|SUMMARY: (Address|Undefined|ThreadSanitizer)" "$LOG_FILE"; then + echo "Sanitizer diagnostic detected in test output, failing task" >&2 + exit 1 +fi + +exit "$PYTEST_STATUS" From 063d9377362290fc032cfef4419a29b90571dab1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 06:55:59 -0500 Subject: [PATCH 02/26] PYTHON-6048 Validate sanitizer runtime library path before use --- .evergreen/scripts/run-sanitizer-tests.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index d44d8a7a90..b803747fa1 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -42,6 +42,11 @@ case "$SANITIZER" in ;; esac +if [ ! -f "$RUNTIME_LIB" ]; then + echo "Could not locate the $SANITIZER runtime library (got: $RUNTIME_LIB). Is the matching sanitizer runtime package installed?" >&2 + exit 1 +fi + "$PYTHON_BIN" -m pip install -e . --no-build-isolation --force-reinstall --no-deps "$PYTHON_BIN" -m pip install pytest From f8a2304f85f3e182ccbaa114550058a060a04e31 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 06:57:58 -0500 Subject: [PATCH 03/26] PYTHON-6048 Add run-sanitizer-tests just recipe --- justfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/justfile b/justfile index df678cdbac..d9eabc0b83 100644 --- a/justfile +++ b/justfile @@ -129,6 +129,9 @@ coverage-xml: run-server *args="": bash .evergreen/scripts/run-server.sh {{args}} +run-sanitizer-tests *args="": + bash .evergreen/scripts/run-sanitizer-tests.sh {{args}} + [group('server')] stop-server: bash .evergreen/scripts/stop-server.sh From f5f5a8aec3efd254763a9cb5c25042cbd5290607 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 07:02:00 -0500 Subject: [PATCH 04/26] PYTHON-6048 Add sanitizer Evergreen function, tasks, and variant --- .evergreen/generated_configs/functions.yml | 14 ++++++++++ .evergreen/generated_configs/tasks.yml | 27 ++++++++++++++++++ .evergreen/generated_configs/variants.yml | 10 +++++++ .evergreen/scripts/generate_config.py | 32 ++++++++++++++++++++++ 4 files changed, 83 insertions(+) diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index 9fbc1f8500..92ac126633 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -85,6 +85,20 @@ functions: params: directory: src + # Run sanitizer tests + run sanitizer tests: + - command: subprocess.exec + params: + binary: bash + args: + - .evergreen/just.sh + - run-sanitizer-tests + working_dir: src + include_expansions_in_env: + - SANITIZER + - UV_PYTHON + type: test + # Run server run server: - command: subprocess.exec diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 9fe6b33406..0538a30c3f 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -2895,6 +2895,33 @@ tasks: - func: send dashboard data tags: [perf] + # Sanitizer tests + - name: test-sanitizer-asan + commands: + - func: run server + vars: + VERSION: latest + TOPOLOGY: standalone + AUTH: noauth + SSL: nossl + - func: run sanitizer tests + vars: + SANITIZER: asan + tags: [sanitizer, pr] + - name: test-sanitizer-tsan + commands: + - func: run server + vars: + VERSION: latest + TOPOLOGY: standalone + AUTH: noauth + SSL: nossl + - func: run sanitizer tests + vars: + SANITIZER: tsan + UV_PYTHON: 3.14t + tags: [sanitizer, pr, free-threaded] + # Search index tests - name: test-search-index-helpers commands: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 72af2599f7..7152e715ee 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -489,6 +489,16 @@ buildvariants: expansions: SUB_TEST_NAME: pyopenssl + # Sanitizer tests + - name: sanitizers-ubuntu-22 + tasks: + - name: test-sanitizer-asan + - name: test-sanitizer-tsan + display_name: Sanitizers Ubuntu-22 + run_on: + - ubuntu2204-small + tags: [pr] + # Search index tests - name: search-index-helpers-rhel8 tasks: diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index aaded3130c..fdb4cffcad 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -341,6 +341,13 @@ def create_mod_wsgi_variants(): return [create_variant(tasks, display_name, host=host, expansions=expansions)] +def create_sanitizer_variants(): + host = HOSTS["ubuntu22"] + tasks = ["test-sanitizer-asan", "test-sanitizer-tsan"] + display_name = get_variant_name("Sanitizers", host) + return [create_variant(tasks, display_name, host=host, tags=["pr"])] + + def create_disable_test_commands_variants(): host = DEFAULT_HOST expansions = dict(AUTH="auth", SSL="ssl", DISABLE_TEST_COMMANDS="1") @@ -659,6 +666,24 @@ def create_no_toolchain_tasks(): return tasks +def create_sanitizer_tasks(): + tasks = [] + # (sanitizer, free-threaded UV_PYTHON or None) + configs = [("asan", None), ("tsan", "3.14t")] + for sanitizer, python in configs: + tags = ["sanitizer", "pr"] + server_vars = dict(VERSION="latest", TOPOLOGY="standalone", AUTH="noauth", SSL="nossl") + server_func = FunctionCall(func="run server", vars=server_vars) + test_vars = dict(SANITIZER=sanitizer) + if python: + test_vars["UV_PYTHON"] = python + tags.append("free-threaded") + test_func = FunctionCall(func="run sanitizer tests", vars=test_vars) + name = f"test-sanitizer-{sanitizer}" + tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) + return tasks + + def create_test_non_standard_tasks(): """For variants that set a TEST_NAME.""" tasks = [] @@ -1315,6 +1340,13 @@ def create_run_tests_func(): return "run tests", [setup_cmd, test_cmd] +def create_run_sanitizer_tests_func(): + includes = ["SANITIZER", "UV_PYTHON"] + args = [".evergreen/just.sh", "run-sanitizer-tests"] + sub_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args) + return "run sanitizer tests", [sub_cmd] + + def create_test_numpy_func(): includes = ["TOOLCHAIN_VERSION", "COVERAGE"] test_cmd = get_subprocess_exec( From 61a4f69b04833ff8c31033bddd65fc6b51f0647c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 08:10:59 -0500 Subject: [PATCH 05/26] PYTHON-6048 Add RawBSONDocument concurrency and buffer-lifetime regression tests --- test/test_raw_bson_shared.py | 92 ++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/test/test_raw_bson_shared.py b/test/test_raw_bson_shared.py index b7d54879aa..b0737b0b45 100644 --- a/test/test_raw_bson_shared.py +++ b/test/test_raw_bson_shared.py @@ -17,6 +17,7 @@ import gc import pickle import sys +import threading import unittest import uuid @@ -40,6 +41,20 @@ b"\x00\x00\x00\x02street\x00\r\x00\x00\x00Baker Street\x00\x00\x00\x00" ) +N_THREADS = 16 +N_ITERS = 200 +BIG_PAYLOAD = "x" * 8000 + + +def _make_shared_doc_bytes() -> bytes: + return encode( + { + "small": {"n": 1}, + "big": {"payload": BIG_PAYLOAD, "n": 42}, + "arr": [{"payload": BIG_PAYLOAD, "idx": i} for i in range(3)], + } + ) + class _TaggedRawBSONDocument(RawBSONDocument): """RawBSONDocument subclass with a different __init__ signature and @@ -321,5 +336,82 @@ def test_contains_dbref(self): self.assertEqual(doc["value"].raw, raw_encoded) +class TestRawBSONDocumentConcurrency(unittest.TestCase): + """Concurrency and buffer-lifetime regression tests for zero-copy + RawBSONDocument, intended to also run under ASan/UBSan/TSan in CI + (see .evergreen/scripts/run-sanitizer-tests.sh).""" + + def test_concurrent_reads_on_shared_document(self): + doc_bytes = _make_shared_doc_bytes() + shared_doc = RawBSONDocument(doc_bytes) + + # Confirm we're actually exercising the zero-copy memoryview path, + # not accidentally testing a byte-copy fallback. + big_check = shared_doc["big"] + self.assertIsInstance(big_check.raw, memoryview) + self.assertTrue(big_check.raw.readonly) + + errors: list[BaseException] = [] + errors_lock = threading.Lock() + + def worker() -> None: + try: + for _ in range(N_ITERS): + big = shared_doc["big"] + assert big["payload"] == BIG_PAYLOAD + assert big["n"] == 42 + assert isinstance(big.raw, memoryview) + + arr = shared_doc["arr"] + for j, item in enumerate(arr): + assert item["idx"] == j + assert item["payload"] == BIG_PAYLOAD + + small = shared_doc["small"] + assert small["n"] == 1 + + # Re-encodes a view-backed subdocument while other + # threads may be reading the same underlying buffer. + reencoded = encode({"again": big}) + assert isinstance(reencoded, bytes) + + multi = decode_all(doc_bytes * 2, DEFAULT_RAW_BSON_OPTIONS) + assert len(multi) == 2 + for d in multi: + assert isinstance(d["big"].raw, memoryview) + except BaseException as exc: + with errors_lock: + errors.append(exc) + raise + + threads = [threading.Thread(target=worker) for _ in range(N_THREADS)] + for t in threads: + t.start() + for t in threads: + t.join() + + self.assertEqual(errors, []) + + def test_buffer_survives_after_source_bytes_are_dropped(self): + # Buffer-lifetime edge case: slice a bytearray, wrap it in a + # RawBSONDocument, then mutate/drop the original backing buffer to + # make sure the document doesn't hold a dangling view. + encoded = _make_shared_doc_bytes() + for _ in range(50): + buf = bytearray(encoded) + b"\x00" * 10 + view = memoryview(buf)[: len(encoded)] + raw = RawBSONDocument(view) + _ = raw["big"] + _ = dict(raw["small"]) + _ = list(raw["arr"]) + copied = dict(raw) + del raw + del view + buf[:] = b"\xff" * len(buf) + del buf + gc.collect() + self.assertEqual(copied["small"]["n"], 1) + + if __name__ == "__main__": unittest.main() From 7c0bc232f72f83d58955d8a36dc5bfa978490f92 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 08:21:27 -0500 Subject: [PATCH 06/26] PYTHON-6048 Add concurrent batched wire-message builder coverage --- test/test_raw_bson_shared.py | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/test/test_raw_bson_shared.py b/test/test_raw_bson_shared.py index b0737b0b45..e5b23f75a3 100644 --- a/test/test_raw_bson_shared.py +++ b/test/test_raw_bson_shared.py @@ -413,5 +413,65 @@ def test_buffer_survives_after_source_bytes_are_dropped(self): self.assertEqual(copied["small"]["n"], 1) +class _FakeBulkWriteContext: + """Stand-in for pymongo.message._BulkWriteContext. + + The C batched-message builders only read four numeric attributes off + the context object, so a plain object exposing those is enough to + drive them without a live server connection. + """ + + max_bson_size = 16 * 1024 * 1024 + max_write_batch_size = 100000 + max_message_size = 48 * 1024 * 1024 + max_split_size = 16 * 1024 * 1024 + + +class TestBatchedMessageBuilderConcurrency(unittest.TestCase): + def test_concurrent_batched_message_building(self): + try: + from pymongo import _cmessage + except ImportError: + self.skipTest("pymongo._cmessage C extension is not built") + + has_op_msg = hasattr(_cmessage, "_encode_batched_op_msg") + has_write_cmd = hasattr(_cmessage, "_encode_batched_write_command") + if not (has_op_msg or has_write_cmd): + self.skipTest("no batched message builder found on pymongo._cmessage") + + ns = "db.coll" + docs = [{"_id": i, "payload": "y" * 100} for i in range(50)] + command = {"insert": "coll", "ordered": True} + ctx = _FakeBulkWriteContext() + insert_op = 0 # pymongo.message._INSERT + + errors: list[BaseException] = [] + errors_lock = threading.Lock() + + def worker() -> None: + try: + for _ in range(N_ITERS): + if has_op_msg: + _cmessage._encode_batched_op_msg( + insert_op, command, docs, True, DEFAULT_RAW_BSON_OPTIONS, ctx + ) + if has_write_cmd: + _cmessage._encode_batched_write_command( + ns, insert_op, command, docs, DEFAULT_RAW_BSON_OPTIONS, ctx + ) + except BaseException as exc: + with errors_lock: + errors.append(exc) + raise + + threads = [threading.Thread(target=worker) for _ in range(N_THREADS)] + for t in threads: + t.start() + for t in threads: + t.join() + + self.assertEqual(errors, []) + + if __name__ == "__main__": unittest.main() From c6dc49a4109ffcc6db6981fdbea6b5f848b1aad5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 08:46:27 -0500 Subject: [PATCH 07/26] PYTHON-6048 Install hatchling build backend before sanitizer rebuild --- .evergreen/scripts/run-sanitizer-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index b803747fa1..5903ff76b4 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -47,6 +47,7 @@ if [ ! -f "$RUNTIME_LIB" ]; then exit 1 fi +"$PYTHON_BIN" -m pip install "hatchling>1.24" "setuptools>=65.0" "hatch-requirements-txt>=0.4.1" "$PYTHON_BIN" -m pip install -e . --no-build-isolation --force-reinstall --no-deps "$PYTHON_BIN" -m pip install pytest From 72ba66021f12721199a35b0a8d49487967c96865 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 10:05:01 -0500 Subject: [PATCH 08/26] PYTHON-6048 Use normal pip build isolation for sanitizer rebuild --- .evergreen/scripts/run-sanitizer-tests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index 5903ff76b4..c0e7204247 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -47,8 +47,7 @@ if [ ! -f "$RUNTIME_LIB" ]; then exit 1 fi -"$PYTHON_BIN" -m pip install "hatchling>1.24" "setuptools>=65.0" "hatch-requirements-txt>=0.4.1" -"$PYTHON_BIN" -m pip install -e . --no-build-isolation --force-reinstall --no-deps +"$PYTHON_BIN" -m pip install -e . --force-reinstall --no-deps "$PYTHON_BIN" -m pip install pytest LOG_FILE=$(mktemp) From 0cf13f9bc035cc0415d5a43fd474953646618d12 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 10:23:35 -0500 Subject: [PATCH 09/26] PYTHON-6048 Constrain packaging version for sanitizer build isolation --- .evergreen/scripts/run-sanitizer-tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index c0e7204247..3e64530c4b 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -47,6 +47,8 @@ if [ ! -f "$RUNTIME_LIB" ]; then exit 1 fi +export PIP_CONSTRAINT="$(mktemp)" +echo "packaging>=24.2" > "$PIP_CONSTRAINT" "$PYTHON_BIN" -m pip install -e . --force-reinstall --no-deps "$PYTHON_BIN" -m pip install pytest From 2dddcc3beddc75fb692bbc0e0f7a2a1839a66135 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 10:32:24 -0500 Subject: [PATCH 10/26] PYTHON-6048 Upgrade pip before sanitizer build to fix dependency resolution --- .evergreen/scripts/run-sanitizer-tests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index 3e64530c4b..2f7764a534 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -10,6 +10,10 @@ SANITIZER=${SANITIZER:?"SANITIZER must be set to 'asan' or 'tsan'"} PYTHON_BIN=${PYTHON_BIN:-python3} TEST_FILES=(test/test_bson.py test/test_raw_bson.py test/test_raw_bson_shared.py test/test_client.py) +# Upgrade pip first so the isolated build environment is created by a modern pip +# that correctly resolves the full dependency graph for build backends like hatchling. +"$PYTHON_BIN" -m pip install --upgrade pip + # A stale build/ can leave a .so linked against the wrong sanitizer's # runtime without a build error, so always start clean. rm -rf build @@ -47,8 +51,6 @@ if [ ! -f "$RUNTIME_LIB" ]; then exit 1 fi -export PIP_CONSTRAINT="$(mktemp)" -echo "packaging>=24.2" > "$PIP_CONSTRAINT" "$PYTHON_BIN" -m pip install -e . --force-reinstall --no-deps "$PYTHON_BIN" -m pip install pytest From d07d4b02344b2124d7e15a9ac2db4e44c2426889 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 10:37:49 -0500 Subject: [PATCH 11/26] PYTHON-6048 Use uv instead of system pip for sanitizer rebuild --- .evergreen/scripts/run-sanitizer-tests.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index 2f7764a534..967a0e5ce1 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -7,17 +7,14 @@ set -eu cd "$(dirname "${BASH_SOURCE[0]}")/../.." SANITIZER=${SANITIZER:?"SANITIZER must be set to 'asan' or 'tsan'"} -PYTHON_BIN=${PYTHON_BIN:-python3} +UV_PYTHON=${UV_PYTHON:-3.13} TEST_FILES=(test/test_bson.py test/test_raw_bson.py test/test_raw_bson_shared.py test/test_client.py) -# Upgrade pip first so the isolated build environment is created by a modern pip -# that correctly resolves the full dependency graph for build backends like hatchling. -"$PYTHON_BIN" -m pip install --upgrade pip - -# A stale build/ can leave a .so linked against the wrong sanitizer's -# runtime without a build error, so always start clean. +# A stale build/ or venv can leave a .so or install linked against the wrong +# sanitizer's runtime without a build error, so always start clean. rm -rf build rm -f bson/*.so pymongo/*.so +rm -rf .sanitizer-venv export CC=${CC:-clang} export CXX=${CXX:-clang++} @@ -51,12 +48,14 @@ if [ ! -f "$RUNTIME_LIB" ]; then exit 1 fi -"$PYTHON_BIN" -m pip install -e . --force-reinstall --no-deps -"$PYTHON_BIN" -m pip install pytest +uv venv --python "$UV_PYTHON" .sanitizer-venv +VENV_PYTHON=.sanitizer-venv/bin/python3 +uv pip install --python "$VENV_PYTHON" -e . --reinstall --no-deps +uv pip install --python "$VENV_PYTHON" pytest LOG_FILE=$(mktemp) set +e -LD_PRELOAD="$RUNTIME_LIB" "$PYTHON_BIN" -m pytest -v "${TEST_FILES[@]}" 2>&1 | tee "$LOG_FILE" +LD_PRELOAD="$RUNTIME_LIB" "$VENV_PYTHON" -m pytest -v "${TEST_FILES[@]}" 2>&1 | tee "$LOG_FILE" PYTEST_STATUS=${PIPESTATUS[0]} set -e From 1b5238b7574576443d1e79b0bf583bc858a011cb Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 10:59:42 -0500 Subject: [PATCH 12/26] PYTHON-6048 Install pytest-asyncio for sanitizer test venv --- .evergreen/scripts/run-sanitizer-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index 967a0e5ce1..cd7a407c67 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -51,7 +51,7 @@ fi uv venv --python "$UV_PYTHON" .sanitizer-venv VENV_PYTHON=.sanitizer-venv/bin/python3 uv pip install --python "$VENV_PYTHON" -e . --reinstall --no-deps -uv pip install --python "$VENV_PYTHON" pytest +uv pip install --python "$VENV_PYTHON" -r requirements/test.txt LOG_FILE=$(mktemp) set +e From 7887cf0f6c0df02a24a1a4893ec47f748ef193fb Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 11:09:01 -0500 Subject: [PATCH 13/26] PYTHON-6048 Install pymongo's runtime deps and scope PYTHONMALLOC to ASan --- .evergreen/scripts/run-sanitizer-tests.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index cd7a407c67..d37f104f08 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -19,9 +19,6 @@ rm -rf .sanitizer-venv export CC=${CC:-clang} export CXX=${CXX:-clang++} export PYMONGO_C_EXT_MUST_BUILD=1 -# Route CPython's own allocations through the system allocator so ASan's -# redzones can see them; pymalloc otherwise hides real bugs and adds noise. -export PYTHONMALLOC=malloc case "$SANITIZER" in asan) @@ -30,6 +27,11 @@ case "$SANITIZER" in RUNTIME_LIB=$("$CC" -print-file-name=libasan.so) export ASAN_OPTIONS="detect_leaks=0" export UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1" + # Route CPython's own allocations through the system allocator so ASan's + # redzones can see them; pymalloc otherwise hides real bugs and adds + # noise. The free-threaded TSan interpreter doesn't accept this value, + # so it's scoped to ASan only. + export PYTHONMALLOC=malloc ;; tsan) export CFLAGS="-fsanitize=thread -fno-omit-frame-pointer -g -O0" @@ -50,7 +52,7 @@ fi uv venv --python "$UV_PYTHON" .sanitizer-venv VENV_PYTHON=.sanitizer-venv/bin/python3 -uv pip install --python "$VENV_PYTHON" -e . --reinstall --no-deps +uv pip install --python "$VENV_PYTHON" -e . --reinstall uv pip install --python "$VENV_PYTHON" -r requirements/test.txt LOG_FILE=$(mktemp) From d29ef765603565c8c2072da180082dc0b3aeb691 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 11:29:23 -0500 Subject: [PATCH 14/26] PYTHON-6048 Disable pytest output capturing so sanitizer reports aren't swallowed --- .evergreen/scripts/run-sanitizer-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index d37f104f08..ca5626a409 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -57,7 +57,7 @@ uv pip install --python "$VENV_PYTHON" -r requirements/test.txt LOG_FILE=$(mktemp) set +e -LD_PRELOAD="$RUNTIME_LIB" "$VENV_PYTHON" -m pytest -v "${TEST_FILES[@]}" 2>&1 | tee "$LOG_FILE" +LD_PRELOAD="$RUNTIME_LIB" "$VENV_PYTHON" -m pytest -v --capture=no "${TEST_FILES[@]}" 2>&1 | tee "$LOG_FILE" PYTEST_STATUS=${PIPESTATUS[0]} set -e From 7c565e8a18e7905b1b1a517520d39e398c8a1e57 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 12:05:33 -0500 Subject: [PATCH 15/26] PYTHON-6048 Add TSan suppressions for CPython free-threading internals --- .evergreen/scripts/run-sanitizer-tests.sh | 2 +- .evergreen/tsan-suppressions.txt | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 .evergreen/tsan-suppressions.txt diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index ca5626a409..24e80808d8 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -37,7 +37,7 @@ case "$SANITIZER" in export CFLAGS="-fsanitize=thread -fno-omit-frame-pointer -g -O0" export LDFLAGS="-fsanitize=thread" RUNTIME_LIB=$("$CC" -print-file-name=libtsan.so) - export TSAN_OPTIONS="halt_on_error=1" + export TSAN_OPTIONS="halt_on_error=1:suppressions=$(pwd)/.evergreen/tsan-suppressions.txt" ;; *) echo "Unknown SANITIZER: $SANITIZER (expected 'asan' or 'tsan')" >&2 diff --git a/.evergreen/tsan-suppressions.txt b/.evergreen/tsan-suppressions.txt new file mode 100644 index 0000000000..fbe534f5b4 --- /dev/null +++ b/.evergreen/tsan-suppressions.txt @@ -0,0 +1,14 @@ +# ThreadSanitizer suppressions for known-benign races inside CPython's own +# free-threaded interpreter internals (biased reference counting machinery +# in Python/brc.c and Python/object_stack.c). These surface because +# .evergreen/scripts/run-sanitizer-tests.sh LD_PRELOADs libtsan.so onto a +# prebuilt, non-TSan-instrumented free-threaded CPython interpreter rather +# than building the interpreter itself from source with -fsanitize=thread +# (which would make CI far slower). Without compile-time instrumentation on +# CPython's own atomics/locks, TSan cannot see their real synchronization +# and reports false positives in the interpreter's own biased-refcounting +# machinery. See PYTHON-6048. +# +# Reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions +race:_Py_brc_merge_refcounts +race:_Py_brc_queue_object From d8f11166de9e79caa4b76dd6851ffa880f5bfd06 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 12:46:25 -0500 Subject: [PATCH 16/26] PYTHON-6048 Wait for monitor executor to stop before resetting pool on close() TSan caught a data race: Server.close() could return, and the caller could then close monitor sockets, while the monitor's background executor was still mid socket.recv() on the same connection. gc_safe_close() only signals the executor to stop (it must stay non-blocking since it also runs from a GC weakref callback); it never waited for the executor to actually exit before proceeding to reset the pool. Join the executor with the same 1-second bound already used by periodic_executor.py's shutdown path in the three affected close() overrides (MonitorBase, Monitor, _RttMonitor) before touching the pool. --- pymongo/asynchronous/monitor.py | 12 ++++++++++++ pymongo/synchronous/monitor.py | 12 ++++++++++++ test/asynchronous/test_monitor.py | 27 +++++++++++++++++++++++++++ test/test_monitor.py | 27 +++++++++++++++++++++++++++ 4 files changed, 78 insertions(+) diff --git a/pymongo/asynchronous/monitor.py b/pymongo/asynchronous/monitor.py index 537d9ea7f5..c43834a9d3 100644 --- a/pymongo/asynchronous/monitor.py +++ b/pymongo/asynchronous/monitor.py @@ -105,6 +105,10 @@ async def close(self) -> None: open() restarts the monitor after closing. """ self.gc_safe_close() + # Wait briefly for the background task to actually stop before + # returning, so callers don't race with it while it may still be + # using a checked-out connection. + await self._executor.join(1) async def join(self) -> None: """Wait for the monitor to stop.""" @@ -192,6 +196,10 @@ async def join(self) -> None: async def close(self) -> None: self.gc_safe_close() await self._rtt_monitor.close() + # Wait briefly for the background task to actually stop before + # resetting the pool, so we don't race with it while it may still + # be using a checked-out connection. + await self._executor.join(1) # Increment the generation and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. await self._reset_connection() @@ -404,6 +412,10 @@ def __init__(self, topology: Topology, topology_settings: TopologySettings, pool async def close(self) -> None: self.gc_safe_close() + # Wait briefly for the background task to actually stop before + # resetting the pool, so we don't race with it while it may still + # be using a checked-out connection. + await self._executor.join(1) # Increment the generation and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. await self._pool.reset() diff --git a/pymongo/synchronous/monitor.py b/pymongo/synchronous/monitor.py index d56206b127..674b7c5948 100644 --- a/pymongo/synchronous/monitor.py +++ b/pymongo/synchronous/monitor.py @@ -105,6 +105,10 @@ def close(self) -> None: open() restarts the monitor after closing. """ self.gc_safe_close() + # Wait briefly for the background task to actually stop before + # returning, so callers don't race with it while it may still be + # using a checked-out connection. + self._executor.join(1) def join(self) -> None: """Wait for the monitor to stop.""" @@ -190,6 +194,10 @@ def join(self) -> None: def close(self) -> None: self.gc_safe_close() self._rtt_monitor.close() + # Wait briefly for the background task to actually stop before + # resetting the pool, so we don't race with it while it may still + # be using a checked-out connection. + self._executor.join(1) # Increment the generation and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. self._reset_connection() @@ -402,6 +410,10 @@ def __init__(self, topology: Topology, topology_settings: TopologySettings, pool def close(self) -> None: self.gc_safe_close() + # Wait briefly for the background task to actually stop before + # resetting the pool, so we don't race with it while it may still + # be using a checked-out connection. + self._executor.join(1) # Increment the generation and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. self._pool.reset() diff --git a/test/asynchronous/test_monitor.py b/test/asynchronous/test_monitor.py index 842c470b8f..2a814ad9e1 100644 --- a/test/asynchronous/test_monitor.py +++ b/test/asynchronous/test_monitor.py @@ -20,6 +20,7 @@ import gc import subprocess import sys +import time import warnings from functools import partial @@ -104,6 +105,32 @@ async def test_cleanup_executors_on_client_close(self): lambda: executor._stopped, f"closed executor: {executor._name}", timeout=5 ) + @async_client_context.require_sync + def test_close_stops_monitor_thread(self): + """PYTHON-6048: close() must join the monitor thread before + returning, so callers don't race with it while it may still be + reading from a socket that close() is about to tear down. + """ + client = self.create_client() + server = next(iter(client._topology._servers.values())) + monitor = server._monitor + + def thread_alive(executor): + try: + return executor._thread is not None and executor._thread.is_alive() + except ReferenceError: + return False + + deadline = time.monotonic() + 10 + while not thread_alive(monitor._executor) and time.monotonic() < deadline: + time.sleep(0.1) + self.assertTrue(thread_alive(monitor._executor), "monitor thread never started") + + client.close() + + self.assertFalse(thread_alive(monitor._executor)) + self.assertFalse(thread_alive(monitor._rtt_monitor._executor)) + @async_client_context.require_sync def test_no_thread_start_runtime_err_on_shutdown(self): """Test we silence noisy runtime errors fired when the AsyncMongoClient spawns a new thread diff --git a/test/test_monitor.py b/test/test_monitor.py index 3f24a5f2a2..929708824c 100644 --- a/test/test_monitor.py +++ b/test/test_monitor.py @@ -20,6 +20,7 @@ import gc import subprocess import sys +import time import warnings from functools import partial @@ -100,6 +101,32 @@ def test_cleanup_executors_on_client_close(self): for executor in executors: wait_until(lambda: executor._stopped, f"closed executor: {executor._name}", timeout=5) + @client_context.require_sync + def test_close_stops_monitor_thread(self): + """PYTHON-6048: close() must join the monitor thread before + returning, so callers don't race with it while it may still be + reading from a socket that close() is about to tear down. + """ + client = self.create_client() + server = next(iter(client._topology._servers.values())) + monitor = server._monitor + + def thread_alive(executor): + try: + return executor._thread is not None and executor._thread.is_alive() + except ReferenceError: + return False + + deadline = time.monotonic() + 10 + while not thread_alive(monitor._executor) and time.monotonic() < deadline: + time.sleep(0.1) + self.assertTrue(thread_alive(monitor._executor), "monitor thread never started") + + client.close() + + self.assertFalse(thread_alive(monitor._executor)) + self.assertFalse(thread_alive(monitor._rtt_monitor._executor)) + @client_context.require_sync def test_no_thread_start_runtime_err_on_shutdown(self): """Test we silence noisy runtime errors fired when the MongoClient spawns a new thread From d6b754a64352a7df6037d235bd3cc321e0d431ce Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 12:48:09 -0500 Subject: [PATCH 17/26] PYTHON-6048 Fix shellcheck SC2155 warning in sanitizer script --- .evergreen/scripts/run-sanitizer-tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index 24e80808d8..3b8f087dfb 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -37,7 +37,8 @@ case "$SANITIZER" in export CFLAGS="-fsanitize=thread -fno-omit-frame-pointer -g -O0" export LDFLAGS="-fsanitize=thread" RUNTIME_LIB=$("$CC" -print-file-name=libtsan.so) - export TSAN_OPTIONS="halt_on_error=1:suppressions=$(pwd)/.evergreen/tsan-suppressions.txt" + TSAN_OPTIONS="halt_on_error=1:suppressions=$(pwd)/.evergreen/tsan-suppressions.txt" + export TSAN_OPTIONS ;; *) echo "Unknown SANITIZER: $SANITIZER (expected 'asan' or 'tsan')" >&2 From 3b37ddb13a922a6692260a266221ea189b87f751 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 13:02:10 -0500 Subject: [PATCH 18/26] PYTHON-6048 Fix monitor close() latency and drop ineffective SrvMonitor join MonitorBase.close()'s join only ever runs for SrvMonitor, which has no pool/socket to race with, so it just added a guaranteed timeout to every mongodb+srv:// client close; drop it and leave the joins in Monitor.close() and _RttMonitor.close(), the classes that actually own the raced socket. Also wake() the executor there before joining so the thread rechecks the stop flag sooner instead of waiting out a full sleep chunk. --- pymongo/asynchronous/monitor.py | 26 ++++++++++++++++---------- pymongo/periodic_executor.py | 6 +++++- pymongo/synchronous/monitor.py | 26 ++++++++++++++++---------- 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/pymongo/asynchronous/monitor.py b/pymongo/asynchronous/monitor.py index c43834a9d3..322db52110 100644 --- a/pymongo/asynchronous/monitor.py +++ b/pymongo/asynchronous/monitor.py @@ -105,10 +105,6 @@ async def close(self) -> None: open() restarts the monitor after closing. """ self.gc_safe_close() - # Wait briefly for the background task to actually stop before - # returning, so callers don't race with it while it may still be - # using a checked-out connection. - await self._executor.join(1) async def join(self) -> None: """Wait for the monitor to stop.""" @@ -196,9 +192,14 @@ async def join(self) -> None: async def close(self) -> None: self.gc_safe_close() await self._rtt_monitor.close() - # Wait briefly for the background task to actually stop before - # resetting the pool, so we don't race with it while it may still - # be using a checked-out connection. + # Wake the executor so it notices the stop flag on its next sleep + # check, then wait briefly for the background task to actually stop + # before resetting the pool, so we don't race with it while it may + # still be using a checked-out connection. This join is a bounded + # backstop: if the task hasn't stopped by the timeout, the + # generation-based deferred close below still closes the socket + # once it's checked in. + self._executor.wake() await self._executor.join(1) # Increment the generation and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. @@ -412,9 +413,14 @@ def __init__(self, topology: Topology, topology_settings: TopologySettings, pool async def close(self) -> None: self.gc_safe_close() - # Wait briefly for the background task to actually stop before - # resetting the pool, so we don't race with it while it may still - # be using a checked-out connection. + # Wake the executor so it notices the stop flag on its next sleep + # check, then wait briefly for the background task to actually stop + # before resetting the pool, so we don't race with it while it may + # still be using a checked-out connection. This join is a bounded + # backstop: if the task hasn't stopped by the timeout, the + # generation-based deferred close below still closes the socket + # once it's checked in. + self._executor.wake() await self._executor.join(1) # Increment the generation and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. diff --git a/pymongo/periodic_executor.py b/pymongo/periodic_executor.py index 4b979ca9f9..f9f6463257 100644 --- a/pymongo/periodic_executor.py +++ b/pymongo/periodic_executor.py @@ -211,7 +211,11 @@ def join(self, timeout: Optional[int] = None) -> None: try: self._thread.join(timeout) except (ReferenceError, RuntimeError): - # Thread already terminated, or not yet started. + # Thread already terminated, or not yet started. This also + # covers a thread joining itself (RuntimeError: cannot join + # current thread), which happens when close() is called from + # within the monitor's own thread; that's safe to ignore + # since the thread is about to exit anyway. pass def wake(self) -> None: diff --git a/pymongo/synchronous/monitor.py b/pymongo/synchronous/monitor.py index 674b7c5948..a3074bf37f 100644 --- a/pymongo/synchronous/monitor.py +++ b/pymongo/synchronous/monitor.py @@ -105,10 +105,6 @@ def close(self) -> None: open() restarts the monitor after closing. """ self.gc_safe_close() - # Wait briefly for the background task to actually stop before - # returning, so callers don't race with it while it may still be - # using a checked-out connection. - self._executor.join(1) def join(self) -> None: """Wait for the monitor to stop.""" @@ -194,9 +190,14 @@ def join(self) -> None: def close(self) -> None: self.gc_safe_close() self._rtt_monitor.close() - # Wait briefly for the background task to actually stop before - # resetting the pool, so we don't race with it while it may still - # be using a checked-out connection. + # Wake the executor so it notices the stop flag on its next sleep + # check, then wait briefly for the background task to actually stop + # before resetting the pool, so we don't race with it while it may + # still be using a checked-out connection. This join is a bounded + # backstop: if the task hasn't stopped by the timeout, the + # generation-based deferred close below still closes the socket + # once it's checked in. + self._executor.wake() self._executor.join(1) # Increment the generation and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. @@ -410,9 +411,14 @@ def __init__(self, topology: Topology, topology_settings: TopologySettings, pool def close(self) -> None: self.gc_safe_close() - # Wait briefly for the background task to actually stop before - # resetting the pool, so we don't race with it while it may still - # be using a checked-out connection. + # Wake the executor so it notices the stop flag on its next sleep + # check, then wait briefly for the background task to actually stop + # before resetting the pool, so we don't race with it while it may + # still be using a checked-out connection. This join is a bounded + # backstop: if the task hasn't stopped by the timeout, the + # generation-based deferred close below still closes the socket + # once it's checked in. + self._executor.wake() self._executor.join(1) # Increment the generation and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. From 66db59543c7dc9f072a4dbadfb70f0f6583f022c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 13:43:00 -0500 Subject: [PATCH 19/26] PYTHON-6048 Extend TSan suppressions for TLBC and local-refcount internals --- .evergreen/tsan-suppressions.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.evergreen/tsan-suppressions.txt b/.evergreen/tsan-suppressions.txt index fbe534f5b4..17cc53fac0 100644 --- a/.evergreen/tsan-suppressions.txt +++ b/.evergreen/tsan-suppressions.txt @@ -1,14 +1,18 @@ # ThreadSanitizer suppressions for known-benign races inside CPython's own # free-threaded interpreter internals (biased reference counting machinery -# in Python/brc.c and Python/object_stack.c). These surface because +# in Python/brc.c and Python/object_stack.c, thread-local bytecode caching +# in Objects/codeobject.c, and per-object local-refcount-table resizing in +# Python/uniqueid.c). These surface because # .evergreen/scripts/run-sanitizer-tests.sh LD_PRELOADs libtsan.so onto a # prebuilt, non-TSan-instrumented free-threaded CPython interpreter rather # than building the interpreter itself from source with -fsanitize=thread # (which would make CI far slower). Without compile-time instrumentation on # CPython's own atomics/locks, TSan cannot see their real synchronization -# and reports false positives in the interpreter's own biased-refcounting -# machinery. See PYTHON-6048. +# and reports false positives in the interpreter's own internal machinery. +# See PYTHON-6048. # # Reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions race:_Py_brc_merge_refcounts race:_Py_brc_queue_object +race:create_tlbc_lock_held +race:resize_local_refcounts From 470b7723cf97434af6b83b25a3369e1df88773de Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 13:56:52 -0500 Subject: [PATCH 20/26] PYTHON-6048 Suppress free_delayed TSan false positive in free-threading GC internals --- .evergreen/tsan-suppressions.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.evergreen/tsan-suppressions.txt b/.evergreen/tsan-suppressions.txt index 17cc53fac0..4ff67c23c4 100644 --- a/.evergreen/tsan-suppressions.txt +++ b/.evergreen/tsan-suppressions.txt @@ -1,8 +1,10 @@ # ThreadSanitizer suppressions for known-benign races inside CPython's own # free-threaded interpreter internals (biased reference counting machinery # in Python/brc.c and Python/object_stack.c, thread-local bytecode caching -# in Objects/codeobject.c, and per-object local-refcount-table resizing in -# Python/uniqueid.c). These surface because +# in Objects/codeobject.c, per-object local-refcount-table resizing in +# Python/uniqueid.c, and CPython's various free-threading memory-reclamation +# mechanisms, e.g. the QSBR-style deferred-free path in free_delayed). +# These surface because # .evergreen/scripts/run-sanitizer-tests.sh LD_PRELOADs libtsan.so onto a # prebuilt, non-TSan-instrumented free-threaded CPython interpreter rather # than building the interpreter itself from source with -fsanitize=thread @@ -16,3 +18,4 @@ race:_Py_brc_merge_refcounts race:_Py_brc_queue_object race:create_tlbc_lock_held race:resize_local_refcounts +race:free_delayed From 820287734dc9cad8578d444a88f4d7f4f89801c5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 14:37:35 -0500 Subject: [PATCH 21/26] PYTHON-6048 Build a TSan-instrumented CPython for the TSan task The TSan task LD_PRELOADed libtsan.so onto a prebuilt, uninstrumented free-threaded interpreter, which reported false races in CPython's own free-threading internals because TSan cannot see synchronization in code compiled without -fsanitize=thread. It now builds CPython 3.14 from source with --with-thread-sanitizer, matching CPython's own CI, so all five suppressions come out. pip keeps build isolation so it resolves hatchling's build dependencies itself; CFLAGS and LDFLAGS are shell env vars, which the isolated build subprocess inherits. The task gets a 7200 second exec timeout because the source build does not fit in the project-wide 3600. --- .evergreen/generated_configs/tasks.yml | 1 + .evergreen/scripts/generate_config.py | 13 ++- .evergreen/scripts/run-sanitizer-tests.sh | 104 +++++++++++++++++++--- .evergreen/tsan-suppressions.txt | 34 ++++--- 4 files changed, 119 insertions(+), 33 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 0538a30c3f..bce51d8cf5 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -2920,6 +2920,7 @@ tasks: vars: SANITIZER: tsan UV_PYTHON: 3.14t + exec_timeout_secs: 7200 tags: [sanitizer, pr, free-threaded] # Search index tests diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index fdb4cffcad..cdb05b3d93 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -680,7 +680,18 @@ def create_sanitizer_tasks(): tags.append("free-threaded") test_func = FunctionCall(func="run sanitizer tests", vars=test_vars) name = f"test-sanitizer-{sanitizer}" - tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) + # TSan builds a fully instrumented free-threaded CPython from source + # before it can run anything, which does not fit in the project-wide + # 60 minute exec timeout. + exec_timeout_secs = 7200 if sanitizer == "tsan" else None + tasks.append( + EvgTask( + name=name, + tags=tags, + exec_timeout_secs=exec_timeout_secs, + commands=[server_func, test_func], + ) + ) return tasks diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index 3b8f087dfb..f0139071cd 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -2,6 +2,12 @@ # Clean sanitizer rebuild of the C extensions, run the fixed BSON/client # test files under the matching sanitizer runtime, and fail on any # sanitizer diagnostic even if pytest itself exits 0. +# +# ASan runs against a prebuilt interpreter with libasan.so LD_PRELOADed. +# TSan builds a fully instrumented free-threaded CPython from source +# instead: TSan cannot see synchronization in code compiled without +# -fsanitize=thread, so LD_PRELOADing it onto a prebuilt interpreter +# reports false races inside CPython's own free-threading internals. set -eu cd "$(dirname "${BASH_SOURCE[0]}")/../.." @@ -10,11 +16,16 @@ SANITIZER=${SANITIZER:?"SANITIZER must be set to 'asan' or 'tsan'"} UV_PYTHON=${UV_PYTHON:-3.13} TEST_FILES=(test/test_bson.py test/test_raw_bson.py test/test_raw_bson_shared.py test/test_client.py) +# The free-threaded CPython the TSan task builds. Kept in step with the +# "3.14t" entry in .evergreen/scripts/generate_config_utils.py's CPYTHONS. +CPYTHON_TAG=v3.14.0 +CPYTHON_SRC=.tsan-cpython-src +CPYTHON_INSTALL=.tsan-cpython-install + # A stale build/ or venv can leave a .so or install linked against the wrong # sanitizer's runtime without a build error, so always start clean. rm -rf build rm -f bson/*.so pymongo/*.so -rm -rf .sanitizer-venv export CC=${CC:-clang} export CXX=${CXX:-clang++} @@ -22,6 +33,7 @@ export PYMONGO_C_EXT_MUST_BUILD=1 case "$SANITIZER" in asan) + rm -rf .sanitizer-venv export CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g -O0" export LDFLAGS="-fsanitize=address,undefined" RUNTIME_LIB=$("$CC" -print-file-name=libasan.so) @@ -32,13 +44,89 @@ case "$SANITIZER" in # noise. The free-threaded TSan interpreter doesn't accept this value, # so it's scoped to ASan only. export PYTHONMALLOC=malloc + + if [ ! -f "$RUNTIME_LIB" ]; then + echo "Could not locate the $SANITIZER runtime library (got: $RUNTIME_LIB). Is the matching sanitizer runtime package installed?" >&2 + exit 1 + fi + + uv venv --python "$UV_PYTHON" .sanitizer-venv + VENV_PYTHON=.sanitizer-venv/bin/python3 + uv pip install --python "$VENV_PYTHON" -e . --reinstall + uv pip install --python "$VENV_PYTHON" -r requirements/test.txt + + PYTEST_CMD=(env "LD_PRELOAD=$RUNTIME_LIB" "$VENV_PYTHON" -m pytest) ;; tsan) + rm -rf "$CPYTHON_SRC" "$CPYTHON_INSTALL" + + # TSan's shadow mapping can fail to reserve its address ranges under the + # default ASLR entropy on recent kernels, which crashes the process at + # startup. CPython's own CI lowers the entropy the same way. Evergreen + # hosts may not permit it, so this is best effort. + sudo sysctl -w vm.mmap_rnd_bits=28 || true + + CPYTHON_INSTALL_ABS="$(pwd)/$CPYTHON_INSTALL" + git clone --depth 1 --branch "$CPYTHON_TAG" https://github.com/python/cpython.git "$CPYTHON_SRC" + # Flags mirror CPython's own TSan CI job (.github/workflows/reusable-san.yml). + # CFLAGS/LDFLAGS are deliberately left unset here: --with-thread-sanitizer + # and --with-pydebug already supply the right flags, and overriding them + # would fight configure. + # + # Unlike CPython's CI this does not rebuild OpenSSL with TSan, which is + # only needed to keep the ssl tests quiet. The tests below don't use ssl, + # but pip does need it to reach PyPI, so _ssl still has to build against + # the system OpenSSL. + ( + cd "$CPYTHON_SRC" + ./configure \ + --with-thread-sanitizer \ + --with-pydebug \ + --disable-gil \ + --prefix="$CPYTHON_INSTALL_ABS" + make -j"$(nproc 2>/dev/null || echo 4)" + make install + ) + + # Free-threaded builds install as pythonX.Yt; fall back to python3 in case + # a future release drops the suffix. + TSAN_PYTHON="" + for candidate in "$CPYTHON_INSTALL"/bin/python3.*t "$CPYTHON_INSTALL"/bin/python3; do + if [ -x "$candidate" ]; then + TSAN_PYTHON="$candidate" + break + fi + done + if [ -z "$TSAN_PYTHON" ]; then + echo "Could not find the interpreter built from source under $CPYTHON_INSTALL/bin:" >&2 + ls -l "$CPYTHON_INSTALL/bin" >&2 || true + exit 1 + fi + echo "Using TSan-instrumented interpreter: $TSAN_PYTHON" + "$TSAN_PYTHON" -VV + "$TSAN_PYTHON" -c 'import sysconfig, sys; sys.exit(0 if sysconfig.get_config_var("Py_GIL_DISABLED") else "interpreter is not free-threaded")' + if ! "$TSAN_PYTHON" -c 'import ssl' >/dev/null 2>&1; then + echo "Warning: the interpreter built from source has no working ssl module, so pip cannot reach PyPI. Install the system OpenSSL development headers on this host." >&2 + fi + + # Build PyMongo's C extensions with the same instrumentation as the + # interpreter. These are plain shell env vars so pip's isolated build + # subprocess inherits them; build isolation is left on so pip resolves + # hatchling's build dependencies itself. export CFLAGS="-fsanitize=thread -fno-omit-frame-pointer -g -O0" export LDFLAGS="-fsanitize=thread" - RUNTIME_LIB=$("$CC" -print-file-name=libtsan.so) + "$TSAN_PYTHON" -m ensurepip --upgrade + "$TSAN_PYTHON" -m pip install -e . + "$TSAN_PYTHON" -m pip install -r requirements/test.txt + + # Set after the build: halt_on_error=1 would abort the CPython build and + # the installs on any diagnostic raised by those tools themselves. TSAN_OPTIONS="halt_on_error=1:suppressions=$(pwd)/.evergreen/tsan-suppressions.txt" export TSAN_OPTIONS + + # No LD_PRELOAD: both the interpreter and the extensions link the TSan + # runtime at build time. + PYTEST_CMD=("$TSAN_PYTHON" -m pytest) ;; *) echo "Unknown SANITIZER: $SANITIZER (expected 'asan' or 'tsan')" >&2 @@ -46,19 +134,9 @@ case "$SANITIZER" in ;; esac -if [ ! -f "$RUNTIME_LIB" ]; then - echo "Could not locate the $SANITIZER runtime library (got: $RUNTIME_LIB). Is the matching sanitizer runtime package installed?" >&2 - exit 1 -fi - -uv venv --python "$UV_PYTHON" .sanitizer-venv -VENV_PYTHON=.sanitizer-venv/bin/python3 -uv pip install --python "$VENV_PYTHON" -e . --reinstall -uv pip install --python "$VENV_PYTHON" -r requirements/test.txt - LOG_FILE=$(mktemp) set +e -LD_PRELOAD="$RUNTIME_LIB" "$VENV_PYTHON" -m pytest -v --capture=no "${TEST_FILES[@]}" 2>&1 | tee "$LOG_FILE" +"${PYTEST_CMD[@]}" -v --capture=no "${TEST_FILES[@]}" 2>&1 | tee "$LOG_FILE" PYTEST_STATUS=${PIPESTATUS[0]} set -e diff --git a/.evergreen/tsan-suppressions.txt b/.evergreen/tsan-suppressions.txt index 4ff67c23c4..a311331455 100644 --- a/.evergreen/tsan-suppressions.txt +++ b/.evergreen/tsan-suppressions.txt @@ -1,21 +1,17 @@ -# ThreadSanitizer suppressions for known-benign races inside CPython's own -# free-threaded interpreter internals (biased reference counting machinery -# in Python/brc.c and Python/object_stack.c, thread-local bytecode caching -# in Objects/codeobject.c, per-object local-refcount-table resizing in -# Python/uniqueid.c, and CPython's various free-threading memory-reclamation -# mechanisms, e.g. the QSBR-style deferred-free path in free_delayed). -# These surface because -# .evergreen/scripts/run-sanitizer-tests.sh LD_PRELOADs libtsan.so onto a -# prebuilt, non-TSan-instrumented free-threaded CPython interpreter rather -# than building the interpreter itself from source with -fsanitize=thread -# (which would make CI far slower). Without compile-time instrumentation on -# CPython's own atomics/locks, TSan cannot see their real synchronization -# and reports false positives in the interpreter's own internal machinery. -# See PYTHON-6048. +# ThreadSanitizer suppressions for this driver's TSan CI task. +# +# As of PYTHON-6048, the TSan task builds a fully TSan-instrumented +# free-threaded CPython interpreter from source (matching CPython's own +# CI: https://github.com/python/cpython/blob/main/.github/workflows/reusable-san.yml), +# rather than LD_PRELOADing libtsan.so onto a prebuilt, uninstrumented +# interpreter. That earlier approach produced several false positives +# inside CPython's own free-threading internals (biased refcounting, +# thread-local bytecode caching, deferred-free reclamation, etc.) +# because TSan cannot see synchronization in code that wasn't compiled +# with -fsanitize=thread. With full instrumentation, those specific +# findings should not recur — matching CPython's own +# Tools/tsan/suppressions_free_threading.txt, which is also kept empty +# for the same reason. This file is kept as a placeholder for any +# genuine future finding that needs a narrow, justified suppression. # # Reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions -race:_Py_brc_merge_refcounts -race:_Py_brc_queue_object -race:create_tlbc_lock_held -race:resize_local_refcounts -race:free_delayed From 54a0fedf4b8e8dc80f0b967d70dae085305a97a4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 14:39:19 -0500 Subject: [PATCH 22/26] PYTHON-6048 Install CPython build dependencies before TSan source build --- .evergreen/scripts/run-sanitizer-tests.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index f0139071cd..aee9453c6d 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -66,6 +66,16 @@ case "$SANITIZER" in # hosts may not permit it, so this is best effort. sudo sysctl -w vm.mmap_rnd_bits=28 || true + # Building CPython from source needs its usual dev dependencies — + # OpenSSL headers in particular, since pip needs a working ssl module + # to reach PyPI for the installs below. Best effort: if apt-get isn't + # available or permitted, the ssl-module check further down will warn + # instead of silently failing the pip installs. + sudo apt-get update -qq || true + sudo apt-get install -y --no-install-recommends \ + build-essential libssl-dev zlib1g-dev libbz2-dev libffi-dev \ + libreadline-dev libsqlite3-dev liblzma-dev || true + CPYTHON_INSTALL_ABS="$(pwd)/$CPYTHON_INSTALL" git clone --depth 1 --branch "$CPYTHON_TAG" https://github.com/python/cpython.git "$CPYTHON_SRC" # Flags mirror CPython's own TSan CI job (.github/workflows/reusable-san.yml). From 712f870f70e57435b03efc74ac5495b240128105 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 14:51:20 -0500 Subject: [PATCH 23/26] PYTHON-6048 Combine CPython's own TSan suppressions with pymongo's The suppressions file was emptied on the assumption that upstream's Tools/tsan/suppressions_free_threading.txt is empty for an instrumented build. That is true on CPython main but not on the pinned v3.14.0 tag, where it carries 24 entries that upstream's own TSan CI needs. Build a combined file from the cloned tag's copy plus this repo's additions and point TSAN_OPTIONS at that, with handle_segv=0 to match upstream. Also fix the interpreter glob, which missed the real python3.14td name that --disable-gil plus --with-pydebug produces; extend apt-get to CPython's official dependency list; and abort before the 20-30 minute build when the OpenSSL headers pip needs are missing. --- .evergreen/scripts/run-sanitizer-tests.sh | 44 +++++++++++++++++------ .evergreen/tsan-suppressions.txt | 26 +++++++------- .gitignore | 6 ++++ 3 files changed, 51 insertions(+), 25 deletions(-) diff --git a/.evergreen/scripts/run-sanitizer-tests.sh b/.evergreen/scripts/run-sanitizer-tests.sh index aee9453c6d..4b15b2a24b 100755 --- a/.evergreen/scripts/run-sanitizer-tests.sh +++ b/.evergreen/scripts/run-sanitizer-tests.sh @@ -66,18 +66,40 @@ case "$SANITIZER" in # hosts may not permit it, so this is best effort. sudo sysctl -w vm.mmap_rnd_bits=28 || true - # Building CPython from source needs its usual dev dependencies — - # OpenSSL headers in particular, since pip needs a working ssl module - # to reach PyPI for the installs below. Best effort: if apt-get isn't - # available or permitted, the ssl-module check further down will warn - # instead of silently failing the pip installs. + # Building CPython from source needs its usual dev dependencies. Best + # effort: if apt-get isn't available or permitted, the OpenSSL header + # check below still catches the case that breaks the pip installs. sudo apt-get update -qq || true + # Package list matches CPython's own Doc/using/unix.rst and + # Tools/scripts/posix-deps-apt.sh, so the optional extension modules + # (_zstd, _gdbm, _tkinter, ...) build instead of silently skipping. sudo apt-get install -y --no-install-recommends \ build-essential libssl-dev zlib1g-dev libbz2-dev libffi-dev \ - libreadline-dev libsqlite3-dev liblzma-dev || true + libreadline-dev libsqlite3-dev liblzma-dev pkg-config libb2-dev \ + libgdbm-dev libgdbm-compat-dev libncurses5-dev libzstd-dev tk-dev \ + uuid-dev curl || true + + # Fail here rather than after the 20-30 minute build: without these + # headers _ssl won't build and the pip installs below cannot reach PyPI. + if [ ! -f /usr/include/openssl/ssl.h ]; then + echo "OpenSSL development headers not found at /usr/include/openssl/ssl.h after apt-get install. pip needs a working ssl module to reach PyPI. Aborting before the CPython build." >&2 + exit 1 + fi CPYTHON_INSTALL_ABS="$(pwd)/$CPYTHON_INSTALL" git clone --depth 1 --branch "$CPYTHON_TAG" https://github.com/python/cpython.git "$CPYTHON_SRC" + + # TSan accepts one suppressions file, so concatenate the pinned tag's own + # suppressions with this repo's additions. The 3.14 branch's file is not + # empty: even a fully instrumented build needs its ~24 entries. + CPYTHON_SUPPRESSIONS="$CPYTHON_SRC/Tools/tsan/suppressions_free_threading.txt" + if [ ! -f "$CPYTHON_SUPPRESSIONS" ]; then + echo "Expected CPython's own TSan suppressions at $CPYTHON_SUPPRESSIONS, but the file is missing from the $CPYTHON_TAG source tree." >&2 + exit 1 + fi + COMBINED_SUPPRESSIONS="$(pwd)/.tsan-suppressions-combined.txt" + cat "$CPYTHON_SUPPRESSIONS" .evergreen/tsan-suppressions.txt > "$COMBINED_SUPPRESSIONS" + # Flags mirror CPython's own TSan CI job (.github/workflows/reusable-san.yml). # CFLAGS/LDFLAGS are deliberately left unset here: --with-thread-sanitizer # and --with-pydebug already supply the right flags, and overriding them @@ -98,10 +120,10 @@ case "$SANITIZER" in make install ) - # Free-threaded builds install as pythonX.Yt; fall back to python3 in case - # a future release drops the suffix. + # --disable-gil adds a "t" suffix and --with-pydebug adds a "d", so this + # build installs as pythonX.Ytd. bin/python3 is the last-resort fallback. TSAN_PYTHON="" - for candidate in "$CPYTHON_INSTALL"/bin/python3.*t "$CPYTHON_INSTALL"/bin/python3; do + for candidate in "$CPYTHON_INSTALL"/bin/python3.*td "$CPYTHON_INSTALL"/bin/python3.*t "$CPYTHON_INSTALL"/bin/python3; do if [ -x "$candidate" ]; then TSAN_PYTHON="$candidate" break @@ -125,13 +147,13 @@ case "$SANITIZER" in # hatchling's build dependencies itself. export CFLAGS="-fsanitize=thread -fno-omit-frame-pointer -g -O0" export LDFLAGS="-fsanitize=thread" - "$TSAN_PYTHON" -m ensurepip --upgrade "$TSAN_PYTHON" -m pip install -e . "$TSAN_PYTHON" -m pip install -r requirements/test.txt # Set after the build: halt_on_error=1 would abort the CPython build and # the installs on any diagnostic raised by those tools themselves. - TSAN_OPTIONS="halt_on_error=1:suppressions=$(pwd)/.evergreen/tsan-suppressions.txt" + # handle_segv=0 matches CPython's own TSan CI job. + TSAN_OPTIONS="halt_on_error=1:handle_segv=0:suppressions=$COMBINED_SUPPRESSIONS" export TSAN_OPTIONS # No LD_PRELOAD: both the interpreter and the extensions link the TSan diff --git a/.evergreen/tsan-suppressions.txt b/.evergreen/tsan-suppressions.txt index a311331455..e0d0bdca57 100644 --- a/.evergreen/tsan-suppressions.txt +++ b/.evergreen/tsan-suppressions.txt @@ -1,17 +1,15 @@ -# ThreadSanitizer suppressions for this driver's TSan CI task. +# PyMongo-specific ThreadSanitizer suppressions. Currently none are needed. # -# As of PYTHON-6048, the TSan task builds a fully TSan-instrumented -# free-threaded CPython interpreter from source (matching CPython's own -# CI: https://github.com/python/cpython/blob/main/.github/workflows/reusable-san.yml), -# rather than LD_PRELOADing libtsan.so onto a prebuilt, uninstrumented -# interpreter. That earlier approach produced several false positives -# inside CPython's own free-threading internals (biased refcounting, -# thread-local bytecode caching, deferred-free reclamation, etc.) -# because TSan cannot see synchronization in code that wasn't compiled -# with -fsanitize=thread. With full instrumentation, those specific -# findings should not recur — matching CPython's own -# Tools/tsan/suppressions_free_threading.txt, which is also kept empty -# for the same reason. This file is kept as a placeholder for any -# genuine future finding that needs a narrow, justified suppression. +# .evergreen/scripts/run-sanitizer-tests.sh concatenates this file with the +# pinned CPython tag's own Tools/tsan/suppressions_free_threading.txt and +# passes the result as TSan's single suppressions= path. That upstream file +# is not empty on the 3.14 branch: it carries roughly two dozen entries +# (assign_version_tag, update_one_slot, _PyFrame_GetCode, rangeiter_next, +# list_ass_slice_lock_held, PyObject_Realloc, mi_block_set_nextx, +# pthread_create, and others) that CPython's own fully instrumented TSan CI +# job needs to run clean. Do not duplicate those here. +# +# Add an entry below only for a race in PyMongo's own C extensions that has +# been investigated and judged benign, with a comment saying why. # # Reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions diff --git a/.gitignore b/.gitignore index 6c4a512018..d637851c9f 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,9 @@ xunit-results/ coverage.xml server.log .coverage + +# sanitizer test task scratch (see .evergreen/scripts/run-sanitizer-tests.sh) +.sanitizer-venv/ +.tsan-cpython-src/ +.tsan-cpython-install/ +.tsan-suppressions-combined.txt From 2f32ceea96db617a579242056545b76c6dac5cbe Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 15:51:51 -0500 Subject: [PATCH 24/26] PYTHON-6048 Join closed monitors outside the topology lock Monitor.close() and _RttMonitor.close() no longer block. Joining there burned up to a second while Topology.close() and _update_servers() held the topology lock, stalling every thread in server selection. Instead, extend the existing _monitor_tasks deferral to sync: closed monitors are queued under the lock and joined by cleanup_monitors() once it is released, from Topology.close() and the next select_servers(). Monitor.join() and cleanup_monitors() drop asyncio.gather, which synchro could not translate and which left a broken call in the generated sync monitor. --- pymongo/asynchronous/monitor.py | 30 +++++----------------------- pymongo/asynchronous/topology.py | 33 ++++++++++++++++++++----------- pymongo/synchronous/monitor.py | 28 +++++--------------------- pymongo/synchronous/topology.py | 33 ++++++++++++++++++++----------- test/asynchronous/test_monitor.py | 21 ++++++++++++++++++++ test/test_monitor.py | 21 ++++++++++++++++++++ 6 files changed, 94 insertions(+), 72 deletions(-) diff --git a/pymongo/asynchronous/monitor.py b/pymongo/asynchronous/monitor.py index 322db52110..45c950404f 100644 --- a/pymongo/asynchronous/monitor.py +++ b/pymongo/asynchronous/monitor.py @@ -16,7 +16,6 @@ from __future__ import annotations -import asyncio import atexit import time import weakref @@ -106,9 +105,9 @@ async def close(self) -> None: """ self.gc_safe_close() - async def join(self) -> None: + async def join(self, timeout: Optional[int] = None) -> None: """Wait for the monitor to stop.""" - await self._executor.join() + await self._executor.join(timeout) def request_check(self) -> None: """If the monitor is sleeping, wake it soon.""" @@ -184,23 +183,13 @@ def gc_safe_close(self) -> None: self._rtt_monitor.gc_safe_close() self.cancel_check() - async def join(self) -> None: - await asyncio.gather( - self._executor.join(), self._rtt_monitor.join(), return_exceptions=True - ) # type: ignore[func-returns-value] + async def join(self, timeout: Optional[int] = None) -> None: + await self._executor.join(timeout) + await self._rtt_monitor.join(timeout) async def close(self) -> None: self.gc_safe_close() await self._rtt_monitor.close() - # Wake the executor so it notices the stop flag on its next sleep - # check, then wait briefly for the background task to actually stop - # before resetting the pool, so we don't race with it while it may - # still be using a checked-out connection. This join is a bounded - # backstop: if the task hasn't stopped by the timeout, the - # generation-based deferred close below still closes the socket - # once it's checked in. - self._executor.wake() - await self._executor.join(1) # Increment the generation and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. await self._reset_connection() @@ -413,15 +402,6 @@ def __init__(self, topology: Topology, topology_settings: TopologySettings, pool async def close(self) -> None: self.gc_safe_close() - # Wake the executor so it notices the stop flag on its next sleep - # check, then wait briefly for the background task to actually stop - # before resetting the pool, so we don't race with it while it may - # still be using a checked-out connection. This join is a bounded - # backstop: if the task hasn't stopped by the timeout, the - # generation-based deferred close below still closes the socket - # once it's checked in. - self._executor.wake() - await self._executor.join(1) # Increment the generation and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. await self._pool.reset() diff --git a/pymongo/asynchronous/topology.py b/pymongo/asynchronous/topology.py index c4f1a3fa96..0ef1f28471 100644 --- a/pymongo/asynchronous/topology.py +++ b/pymongo/asynchronous/topology.py @@ -16,7 +16,6 @@ from __future__ import annotations -import asyncio import os import queue import random @@ -241,8 +240,9 @@ async def select_servers( else: server_timeout = server_selection_timeout - # Cleanup any completed monitor tasks safely - if not _IS_SYNC and self._monitor_tasks: + # Join any monitors closed under the topology lock. Doing it here, + # outside the lock, keeps the join off the paths that hold it. + if self._monitor_tasks: await self.cleanup_monitors() async with self._lock: @@ -454,8 +454,7 @@ async def _process_change( and self._description.topology_type not in SRV_POLLING_TOPOLOGIES ): await self._srv_monitor.close() - if not _IS_SYNC: - self._monitor_tasks.append(self._srv_monitor) + self._monitor_tasks.append(self._srv_monitor) # Wake anything waiting in select_servers(). self._condition.notify_all() @@ -614,8 +613,7 @@ async def close(self) -> None: old_td = self._description for server in self._servers.values(): await server.close() - if not _IS_SYNC: - self._monitor_tasks.append(server._monitor) + self._monitor_tasks.append(server._monitor) # Mark all servers Unknown. self._description = self._description.reset() @@ -626,12 +624,14 @@ async def close(self) -> None: # Stop SRV polling thread. if self._srv_monitor: await self._srv_monitor.close() - if not _IS_SYNC: - self._monitor_tasks.append(self._srv_monitor) + self._monitor_tasks.append(self._srv_monitor) self._opened = False self._closed = True + # Join the monitors we just closed, now that the lock is released. + await self.cleanup_monitors() + # Publish only after releasing the lock. if self._sdam._publish_tp: self._description = TopologyDescription( @@ -847,8 +847,7 @@ async def _update_servers(self) -> None: for address, server in list(self._servers.items()): if not self._description.has_server(address): await server.close() - if not _IS_SYNC: - self._monitor_tasks.append(server._monitor) + self._monitor_tasks.append(server._monitor) self._servers.pop(address) def _create_pool_for_server(self, address: _Address) -> Pool: @@ -934,13 +933,23 @@ def _error_message(self, selector: Callable[[Selection], Selection]) -> str: return ",".join(str(server.error) for server in servers if server.error) async def cleanup_monitors(self) -> None: + """Join monitors closed earlier while the topology lock was held. + + Always call this with the lock released: a monitor being joined may + itself be blocked acquiring the lock. + """ tasks = [] try: while self._monitor_tasks: tasks.append(self._monitor_tasks.pop()) except IndexError: pass - await asyncio.gather(*[t.join() for t in tasks], return_exceptions=True) # type: ignore[func-returns-value] + for t in tasks: + try: + await t.join(1) + except Exception: # noqa: S110 + # One monitor failing to stop must not block the rest. + pass def __repr__(self) -> str: msg = "" diff --git a/pymongo/synchronous/monitor.py b/pymongo/synchronous/monitor.py index a3074bf37f..8a39ae3603 100644 --- a/pymongo/synchronous/monitor.py +++ b/pymongo/synchronous/monitor.py @@ -16,7 +16,6 @@ from __future__ import annotations -import asyncio import atexit import time import weakref @@ -106,9 +105,9 @@ def close(self) -> None: """ self.gc_safe_close() - def join(self) -> None: + def join(self, timeout: Optional[int] = None) -> None: """Wait for the monitor to stop.""" - self._executor.join() + self._executor.join(timeout) def request_check(self) -> None: """If the monitor is sleeping, wake it soon.""" @@ -184,21 +183,13 @@ def gc_safe_close(self) -> None: self._rtt_monitor.gc_safe_close() self.cancel_check() - def join(self) -> None: - asyncio.gather(self._executor.join(), self._rtt_monitor.join(), return_exceptions=True) # type: ignore[func-returns-value] + def join(self, timeout: Optional[int] = None) -> None: + self._executor.join(timeout) + self._rtt_monitor.join(timeout) def close(self) -> None: self.gc_safe_close() self._rtt_monitor.close() - # Wake the executor so it notices the stop flag on its next sleep - # check, then wait briefly for the background task to actually stop - # before resetting the pool, so we don't race with it while it may - # still be using a checked-out connection. This join is a bounded - # backstop: if the task hasn't stopped by the timeout, the - # generation-based deferred close below still closes the socket - # once it's checked in. - self._executor.wake() - self._executor.join(1) # Increment the generation and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. self._reset_connection() @@ -411,15 +402,6 @@ def __init__(self, topology: Topology, topology_settings: TopologySettings, pool def close(self) -> None: self.gc_safe_close() - # Wake the executor so it notices the stop flag on its next sleep - # check, then wait briefly for the background task to actually stop - # before resetting the pool, so we don't race with it while it may - # still be using a checked-out connection. This join is a bounded - # backstop: if the task hasn't stopped by the timeout, the - # generation-based deferred close below still closes the socket - # once it's checked in. - self._executor.wake() - self._executor.join(1) # Increment the generation and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. self._pool.reset() diff --git a/pymongo/synchronous/topology.py b/pymongo/synchronous/topology.py index c6468c8912..367774f663 100644 --- a/pymongo/synchronous/topology.py +++ b/pymongo/synchronous/topology.py @@ -16,7 +16,6 @@ from __future__ import annotations -import asyncio import os import queue import random @@ -241,8 +240,9 @@ def select_servers( else: server_timeout = server_selection_timeout - # Cleanup any completed monitor tasks safely - if not _IS_SYNC and self._monitor_tasks: + # Join any monitors closed under the topology lock. Doing it here, + # outside the lock, keeps the join off the paths that hold it. + if self._monitor_tasks: self.cleanup_monitors() with self._lock: @@ -454,8 +454,7 @@ def _process_change( and self._description.topology_type not in SRV_POLLING_TOPOLOGIES ): self._srv_monitor.close() - if not _IS_SYNC: - self._monitor_tasks.append(self._srv_monitor) + self._monitor_tasks.append(self._srv_monitor) # Wake anything waiting in select_servers(). self._condition.notify_all() @@ -612,8 +611,7 @@ def close(self) -> None: old_td = self._description for server in self._servers.values(): server.close() - if not _IS_SYNC: - self._monitor_tasks.append(server._monitor) + self._monitor_tasks.append(server._monitor) # Mark all servers Unknown. self._description = self._description.reset() @@ -624,12 +622,14 @@ def close(self) -> None: # Stop SRV polling thread. if self._srv_monitor: self._srv_monitor.close() - if not _IS_SYNC: - self._monitor_tasks.append(self._srv_monitor) + self._monitor_tasks.append(self._srv_monitor) self._opened = False self._closed = True + # Join the monitors we just closed, now that the lock is released. + self.cleanup_monitors() + # Publish only after releasing the lock. if self._sdam._publish_tp: self._description = TopologyDescription( @@ -845,8 +845,7 @@ def _update_servers(self) -> None: for address, server in list(self._servers.items()): if not self._description.has_server(address): server.close() - if not _IS_SYNC: - self._monitor_tasks.append(server._monitor) + self._monitor_tasks.append(server._monitor) self._servers.pop(address) def _create_pool_for_server(self, address: _Address) -> Pool: @@ -932,13 +931,23 @@ def _error_message(self, selector: Callable[[Selection], Selection]) -> str: return ",".join(str(server.error) for server in servers if server.error) def cleanup_monitors(self) -> None: + """Join monitors closed earlier while the topology lock was held. + + Always call this with the lock released: a monitor being joined may + itself be blocked acquiring the lock. + """ tasks = [] try: while self._monitor_tasks: tasks.append(self._monitor_tasks.pop()) except IndexError: pass - asyncio.gather(*[t.join() for t in tasks], return_exceptions=True) # type: ignore[func-returns-value] + for t in tasks: + try: + t.join(1) + except Exception: # noqa: S110 + # One monitor failing to stop must not block the rest. + pass def __repr__(self) -> str: msg = "" diff --git a/test/asynchronous/test_monitor.py b/test/asynchronous/test_monitor.py index 2a814ad9e1..25c332cbf4 100644 --- a/test/asynchronous/test_monitor.py +++ b/test/asynchronous/test_monitor.py @@ -128,9 +128,30 @@ def thread_alive(executor): client.close() + # close() drains the deferred-join queue after releasing the + # topology lock, so every monitor it closed has been joined. + self.assertEqual(client._topology._monitor_tasks, []) self.assertFalse(thread_alive(monitor._executor)) self.assertFalse(thread_alive(monitor._rtt_monitor._executor)) + @async_client_context.require_async + async def test_close_stops_monitor_task(self): + """PYTHON-6048: the async counterpart of + test_close_stops_monitor_thread. + """ + client = await self.create_client() + server = next(iter(client._topology._servers.values())) + monitor = server._monitor + + self.assertIsNotNone(monitor._executor._task) + self.assertFalse(monitor._executor._task.done()) + + await client.close() + + self.assertEqual(client._topology._monitor_tasks, []) + self.assertTrue(monitor._executor._task.done()) + self.assertTrue(monitor._rtt_monitor._executor._task.done()) + @async_client_context.require_sync def test_no_thread_start_runtime_err_on_shutdown(self): """Test we silence noisy runtime errors fired when the AsyncMongoClient spawns a new thread diff --git a/test/test_monitor.py b/test/test_monitor.py index 929708824c..bb00b7403c 100644 --- a/test/test_monitor.py +++ b/test/test_monitor.py @@ -124,9 +124,30 @@ def thread_alive(executor): client.close() + # close() drains the deferred-join queue after releasing the + # topology lock, so every monitor it closed has been joined. + self.assertEqual(client._topology._monitor_tasks, []) self.assertFalse(thread_alive(monitor._executor)) self.assertFalse(thread_alive(monitor._rtt_monitor._executor)) + @client_context.require_async + def test_close_stops_monitor_task(self): + """PYTHON-6048: the async counterpart of + test_close_stops_monitor_thread. + """ + client = self.create_client() + server = next(iter(client._topology._servers.values())) + monitor = server._monitor + + self.assertIsNotNone(monitor._executor._task) + self.assertFalse(monitor._executor._task.done()) + + client.close() + + self.assertEqual(client._topology._monitor_tasks, []) + self.assertTrue(monitor._executor._task.done()) + self.assertTrue(monitor._rtt_monitor._executor._task.done()) + @client_context.require_sync def test_no_thread_start_runtime_err_on_shutdown(self): """Test we silence noisy runtime errors fired when the MongoClient spawns a new thread From 6f75f58cdb16089218c51c95b62548a59c8c8333 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 16:00:44 -0500 Subject: [PATCH 25/26] PYTHON-6048 Document cleanup_monitors' per-monitor latency bound --- pymongo/asynchronous/topology.py | 4 +++- pymongo/synchronous/topology.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/topology.py b/pymongo/asynchronous/topology.py index 0ef1f28471..b390e0fb94 100644 --- a/pymongo/asynchronous/topology.py +++ b/pymongo/asynchronous/topology.py @@ -936,7 +936,9 @@ async def cleanup_monitors(self) -> None: """Join monitors closed earlier while the topology lock was held. Always call this with the lock released: a monitor being joined may - itself be blocked acquiring the lock. + itself be blocked acquiring the lock. Per-monitor join timeout: up to + 1s for RttMonitor/SrvMonitor, up to 2s for Monitor (sequential joins + of executor and rtt_monitor). Called from select_servers() and close(). """ tasks = [] try: diff --git a/pymongo/synchronous/topology.py b/pymongo/synchronous/topology.py index 367774f663..4ab845aee4 100644 --- a/pymongo/synchronous/topology.py +++ b/pymongo/synchronous/topology.py @@ -934,7 +934,9 @@ def cleanup_monitors(self) -> None: """Join monitors closed earlier while the topology lock was held. Always call this with the lock released: a monitor being joined may - itself be blocked acquiring the lock. + itself be blocked acquiring the lock. Per-monitor join timeout: up to + 1s for RttMonitor/SrvMonitor, up to 2s for Monitor (sequential joins + of executor and rtt_monitor). Called from select_servers() and close(). """ tasks = [] try: From 7ee53821a08f347b61e7b26ed66527a4e9c4230d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 1 Sep 2026 19:46:17 -0500 Subject: [PATCH 26/26] PYTHON-6048 Add changelog entry for monitor close() latency change MongoClient.close() now waits (bounded) for monitor threads to stop before returning, fixing a real race between monitor threads and pool teardown. Document the resulting latency change for users. --- doc/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/changelog.rst b/doc/changelog.rst index f90025963f..bf133c4c5e 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -50,6 +50,11 @@ PyMongo 4.18 brings a number of changes including: - Fixed a bug on Windows, and on macOS when using PyOpenSSL, where ``SSL_CERT_FILE``/``SSL_CERT_DIR`` were merged with, rather than replacing, the OS/certifi certificate store. +- Fixed a race between a server monitor's background thread and connection + pool teardown during + :meth:`~pymongo.synchronous.mongo_client.MongoClient.close`. Closing a + client now waits (with a bounded timeout) for its monitor threads to stop + before returning, adding up to a couple hundred milliseconds to ``close()``. - Added general availability support for Queryable Encryption prefix, suffix, and substring queries against MongoDB 9.0+. These queries require libmongocrypt 1.20.0 or later: