PYTHON-6048 Add a sanitizers variant in Evergreen

.evergreen/generated_configs/functions.yml

@@ -85,6 +85,20 @@ functions:
       params:
         directory: src
 
+  # Run sanitizer tests
+  run sanitizer tests:
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/just.sh
+          - run-sanitizer-tests
+        working_dir: src
+        include_expansions_in_env:
+          - SANITIZER
+          - UV_PYTHON
+      type: test
+
   # Run server
   run server:
     - command: subprocess.exec

.evergreen/generated_configs/tasks.yml

@@ -2895,6 +2895,34 @@ tasks:
       - func: send dashboard data
     tags: [perf]
 
+  # Sanitizer tests
+  - name: test-sanitizer-asan
+    commands:
+      - func: run server
+        vars:
+          VERSION: latest
+          TOPOLOGY: standalone
+          AUTH: noauth
+          SSL: nossl
+      - func: run sanitizer tests
+        vars:
+          SANITIZER: asan
+    tags: [sanitizer, pr]
+  - name: test-sanitizer-tsan
+    commands:
+      - func: run server
+        vars:
+          VERSION: latest
+          TOPOLOGY: standalone
+          AUTH: noauth
+          SSL: nossl
+      - func: run sanitizer tests
+        vars:
+          SANITIZER: tsan
+          UV_PYTHON: 3.14t
+    exec_timeout_secs: 7200
+    tags: [sanitizer, pr, free-threaded]
+
   # Search index tests
   - name: test-search-index-helpers
     commands:

.evergreen/generated_configs/variants.yml

@@ -489,6 +489,16 @@ buildvariants:
     expansions:
       SUB_TEST_NAME: pyopenssl
 
+  # Sanitizer tests
+  - name: sanitizers-ubuntu-22
+    tasks:
+      - name: test-sanitizer-asan
+      - name: test-sanitizer-tsan
+    display_name: Sanitizers Ubuntu-22
+    run_on:
+      - ubuntu2204-small
+    tags: [pr]
+
   # Search index tests
   - name: search-index-helpers-rhel8
     tasks:

.evergreen/scripts/generate_config.py

@@ -341,6 +341,13 @@ def create_mod_wsgi_variants():
     return [create_variant(tasks, display_name, host=host, expansions=expansions)]
 
 
+def create_sanitizer_variants():
+    host = HOSTS["ubuntu22"]
+    tasks = ["test-sanitizer-asan", "test-sanitizer-tsan"]
+    display_name = get_variant_name("Sanitizers", host)
+    return [create_variant(tasks, display_name, host=host, tags=["pr"])]
+
+
 def create_disable_test_commands_variants():
     host = DEFAULT_HOST
     expansions = dict(AUTH="auth", SSL="ssl", DISABLE_TEST_COMMANDS="1")
@@ -659,6 +666,35 @@ def create_no_toolchain_tasks():
     return tasks
 
 
+def create_sanitizer_tasks():
+    tasks = []
+    # (sanitizer, free-threaded UV_PYTHON or None)
+    configs = [("asan", None), ("tsan", "3.14t")]
+    for sanitizer, python in configs:
+        tags = ["sanitizer", "pr"]
+        server_vars = dict(VERSION="latest", TOPOLOGY="standalone", AUTH="noauth", SSL="nossl")
+        server_func = FunctionCall(func="run server", vars=server_vars)
+        test_vars = dict(SANITIZER=sanitizer)
+        if python:
+            test_vars["UV_PYTHON"] = python
+            tags.append("free-threaded")
+        test_func = FunctionCall(func="run sanitizer tests", vars=test_vars)
+        name = f"test-sanitizer-{sanitizer}"
+        # TSan builds a fully instrumented free-threaded CPython from source
+        # before it can run anything, which does not fit in the project-wide
+        # 60 minute exec timeout.
+        exec_timeout_secs = 7200 if sanitizer == "tsan" else None
+        tasks.append(
+            EvgTask(
+                name=name,
+                tags=tags,
+                exec_timeout_secs=exec_timeout_secs,
+                commands=[server_func, test_func],
+            )
+        )
+    return tasks
+
+
 def create_test_non_standard_tasks():
     """For variants that set a TEST_NAME."""
     tasks = []
@@ -1315,6 +1351,13 @@ def create_run_tests_func():
     return "run tests", [setup_cmd, test_cmd]
 
 
+def create_run_sanitizer_tests_func():
+    includes = ["SANITIZER", "UV_PYTHON"]
+    args = [".evergreen/just.sh", "run-sanitizer-tests"]
+    sub_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args)
+    return "run sanitizer tests", [sub_cmd]
+
+
 def create_test_numpy_func():
     includes = ["TOOLCHAIN_VERSION", "COVERAGE"]
     test_cmd = get_subprocess_exec(

.evergreen/scripts/run-sanitizer-tests.sh

@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+# Clean sanitizer rebuild of the C extensions, run the fixed BSON/client
+# test files under the matching sanitizer runtime, and fail on any
+# sanitizer diagnostic even if pytest itself exits 0.
+#
+# ASan runs against a prebuilt interpreter with libasan.so LD_PRELOADed.
+# TSan builds a fully instrumented free-threaded CPython from source
+# instead: TSan cannot see synchronization in code compiled without
+# -fsanitize=thread, so LD_PRELOADing it onto a prebuilt interpreter
+# reports false races inside CPython's own free-threading internals.
+set -eu
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../.."
+
+SANITIZER=${SANITIZER:?"SANITIZER must be set to 'asan' or 'tsan'"}
+UV_PYTHON=${UV_PYTHON:-3.13}
+TEST_FILES=(test/test_bson.py test/test_raw_bson.py test/test_raw_bson_shared.py test/test_client.py)
+
+# The free-threaded CPython the TSan task builds. Kept in step with the
+# "3.14t" entry in .evergreen/scripts/generate_config_utils.py's CPYTHONS.
+CPYTHON_TAG=v3.14.0
+CPYTHON_SRC=.tsan-cpython-src
+CPYTHON_INSTALL=.tsan-cpython-install
+
+# A stale build/ or venv can leave a .so or install linked against the wrong
+# sanitizer's runtime without a build error, so always start clean.
+rm -rf build
+rm -f bson/*.so pymongo/*.so
+
+export CC=${CC:-clang}
+export CXX=${CXX:-clang++}
+export PYMONGO_C_EXT_MUST_BUILD=1
+
+case "$SANITIZER" in
+  asan)
+    rm -rf .sanitizer-venv
+    export CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g -O0"
+    export LDFLAGS="-fsanitize=address,undefined"
+    RUNTIME_LIB=$("$CC" -print-file-name=libasan.so)
+    export ASAN_OPTIONS="detect_leaks=0"
+    export UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1"
+    # Route CPython's own allocations through the system allocator so ASan's
+    # redzones can see them; pymalloc otherwise hides real bugs and adds
+    # noise. The free-threaded TSan interpreter doesn't accept this value,
+    # so it's scoped to ASan only.
+    export PYTHONMALLOC=malloc
+
+    if [ ! -f "$RUNTIME_LIB" ]; then
+      echo "Could not locate the $SANITIZER runtime library (got: $RUNTIME_LIB). Is the matching sanitizer runtime package installed?" >&2
+      exit 1
+    fi
+
+    uv venv --python "$UV_PYTHON" .sanitizer-venv
+    VENV_PYTHON=.sanitizer-venv/bin/python3
+    uv pip install --python "$VENV_PYTHON" -e . --reinstall
+    uv pip install --python "$VENV_PYTHON" -r requirements/test.txt
+
+    PYTEST_CMD=(env "LD_PRELOAD=$RUNTIME_LIB" "$VENV_PYTHON" -m pytest)
+    ;;
+  tsan)
+    rm -rf "$CPYTHON_SRC" "$CPYTHON_INSTALL"
+
+    # TSan's shadow mapping can fail to reserve its address ranges under the
+    # default ASLR entropy on recent kernels, which crashes the process at
+    # startup. CPython's own CI lowers the entropy the same way. Evergreen
+    # hosts may not permit it, so this is best effort.
+    sudo sysctl -w vm.mmap_rnd_bits=28 || true
+
+    # Building CPython from source needs its usual dev dependencies. Best
+    # effort: if apt-get isn't available or permitted, the OpenSSL header
+    # check below still catches the case that breaks the pip installs.
+    sudo apt-get update -qq || true
+    # Package list matches CPython's own Doc/using/unix.rst and
+    # Tools/scripts/posix-deps-apt.sh, so the optional extension modules
+    # (_zstd, _gdbm, _tkinter, ...) build instead of silently skipping.
+    sudo apt-get install -y --no-install-recommends \
+      build-essential libssl-dev zlib1g-dev libbz2-dev libffi-dev \
+      libreadline-dev libsqlite3-dev liblzma-dev pkg-config libb2-dev \
+      libgdbm-dev libgdbm-compat-dev libncurses5-dev libzstd-dev tk-dev \
+      uuid-dev curl || true
+
+    # Fail here rather than after the 20-30 minute build: without these
+    # headers _ssl won't build and the pip installs below cannot reach PyPI.
+    if [ ! -f /usr/include/openssl/ssl.h ]; then
+      echo "OpenSSL development headers not found at /usr/include/openssl/ssl.h after apt-get install. pip needs a working ssl module to reach PyPI. Aborting before the CPython build." >&2
+      exit 1
+    fi
+
+    CPYTHON_INSTALL_ABS="$(pwd)/$CPYTHON_INSTALL"
+    git clone --depth 1 --branch "$CPYTHON_TAG" https://github.com/python/cpython.git "$CPYTHON_SRC"
+
+    # TSan accepts one suppressions file, so concatenate the pinned tag's own
+    # suppressions with this repo's additions. The 3.14 branch's file is not
+    # empty: even a fully instrumented build needs its ~24 entries.
+    CPYTHON_SUPPRESSIONS="$CPYTHON_SRC/Tools/tsan/suppressions_free_threading.txt"
+    if [ ! -f "$CPYTHON_SUPPRESSIONS" ]; then
+      echo "Expected CPython's own TSan suppressions at $CPYTHON_SUPPRESSIONS, but the file is missing from the $CPYTHON_TAG source tree." >&2
+      exit 1
+    fi
+    COMBINED_SUPPRESSIONS="$(pwd)/.tsan-suppressions-combined.txt"
+    cat "$CPYTHON_SUPPRESSIONS" .evergreen/tsan-suppressions.txt > "$COMBINED_SUPPRESSIONS"
+
+    # Flags mirror CPython's own TSan CI job (.github/workflows/reusable-san.yml).
+    # CFLAGS/LDFLAGS are deliberately left unset here: --with-thread-sanitizer
+    # and --with-pydebug already supply the right flags, and overriding them
+    # would fight configure.
+    #
+    # Unlike CPython's CI this does not rebuild OpenSSL with TSan, which is
+    # only needed to keep the ssl tests quiet. The tests below don't use ssl,
+    # but pip does need it to reach PyPI, so _ssl still has to build against
+    # the system OpenSSL.
+    (
+      cd "$CPYTHON_SRC"
+      ./configure \
+        --with-thread-sanitizer \
+        --with-pydebug \
+        --disable-gil \
+        --prefix="$CPYTHON_INSTALL_ABS"
+      make -j"$(nproc 2>/dev/null || echo 4)"
+      make install
+    )
+
+    # --disable-gil adds a "t" suffix and --with-pydebug adds a "d", so this
+    # build installs as pythonX.Ytd. bin/python3 is the last-resort fallback.
+    TSAN_PYTHON=""
+    for candidate in "$CPYTHON_INSTALL"/bin/python3.*td "$CPYTHON_INSTALL"/bin/python3.*t "$CPYTHON_INSTALL"/bin/python3; do
+      if [ -x "$candidate" ]; then
+        TSAN_PYTHON="$candidate"
+        break
+      fi
+    done
+    if [ -z "$TSAN_PYTHON" ]; then
+      echo "Could not find the interpreter built from source under $CPYTHON_INSTALL/bin:" >&2
+      ls -l "$CPYTHON_INSTALL/bin" >&2 || true
+      exit 1
+    fi
+    echo "Using TSan-instrumented interpreter: $TSAN_PYTHON"
+    "$TSAN_PYTHON" -VV
+    "$TSAN_PYTHON" -c 'import sysconfig, sys; sys.exit(0 if sysconfig.get_config_var("Py_GIL_DISABLED") else "interpreter is not free-threaded")'
+    if ! "$TSAN_PYTHON" -c 'import ssl' >/dev/null 2>&1; then
+      echo "Warning: the interpreter built from source has no working ssl module, so pip cannot reach PyPI. Install the system OpenSSL development headers on this host." >&2
+    fi
+
+    # Build PyMongo's C extensions with the same instrumentation as the
+    # interpreter. These are plain shell env vars so pip's isolated build
+    # subprocess inherits them; build isolation is left on so pip resolves
+    # hatchling's build dependencies itself.
+    export CFLAGS="-fsanitize=thread -fno-omit-frame-pointer -g -O0"
+    export LDFLAGS="-fsanitize=thread"
+    "$TSAN_PYTHON" -m pip install -e .
+    "$TSAN_PYTHON" -m pip install -r requirements/test.txt
+
+    # Set after the build: halt_on_error=1 would abort the CPython build and
+    # the installs on any diagnostic raised by those tools themselves.
+    # handle_segv=0 matches CPython's own TSan CI job.
+    TSAN_OPTIONS="halt_on_error=1:handle_segv=0:suppressions=$COMBINED_SUPPRESSIONS"
+    export TSAN_OPTIONS
+
+    # No LD_PRELOAD: both the interpreter and the extensions link the TSan
+    # runtime at build time.
+    PYTEST_CMD=("$TSAN_PYTHON" -m pytest)
+    ;;
+  *)
+    echo "Unknown SANITIZER: $SANITIZER (expected 'asan' or 'tsan')" >&2
+    exit 1
+    ;;
+esac
+
+LOG_FILE=$(mktemp)
+set +e
+"${PYTEST_CMD[@]}" -v --capture=no "${TEST_FILES[@]}" 2>&1 | tee "$LOG_FILE"
+PYTEST_STATUS=${PIPESTATUS[0]}
+set -e
+
+if grep -qE "ERROR: (AddressSanitizer|LeakSanitizer)|runtime error:|WARNING: ThreadSanitizer|SUMMARY: (Address|Undefined|ThreadSanitizer)" "$LOG_FILE"; then
+  echo "Sanitizer diagnostic detected in test output, failing task" >&2
+  exit 1
+fi
+
+exit "$PYTEST_STATUS"

.evergreen/tsan-suppressions.txt

@@ -0,0 +1,15 @@
+# PyMongo-specific ThreadSanitizer suppressions. Currently none are needed.
+#
+# .evergreen/scripts/run-sanitizer-tests.sh concatenates this file with the
+# pinned CPython tag's own Tools/tsan/suppressions_free_threading.txt and
+# passes the result as TSan's single suppressions= path. That upstream file
+# is not empty on the 3.14 branch: it carries roughly two dozen entries
+# (assign_version_tag, update_one_slot, _PyFrame_GetCode, rangeiter_next,
+# list_ass_slice_lock_held, PyObject_Realloc, mi_block_set_nextx,
+# pthread_create, and others) that CPython's own fully instrumented TSan CI
+# job needs to run clean. Do not duplicate those here.
+#
+# Add an entry below only for a race in PyMongo's own C extensions that has
+# been investigated and judged benign, with a comment saying why.
+#
+# Reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions

.gitignore

@@ -46,3 +46,9 @@ xunit-results/
 coverage.xml
 server.log
 .coverage
+
+# sanitizer test task scratch (see .evergreen/scripts/run-sanitizer-tests.sh)
+.sanitizer-venv/
+.tsan-cpython-src/
+.tsan-cpython-install/
+.tsan-suppressions-combined.txt

doc/changelog.rst

@@ -50,6 +50,11 @@ PyMongo 4.18 brings a number of changes including:
 - Fixed a bug on Windows, and on macOS when using PyOpenSSL, where
   ``SSL_CERT_FILE``/``SSL_CERT_DIR`` were merged with, rather than replacing,
   the OS/certifi certificate store.
+- Fixed a race between a server monitor's background thread and connection
+  pool teardown during
+  :meth:`~pymongo.synchronous.mongo_client.MongoClient.close`. Closing a
+  client now waits (with a bounded timeout) for its monitor threads to stop
+  before returning, adding up to a couple hundred milliseconds to ``close()``.
 - Added general availability support for Queryable Encryption prefix, suffix,
   and substring queries against MongoDB 9.0+. These queries require
   libmongocrypt 1.20.0 or later:

justfile

@@ -129,6 +129,9 @@ coverage-xml:
 run-server *args="":
     bash .evergreen/scripts/run-server.sh {{args}}
 
+run-sanitizer-tests *args="":
+    bash .evergreen/scripts/run-sanitizer-tests.sh {{args}}
+
 [group('server')]
 stop-server:
     bash .evergreen/scripts/stop-server.sh

pymongo/asynchronous/monitor.py

@@ -16,7 +16,6 @@
 
 from __future__ import annotations
 
-import asyncio
 import atexit
 import time
 import weakref
@@ -106,9 +105,9 @@ async def close(self) -> None:
         """
         self.gc_safe_close()
 
-    async def join(self) -> None:
+    async def join(self, timeout: Optional[int] = None) -> None:
         """Wait for the monitor to stop."""
-        await self._executor.join()
+        await self._executor.join(timeout)
 
     def request_check(self) -> None:
         """If the monitor is sleeping, wake it soon."""
@@ -184,10 +183,9 @@ def gc_safe_close(self) -> None:
         self._rtt_monitor.gc_safe_close()
         self.cancel_check()
 
-    async def join(self) -> None:
-        await asyncio.gather(
-            self._executor.join(), self._rtt_monitor.join(), return_exceptions=True
-        )  # type: ignore[func-returns-value]
+    async def join(self, timeout: Optional[int] = None) -> None:
+        await self._executor.join(timeout)
+        await self._rtt_monitor.join(timeout)
 
     async def close(self) -> None:
         self.gc_safe_close()