From 8b0ad8c028dd16ba6901982897264c460f74c8ee Mon Sep 17 00:00:00 2001 From: nimbrel <331679006+nimbrel@users.noreply.github.com> Date: Sat, 26 Sep 2026 09:58:48 +0000 Subject: [PATCH 1/7] fix(tsan): a bare file name in debug info is never a crate file An uninstrumented library can record a source file with no directory. It was resolved against the working directory, which is the crate root in the image, so it passed for a crate file and became the primary location and the merge key. Only an absolute path under the crate root now counts. Signed-off-by: nimbrel <331679006+nimbrel@users.noreply.github.com> --- python/ftcheck/ci/tsan.py | 10 +++++++++- tests/test_tsan_parse.py | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/python/ftcheck/ci/tsan.py b/python/ftcheck/ci/tsan.py index c452196..f9fc4f9 100644 --- a/python/ftcheck/ci/tsan.py +++ b/python/ftcheck/ci/tsan.py @@ -320,7 +320,15 @@ def _rule(kind: str) -> str: def _relative(path: str, root: str) -> str | None: - """`path` relative to `root`, or None when it lies outside it.""" + """`path` relative to `root`, or None when it lies outside it. + + Only an absolute path can be in the crate. A bare file name comes from an + uninstrumented library's debug info (`lowlevel_strided_loops.c`); resolved + against the working directory, which is the crate root in the image, it + would pass for a crate file. + """ + if not os.path.isabs(path): + return None try: rel = os.path.relpath(os.path.realpath(path), os.path.realpath(root)) except ValueError: diff --git a/tests/test_tsan_parse.py b/tests/test_tsan_parse.py index 39dd924..518d6b7 100644 --- a/tests/test_tsan_parse.py +++ b/tests/test_tsan_parse.py @@ -336,3 +336,23 @@ def test_a_thin_ffi_wrapper_is_not_the_primary_location(): ) (finding,), _, _ = to_findings([report], EXAMPLE, CRATE) assert finding["primary"]["file"] == "src/encode/model.rs" + + +MUTATOR_COPY = ["memmove (python3.14+0x9)", + "_contig_to_contig lowlevel_strided_loops.c (_multiarray_umath.cpython-314t-x86_64-linux-gnu.so+0x7)"] + + +def test_a_bare_file_name_is_not_a_file_in_the_crate(tmp_path, monkeypatch): + """An uninstrumented library's debug info can name a source file with no + directory. Resolved against the working directory — the crate root, in + the image — it looked like a crate file and became the primary location. + The test must run from the crate root: that is what exposed it.""" + root = str(tmp_path) + monkeypatch.chdir(tmp_path) + # The dependency's side comes first, as it did in the report that exposed it. + report = _race(MUTATOR_COPY, [f"scan {root}/src/lib.rs:117:14 {EXT}"]) + (finding,), _, _ = to_findings([report], EXAMPLE, root) + assert finding["primary"]["file"] == "src/lib.rs" + assert finding["primary"]["line"] == 117 + files = [fr["location"]["file"] for s in finding["stacks"] for fr in s["frames"] if fr["location"]] + assert "lowlevel_strided_loops.c" in files, "kept as the symbolizer wrote it" From 5e0a0f7757fd04199a2b388bbb37b435dd804846 Mon Sep 17 00:00:00 2001 From: nimbrel <331679006+nimbrel@users.noreply.github.com> Date: Sat, 26 Sep 2026 10:00:14 +0000 Subject: [PATCH 2/7] fix(tsan): a mutator's copy into another library's buffer is a harness race Only CPython's own content writers were recognised. A mutator refilling a numpy array reaches the buffer through numpy's copy loop and memmove, so the report was filed as the extension's certain race and failed the run. A mutator stack now counts as a content rewrite when a copy primitive is at the top, called from a library outside the interpreter, with no frame of the extension and no free, realloc or resize anywhere on it. CPython's containers stay excluded: their copies run under a critical section the extension must also hold. Harness races are located at the extension's access rather than in the mutator's library. Signed-off-by: nimbrel <331679006+nimbrel@users.noreply.github.com> --- docs/stress.md | 9 ++- python/ftcheck/ci/tsan.py | 48 +++++++++++++--- tests/data/tsan/mutator-numpy-copy.log | 76 ++++++++++++++++++++++++++ tests/test_tsan_parse.py | 70 ++++++++++++++++++++++++ 4 files changed, 191 insertions(+), 12 deletions(-) create mode 100644 tests/data/tsan/mutator-numpy-copy.log diff --git a/docs/stress.md b/docs/stress.md index 76fcf2e..65f79b9 100644 --- a/docs/stress.md +++ b/docs/stress.md @@ -137,9 +137,12 @@ a mutator's transient state, so a call that panics on input the mutator made inv reported as a panic under concurrency. A mutator that only **rewrites a buffer's contents** in place (a slice assignment into a -`bytearray` the extension is reading) races by the buffer protocol's own contract; it is -reported as a **harness** race — shown, not failing. A mutator whose resize or free makes -the extension touch freed memory is a finding like any other. Mutator threads are named +`bytearray` the extension is reading, or a numpy array refilled with `arr[:] = ...`) races +by the buffer protocol's own contract; it is reported as a **harness** race — shown, not +failing, and located at the extension's read. A mutator whose resize or free makes +the extension touch freed memory is a finding like any other, and so is a copy inside +one of CPython's own containers (`list.insert`): those run under the container's critical +section, which the extension must take too. Mutator threads are named `ftm-` (cut to Linux's 15 characters), so TSan reports say which mutator it was. ### Skipping diff --git a/python/ftcheck/ci/tsan.py b/python/ftcheck/ci/tsan.py index f9fc4f9..946f39b 100644 --- a/python/ftcheck/ci/tsan.py +++ b/python/ftcheck/ci/tsan.py @@ -66,6 +66,13 @@ "bytearray_setslice_linear", "bytearray_setslice", "bytearray_ass_subscript", "copy_base", "copy_single", "memory_ass_sub", "bytearray_setitem", ) # fmt: skip +# TSan's interceptors for the copy primitives, as the symbolizer names them. +_COPY_PRIMITIVES = frozenset( + p for name in ("memmove", "memcpy", "memset") for p in (name, "__tsan_" + name) +) +# A frame that frees, reallocates or resizes: not a rewrite in place. +_RELEASES = re.compile(r"free|realloc|resize|dealloc", re.IGNORECASE) +_INTERPRETER_LIBRARY = re.compile(r"^libpython") # The other access being an allocation means the memory was freed and reused: # the extension touched an object that no longer exists. _ALLOCATORS = ( @@ -252,17 +259,38 @@ def _first_real(section: Section) -> Frame | None: } -def _harness_rewrite(report: Report) -> bool: +def _rewrites_contents(section: Section, extension_modules: set[str]) -> bool: + """This stack only rewrites a buffer's contents in place. + + Either a CPython routine that does nothing else (`_CONTENT_WRITERS`), or a + copy primitive at the top called from a library outside the interpreter — + numpy refilling an array with `memmove`. CPython's own containers are left + out of the second case: a `memmove` inside `list.insert` runs under the + list's critical section, which an extension reading the list must take + too. A frame of yours, or any free, realloc or resize on the stack, and it + is not a rewrite: memory the extension reads may be gone. + """ + frames = section.frames + if any(f.module in extension_modules or _RELEASES.search(f.symbol) for f in frames): + return False + first = _first_real(section) + if first is None: + return False + if first.symbol.startswith(_CONTENT_WRITERS): + return True + copies = bool(frames) and _is_interceptor(frames[0]) and frames[0].symbol in _COPY_PRIMITIVES + return copies and not _INTERPRETER_LIBRARY.match(first.module or "") + + +def _harness_rewrite(report: Report, extension_modules: set[str]) -> bool: """A mutator thread only rewriting buffer contents in place.""" names = report.thread_names() for section in report.access_stacks: tid = report.access_thread(section) - if not names.get(tid or "", "").startswith(_MUTATOR_PREFIX): - continue - for frame in section.frames: - if _is_interceptor(frame): - continue - return frame.symbol.startswith(_CONTENT_WRITERS) + if names.get(tid or "", "").startswith(_MUTATOR_PREFIX) and _rewrites_contents( + section, extension_modules + ): + return True return False @@ -285,7 +313,7 @@ def attribute(report: Report, extension_modules: set[str]) -> str: anywhere = any(f.module in extension_modules for s in stacks for f in s.frames) if report.is_crash: return YOURS if anywhere or not stacks else EXTERNAL - if anywhere and _harness_rewrite(report): + if anywhere and _harness_rewrite(report, extension_modules): return HARNESS tops = [_first_real(stack) for stack in stacks] if anywhere and any(t is not None and t.symbol.startswith(_ALLOCATORS) for t in tops): @@ -443,7 +471,9 @@ def to_findings( stacks.sort(key=lambda s: s[1][0].symbol if s[1] else "") if stacks: frame, path = _primary(report, extension_modules, crate_root) - if owner != YOURS: + # A harness race is placed at the extension's access, like one of + # yours: the mutator's side is the harness's own code. + if owner not in (YOURS, HARNESS): frame = stacks[0][1][0] if stacks[0][1] else frame path = (frame.file and (_relative(frame.file, crate_root) or frame.file)) or ( frame.module or "" diff --git a/tests/data/tsan/mutator-numpy-copy.log b/tests/data/tsan/mutator-numpy-copy.log new file mode 100644 index 0000000..04cd4e0 --- /dev/null +++ b/tests/data/tsan/mutator-numpy-copy.log @@ -0,0 +1,76 @@ +================== +WARNING: ThreadSanitizer: data race (pid=41) + Write of size 8 at 0x724800012300 by thread T5: + #0 memmove (python3.14+0x60432) + #1 _contig_to_contig lowlevel_strided_loops.c (_multiarray_umath.cpython-314t-x86_64-linux-gnu.so+0x735c4) + #2 _PyEval_EvalFrameDefault /work/.pyenv/sources/3.14.7t/Python-3.14.7/Python/generated_cases.c.h:11503:27 (libpython3.14t.so.1.0+0x35495c) + #3 _PyEval_EvalFrame /work/.pyenv/sources/3.14.7t/Python-3.14.7/./Include/internal/pycore_ceval.h:120:16 (libpython3.14t.so.1.0+0x3328a8) + #4 _PyEval_Vector /work/.pyenv/sources/3.14.7t/Python-3.14.7/Python/ceval.c:2113:12 (libpython3.14t.so.1.0+0x3328a8) + #5 _PyFunction_Vectorcall /work/.pyenv/sources/3.14.7t/Python-3.14.7/Objects/call.c (libpython3.14t.so.1.0+0x12f621) + #6 context_run /work/.pyenv/sources/3.14.7t/Python-3.14.7/Python/context.c:725:29 (libpython3.14t.so.1.0+0x3892c2) + + Previous read of size 8 at 0x724800012300 by thread T9: + #0 __tsan_memcpy (python3.14+0x60952) + #1 copy_nonoverlapping /opt/rustup/toolchains/nightly-2026-01-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:547:14 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x5d2786) + #2 to_vec /opt/rustup/toolchains/nightly-2026-01-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/slice.rs:453:36 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x5d2786) + #3 ::normalise /opt/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/examplecore-0.4.2/src/table.rs (examplelib.cpython-314t-x86_64-linux-gnu.so+0x5d2786) + #4 ::lookup /opt/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/examplecore-0.4.2/src/table.rs:212:18 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x5e7e95) + #5 lookup /src/src/lib.rs:141:23 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x6a2d34) + #6 ::__pymethod_lookup__ /src/src/lib.rs:88:1 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x6a2d34) + #7 {closure#0} /opt/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/pyo3-0.27.2/src/impl_/trampoline.rs:44:37 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x67fd0a) + #8 pyo3::impl_::trampoline::trampoline:: /opt/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/pyo3-0.27.2/src/impl_/trampoline.rs:190:9 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x6b2d13) + #9 fastcall_with_keywords /opt/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/pyo3-0.27.2/src/impl_/trampoline.rs:44:21 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x6b2d13) + #10 method_vectorcall_FASTCALL_KEYWORDS /work/.pyenv/sources/3.14.7t/Python-3.14.7/Objects/descrobject.c:421:24 (libpython3.14t.so.1.0+0x1486c3) + + Location is heap block of size 384 at 0x724800012300 allocated by main thread: + #0 malloc (python3.14+0x63254) + #1 _PyMem_RawMalloc /work/.pyenv/sources/3.14.7t/Python-3.14.7/Objects/obmalloc.c:63:12 (libpython3.14t.so.1.0+0x209c9d) + #2 default_malloc alloc.c (_multiarray_umath.cpython-314t-x86_64-linux-gnu.so+0x176ba0) + #3 array_multiply number.c (_multiarray_umath.cpython-314t-x86_64-linux-gnu.so+0x223fd5) + + Thread T5 'ftm-refill_rows' (tid=21, running) created by main thread at: + #0 pthread_create (python3.14+0x655de) + #1 do_start_joinable_thread /work/.pyenv/sources/3.14.7t/Python-3.14.7/Python/thread_pthread.h:289:14 (libpython3.14t.so.1.0+0x43746a) + + Thread T9 'ftw-pair-2' (tid=25, running) created by main thread at: + #0 pthread_create (python3.14+0x655de) + #1 do_start_joinable_thread /work/.pyenv/sources/3.14.7t/Python-3.14.7/Python/thread_pthread.h:289:14 (libpython3.14t.so.1.0+0x43746a) + +SUMMARY: ThreadSanitizer: data race (/work/.pyenv/versions/3.14.7t/bin/python3.14+0x60432) in memmove +================== +WARNING: ThreadSanitizer: data race (pid=41) + Read of size 8 at 0x724800012300 by thread T11: + #0 all /opt/rustup/toolchains/nightly-2026-01-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/slice/iter/macros.rs:315:25 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x5e79d5) + #1 ::scan /opt/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/examplecore-0.4.2/src/table.rs:174:26 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x5e79d5) + #2 scan /src/src/lib.rs:117:14 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x697e54) + #3 ::__pymethod_scan__ /src/src/lib.rs:88:1 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x697e54) + #4 {closure#0} /opt/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/pyo3-0.27.2/src/impl_/trampoline.rs:44:37 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x67fd0a) + #5 pyo3::impl_::trampoline::trampoline:: /opt/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/pyo3-0.27.2/src/impl_/trampoline.rs:190:9 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x6b2d13) + #6 fastcall_with_keywords /opt/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/pyo3-0.27.2/src/impl_/trampoline.rs:44:21 (examplelib.cpython-314t-x86_64-linux-gnu.so+0x6b2d13) + #7 method_vectorcall_FASTCALL_KEYWORDS /work/.pyenv/sources/3.14.7t/Python-3.14.7/Objects/descrobject.c:421:24 (libpython3.14t.so.1.0+0x1486c3) + + Previous write of size 8 at 0x724800012300 by thread T5: + #0 memmove (python3.14+0x60432) + #1 _contig_to_contig lowlevel_strided_loops.c (_multiarray_umath.cpython-314t-x86_64-linux-gnu.so+0x735c4) + #2 _PyEval_EvalFrameDefault /work/.pyenv/sources/3.14.7t/Python-3.14.7/Python/generated_cases.c.h:11503:27 (libpython3.14t.so.1.0+0x35495c) + #3 _PyEval_EvalFrame /work/.pyenv/sources/3.14.7t/Python-3.14.7/./Include/internal/pycore_ceval.h:120:16 (libpython3.14t.so.1.0+0x3328a8) + #4 _PyEval_Vector /work/.pyenv/sources/3.14.7t/Python-3.14.7/Python/ceval.c:2113:12 (libpython3.14t.so.1.0+0x3328a8) + #5 _PyFunction_Vectorcall /work/.pyenv/sources/3.14.7t/Python-3.14.7/Objects/call.c (libpython3.14t.so.1.0+0x12f621) + #6 context_run /work/.pyenv/sources/3.14.7t/Python-3.14.7/Python/context.c:725:29 (libpython3.14t.so.1.0+0x3892c2) + + Location is heap block of size 384 at 0x724800012300 allocated by main thread: + #0 malloc (python3.14+0x63254) + #1 _PyMem_RawMalloc /work/.pyenv/sources/3.14.7t/Python-3.14.7/Objects/obmalloc.c:63:12 (libpython3.14t.so.1.0+0x209c9d) + #2 default_malloc alloc.c (_multiarray_umath.cpython-314t-x86_64-linux-gnu.so+0x176ba0) + #3 array_multiply number.c (_multiarray_umath.cpython-314t-x86_64-linux-gnu.so+0x223fd5) + + Thread T11 'ftw-pair-4' (tid=27, running) created by main thread at: + #0 pthread_create (python3.14+0x655de) + #1 do_start_joinable_thread /work/.pyenv/sources/3.14.7t/Python-3.14.7/Python/thread_pthread.h:289:14 (libpython3.14t.so.1.0+0x43746a) + + Thread T5 'ftm-refill_rows' (tid=21, running) created by main thread at: + #0 pthread_create (python3.14+0x655de) + #1 do_start_joinable_thread /work/.pyenv/sources/3.14.7t/Python-3.14.7/Python/thread_pthread.h:289:14 (libpython3.14t.so.1.0+0x43746a) + +SUMMARY: ThreadSanitizer: data race (/work/.pyenv/versions/3.14.7t/bin/python3.14+0x60432) in memmove +================== diff --git a/tests/test_tsan_parse.py b/tests/test_tsan_parse.py index 518d6b7..3141cb5 100644 --- a/tests/test_tsan_parse.py +++ b/tests/test_tsan_parse.py @@ -356,3 +356,73 @@ def test_a_bare_file_name_is_not_a_file_in_the_crate(tmp_path, monkeypatch): assert finding["primary"]["line"] == 117 files = [fr["location"]["file"] for s in finding["stacks"] for fr in s["frames"] if fr["location"]] assert "lowlevel_strided_loops.c" in files, "kept as the symbolizer wrote it" + + +# mutator-numpy-copy.log is trimmed and renamed from a real report: a mutator +# (`ftm-refill_rows`) refills a numpy array in place from Python while two +# methods of the extension read it through the buffer protocol. numpy's copy +# loop has only a bare file name in its debug info. +MUTATOR_LOG = "mutator-numpy-copy.log" + + +def test_a_mutator_copying_into_a_foreign_buffer_is_a_harness_race(): + """No CPython routine on the mutator's stack: numpy's copy loop calls + memmove. Still a content rewrite, not the extension's bug.""" + from ftcheck.ci.tsan import HARNESS + + reports = parse(load(MUTATOR_LOG)) + assert len(reports) == 2 + assert [attribute(r, EXAMPLE) for r in reports] == [HARNESS, HARNESS] + mine, reached, other = to_findings(reports, EXAMPLE, "/src") + assert mine == [] and reached == [] + assert other and all(f["message"].startswith("A mutator rewrote") for f in other) + + +def test_a_harness_race_is_located_at_the_extensions_read(): + """The mutator's side is the harness; the line worth reading is yours.""" + _, _, other = to_findings(parse(load(MUTATOR_LOG)), EXAMPLE, "/src") + assert sorted((f["primary"]["file"], f["primary"]["line"]) for f in other) == [ + ("src/lib.rs", 117), + ("src/lib.rs", 141), + ] + + +def test_a_mutator_that_resizes_while_copying_is_still_yours(): + """A realloc on the mutator's side can free what the extension reads.""" + report = _race( + [f"scan /tmp/c/src/lib.rs:117:14 {EXT}"], + [*MUTATOR_COPY, "PyArray_Resize shape.c (_multiarray_umath.cpython-314t-x86_64-linux-gnu.so+0x8)"], + writer_name="ftm-grow", + ) + assert attribute(report, EXAMPLE) == YOURS + + +def test_a_mutator_memmove_inside_a_cpython_container_is_still_yours(): + """list.insert shifts items with memmove under the list's critical + section; an extension reading the list without one is at fault.""" + report = _race( + [f"scan /tmp/c/src/lib.rs:117:14 {EXT}"], + ["__tsan_memmove (python3.14+0x9)", f"list_ass_slice_lock_held /cpython/Objects/listobject.c:700:5 {PY}"], + writer_name="ftm-shift", + ) + assert attribute(report, EXAMPLE) == YOURS + + +def test_a_mutator_copying_through_the_extension_is_still_yours(): + """With a frame of yours on the mutator's side, the copy is your code's.""" + report = _race( + [f"scan /tmp/c/src/lib.rs:117:14 {EXT}"], + ["__tsan_memcpy (python3.14+0x9)", f"fill /tmp/c/src/lib.rs:60:9 {EXT}"], + writer_name="ftm-fill", + ) + assert attribute(report, EXAMPLE) == YOURS + + +def test_a_harness_race_run_from_the_crate_root_is_not_located_in_the_dependency(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + text = load(MUTATOR_LOG).replace(" /src/src/", f" {tmp_path}/src/") + _, _, other = to_findings(parse(text), EXAMPLE, str(tmp_path)) + assert other and {f["primary"]["file"] for f in other} == {"src/lib.rs"} + + +PYO3_FFI = "/opt/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/pyo3-ffi-0.26.0" From f93e76c1e335fca227e5be04cb36d2973ce7db70 Mon Sep 17 00:00:00 2001 From: nimbrel <331679006+nimbrel@users.noreply.github.com> Date: Sat, 26 Sep 2026 10:00:49 +0000 Subject: [PATCH 3/7] fix(tsan): locate a race outside the extension at its plain access When one side of a report not attributed to the extension was an atomic helper from CPython's pyatomic headers, that side could become the primary location. The plain access is the racy one, so it is preferred. Signed-off-by: nimbrel <331679006+nimbrel@users.noreply.github.com> --- python/ftcheck/ci/tsan.py | 12 +++++++++++- tests/test_tsan_parse.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/python/ftcheck/ci/tsan.py b/python/ftcheck/ci/tsan.py index 946f39b..601405b 100644 --- a/python/ftcheck/ci/tsan.py +++ b/python/ftcheck/ci/tsan.py @@ -364,6 +364,13 @@ def _relative(path: str, root: str) -> str | None: return None if rel == os.pardir or rel.startswith(os.pardir + os.sep) else rel +def _is_atomic(frame: Frame) -> bool: + """CPython's atomic helpers, inlined from `pyatomic*.h`.""" + return frame.symbol.startswith("_Py_atomic_") or bool( + frame.file and os.path.basename(frame.file).startswith("pyatomic") + ) + + def _frames_from_top(section: Section) -> list[Frame]: """The stack from its first non-interceptor frame: where the access happened.""" frames = section.frames @@ -474,7 +481,10 @@ def to_findings( # A harness race is placed at the extension's access, like one of # yours: the mutator's side is the harness's own code. if owner not in (YOURS, HARNESS): - frame = stacks[0][1][0] if stacks[0][1] else frame + # The plain access is the racy one; an atomic on the other + # side (`_Py_atomic_load_ptr`) is only where TSan noticed it. + firsts = [s[1][0] for s in stacks if s[1]] + frame = next((t for t in firsts if not _is_atomic(t)), firsts[0] if firsts else frame) path = (frame.file and (_relative(frame.file, crate_root) or frame.file)) or ( frame.module or "" ) diff --git a/tests/test_tsan_parse.py b/tests/test_tsan_parse.py index 3141cb5..fa8097f 100644 --- a/tests/test_tsan_parse.py +++ b/tests/test_tsan_parse.py @@ -426,3 +426,24 @@ def test_a_harness_race_run_from_the_crate_root_is_not_located_in_the_dependency PYO3_FFI = "/opt/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/pyo3-ffi-0.26.0" + + +def test_the_atomic_side_of_a_race_is_not_its_location(): + """An atomic load racing a plain write: the plain write is the racy one.""" + report = _race( + [f"_Py_atomic_load_ptr_relaxed /cpython/Include/cpython/pyatomic_gcc.h:513:10 {PY}", + f"list_get_item_ref /cpython/Objects/listobject.c:340:12 {PY}"], + [f"list_append /cpython/Objects/listobject.c:520:5 {PY}"], + ) + _, _, (finding,) = to_findings([report], EXAMPLE, CRATE) + assert finding["symbol"] == "list_append" + assert finding["primary"] == {"file": "/cpython/Objects/listobject.c", "line": 520, "column": 5} + + +def test_an_atomic_frame_in_a_pyatomic_header_is_skipped_too(): + report = _race( + [f"atomic_load_relaxed /cpython/Include/cpython/pyatomic_std.h:90:10 {PY}"], + [f"list_append /cpython/Objects/listobject.c:520:5 {PY}"], + ) + _, _, (finding,) = to_findings([report], EXAMPLE, CRATE) + assert finding["symbol"] == "list_append" From f72cb7b958e88327e1833fdb2b16e8a3da967f53 Mon Sep 17 00:00:00 2001 From: nimbrel <331679006+nimbrel@users.noreply.github.com> Date: Sat, 26 Sep 2026 10:01:32 +0000 Subject: [PATCH 4/7] feat(tsan): say when a racing access is inside PyO3 A race whose access happened in PyO3's own source was reported at the extension's frame with no mention of PyO3. The classification and location are unchanged; the message now names the PyO3 crate, version and line, since a newer PyO3 may have changed that access. Signed-off-by: nimbrel <331679006+nimbrel@users.noreply.github.com> --- docs/ci.md | 4 +++- python/ftcheck/ci/tsan.py | 23 +++++++++++++++++++++++ tests/test_tsan_parse.py | 25 +++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/docs/ci.md b/docs/ci.md index b18fc7b..837c449 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -115,7 +115,9 @@ Each report is placed by looking at where each conflicting access actually happe of your extension modules. Interceptors live in the interpreter binary, so a race on a buffer your C code writes through `snprintf` (top frame: TSan's `vsnprintf`) is yours. PyO3, and the standard library under `-Zbuild-std`, are compiled into your module, so - races there are yours too — you shipped them. These fail the run. + races there are yours too — you shipped them. These fail the run. When the access + itself is in PyO3's source, the message says so and names the PyO3 version, since what + PyO3 makes atomic changes between releases. - **Reached from yours** — both accesses happen inside CPython or another library, and **every** racing thread passed through your module further down its stack. For example, PyO3 methods calling `decimal.Decimal()` from many threads race inside diff --git a/python/ftcheck/ci/tsan.py b/python/ftcheck/ci/tsan.py index 601405b..61588bb 100644 --- a/python/ftcheck/ci/tsan.py +++ b/python/ftcheck/ci/tsan.py @@ -86,6 +86,8 @@ _BUILD_ID = re.compile(r"\s*\(BuildId: [0-9a-f]+\)\s*$") _ADDRESS = re.compile(r" at 0x[0-9a-f]+") _CODEGEN_UNIT = re.compile(r"^[^/]*-cgu\.\d+$") +# A source file of PyO3 itself, as Cargo unpacks it: `.../pyo3-ffi-0.26.0/src/...`. +_PYO3_SOURCE = re.compile(r"/(?Ppyo3(?:-ffi)?)-(?P\d+\.\d+\.\d+[^/]*)/(?P.+)$") _LOCATION = re.compile(r"^(?P.+?)(?::(?P\d+))?(?::(?P\d+))?$") # Sections of a report that describe context rather than a conflicting access. @@ -371,6 +373,26 @@ def _is_atomic(frame: Frame) -> bool: ) +def _pyo3_notes(report: Report) -> list[str]: + """A sentence per access that happened inside PyO3's own source. + + The race stays where it was attributed, but the code that raced is PyO3's, + and PyO3 changes what it makes atomic between releases. + """ + notes: dict[str, None] = {} + for section in report.access_stacks: + top = _first_real(section) + m = _PYO3_SOURCE.search(top.file) if top is not None and top.file else None + if m: + where = m.group("rest") + (f":{top.line}" if top.line else "") + notes.setdefault( + f" The access in `{top.symbol}` is inside {m.group('crate')} {m.group('version')} " + f"(`{where}`); a newer PyO3 may change it.", + None, + ) + return list(notes) + + def _frames_from_top(section: Section) -> list[Frame]: """The stack from its first non-interceptor frame: where the access happened.""" frames = section.frames @@ -506,6 +528,7 @@ def to_findings( for header, frames in stacks if frames ) + described += "".join(_pyo3_notes(report)) if report.is_crash and not stacks: described = ( "The process crashed and TSan captured no stack. It died while your " diff --git a/tests/test_tsan_parse.py b/tests/test_tsan_parse.py index fa8097f..27ca5e2 100644 --- a/tests/test_tsan_parse.py +++ b/tests/test_tsan_parse.py @@ -447,3 +447,28 @@ def test_an_atomic_frame_in_a_pyatomic_header_is_skipped_too(): ) _, _, (finding,) = to_findings([report], EXAMPLE, CRATE) assert finding["symbol"] == "list_append" + + +def test_an_access_inside_pyo3_says_so(): + """The race is still yours and still located in your code, but the access + itself is PyO3's, which a newer PyO3 may have changed.""" + report = _race( + [f"PyList_GET_SIZE {PYO3_FFI}/src/cpython/listobject.rs:22:5 {EXT}", + f"count /tmp/c/src/lib.rs:30:9 {EXT}"], + [f"list_append /cpython/Objects/listobject.c:520:5 {PY}"], + ) + assert attribute(report, EXAMPLE) == YOURS + (finding,), _, _ = to_findings([report], EXAMPLE, CRATE) + assert finding["primary"]["file"] == "src/lib.rs" + assert "inside pyo3-ffi 0.26.0 (`src/cpython/listobject.rs:22`)" in finding["message"] + assert "a newer PyO3 may change it" in finding["message"] + + +def test_pyo3_further_down_the_stack_is_not_mentioned(): + report = _race( + [f"count /tmp/c/src/lib.rs:30:9 {EXT}", + f"{{closure#0}} /opt/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/pyo3-0.27.2/src/impl_/trampoline.rs:44:37 {EXT}"], + [f"list_append /cpython/Objects/listobject.c:520:5 {PY}"], + ) + (finding,), _, _ = to_findings([report], EXAMPLE, CRATE) + assert "PyO3" not in finding["message"] From d910256c6b582fbee6bd587faa195533909342b1 Mon Sep 17 00:00:00 2001 From: nimbrel <331679006+nimbrel@users.noreply.github.com> Date: Sat, 26 Sep 2026 10:02:28 +0000 Subject: [PATCH 5/7] feat(tsan): name the threads in race messages and stack labels Messages said "by thread T7", so a report caused by a mutator could not be recognised without the raw log. The thread's name now follows its id in the message, each stack carries a "thread" field in the JSON, and the text summary labels stacks with it. Deduplication is unaffected: the signature does not include the message. Signed-off-by: nimbrel <331679006+nimbrel@users.noreply.github.com> --- python/ftcheck/ci/tsan.py | 32 ++++++++++++++++++++++++++------ python/ftcheck/report/text.py | 10 ++++++++-- tests/test_tsan_parse.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/python/ftcheck/ci/tsan.py b/python/ftcheck/ci/tsan.py index 61588bb..a3e6dd6 100644 --- a/python/ftcheck/ci/tsan.py +++ b/python/ftcheck/ci/tsan.py @@ -393,6 +393,22 @@ def _pyo3_notes(report: Report) -> list[str]: return list(notes) +def _thread_label(report: Report, section: Section, names: dict[str, str]) -> str | None: + """The accessing thread's name when TSan printed one, else its id (`T5`).""" + tid = report.access_thread(section) + return names.get(tid, tid) if tid else None + + +def _named_header(report: Report, section: Section, names: dict[str, str]) -> str: + """The access header without its address, the thread's name after its id — + `by thread T5 (ftm-refill)` says a mutator did it; `T5` alone needs the log.""" + header = _ADDRESS.sub("", section.header) + tid = report.access_thread(section) + if tid and tid in names: + header = header.replace(f"by thread {tid}", f"by thread {tid} ({names[tid]})", 1) + return header + + def _frames_from_top(section: Section) -> list[Frame]: """The stack from its first non-interceptor frame: where the access happened.""" frames = section.frames @@ -481,6 +497,7 @@ def to_findings( for report in reports: owner = attribute(report, extension_modules) + names = report.thread_names() if owner == YOURS: # Each side from where it matters: your frame when the access was # yours, otherwise where it happened — a Python callback mutating @@ -488,7 +505,7 @@ def to_findings( # the callback. stacks = [ ( - section.header, + section, _user_frames(section, extension_modules, crate_root) if (_first_real(section) or Frame("", None)).module in extension_modules else _frames_from_top(section), @@ -496,7 +513,7 @@ def to_findings( for section in report.access_stacks ] else: - stacks = [(section.header, _frames_from_top(section)) for section in report.access_stacks] + stacks = [(section, _frames_from_top(section)) for section in report.access_stacks] stacks.sort(key=lambda s: s[1][0].symbol if s[1] else "") if stacks: frame, path = _primary(report, extension_modules, crate_root) @@ -524,8 +541,8 @@ def to_findings( # Addresses differ on every run; a message that changes when nothing # else has is a message nobody can diff. described = "; ".join( - f"{_ADDRESS.sub('', header)} in `{frames[0].symbol}`".lstrip() - for header, frames in stacks + f"{_named_header(report, section, names)} in `{frames[0].symbol}`".lstrip() + for section, frames in stacks if frames ) described += "".join(_pyo3_notes(report)) @@ -553,8 +570,11 @@ def to_findings( "column": max(frame.column or 1, 1), }, "stacks": [ - {"frames": [_frame_json(f, crate_root) for f in frames]} - for _, frames in stacks + { + "frames": [_frame_json(f, crate_root) for f in frames], + "thread": _thread_label(report, section, names), + } + for section, frames in stacks ], "justification": None, "occurrences": 1, diff --git a/python/ftcheck/report/text.py b/python/ftcheck/report/text.py index 1f70312..8ac6737 100644 --- a/python/ftcheck/report/text.py +++ b/python/ftcheck/report/text.py @@ -115,6 +115,12 @@ def ci_json(env, outcome, code: int, headline: str, threads: int) -> dict[str, A } +def _thread(i: int, stack: dict) -> str: + """`thread 1 (ftm-refill)`: which thread made the access, when known.""" + name = stack.get("thread") + return f"thread {i} ({name})" if name else f"thread {i}" + + def _render_findings(outcome, stream: TextIO) -> None: """Warnings, findings in the user's extension, then reports outside it.""" for warning in outcome.warnings: @@ -139,7 +145,7 @@ def _render_findings(outcome, stream: TextIO) -> None: print(_wrap(f["message"], " "), file=stream) for i, stack in enumerate(f["stacks"], 1): frames = stack["frames"][:4] - print(f" thread {i}:", file=stream) + print(f" {_thread(i, stack)}:", file=stream) for fr in frames: print(f" {fr['symbol']} {_location(fr['location'])}", file=stream) print(file=stream) @@ -155,7 +161,7 @@ def _render_findings(outcome, stream: TextIO) -> None: for i, stack in enumerate(f["stacks"], 1): top = stack["frames"][:2] path = " <- ".join(fr["symbol"] for fr in top) - print(f" thread {i}: {path}", file=stream) + print(f" {_thread(i, stack)}: {path}", file=stream) if outcome.external: print( diff --git a/tests/test_tsan_parse.py b/tests/test_tsan_parse.py index 27ca5e2..ad40ac6 100644 --- a/tests/test_tsan_parse.py +++ b/tests/test_tsan_parse.py @@ -472,3 +472,36 @@ def test_pyo3_further_down_the_stack_is_not_mentioned(): ) (finding,), _, _ = to_findings([report], EXAMPLE, CRATE) assert "PyO3" not in finding["message"] + + +def test_messages_name_the_threads(): + """`thread T5` means nothing without the raw log; `ftm-refill_rows` says a + mutator did it.""" + _, _, other = to_findings(parse(load(MUTATOR_LOG)), EXAMPLE, "/src") + assert other + for f in other: + assert "thread T5 (ftm-refill_rows)" in f["message"] + assert {s["thread"] for s in f["stacks"]} <= {"ftm-refill_rows", "ftw-pair-2", "ftw-pair-4"} + + +def test_a_thread_without_a_name_keeps_its_id(): + report = _race([f"count /tmp/c/src/lib.rs:30:9 {EXT}"], [f"count /tmp/c/src/lib.rs:31:9 {EXT}"]) + (finding,), _, _ = to_findings([report], EXAMPLE, CRATE) + assert "by thread T1 in" in finding["message"] + assert sorted(s["thread"] for s in finding["stacks"]) == ["T1", "T2"] + + +def test_the_text_summary_labels_stacks_with_thread_names(): + import io + import types + + from ftcheck.report.text import _render_findings + + report = _race([f"count /tmp/c/src/lib.rs:30:9 {EXT}"], [f"count /tmp/c/src/lib.rs:31:9 {EXT}"], + writer_name="ftw-pair-3") + mine, _, _ = to_findings([report], EXAMPLE, CRATE) + outcome = types.SimpleNamespace(warnings=[], findings=mine, reached=[], external=[]) + out = io.StringIO() + _render_findings(outcome, out) + assert "thread 1 (T1):" in out.getvalue() or "thread 2 (T1):" in out.getvalue() + assert "(ftw-pair-3):" in out.getvalue() From 33fd3ff00c3e92c3a88756fed0e6cc62f8e7d1a4 Mon Sep 17 00:00:00 2001 From: nimbrel <331679006+nimbrel@users.noreply.github.com> Date: Sat, 26 Sep 2026 10:02:36 +0000 Subject: [PATCH 6/7] docs(changelog): record the attribution fixes under Unreleased Signed-off-by: nimbrel <331679006+nimbrel@users.noreply.github.com> --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3573346..ef4076d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## Unreleased + +### Fixed + +- **A mutator refilling another library's buffer is a harness race.** A mutator writing + into a numpy array (`arr[:] = ...`) reaches the memory through numpy's copy loop, not + CPython's, and was filed as the extension's `certain` race, failing the run. A copy + primitive called from a library outside the interpreter, with no frame of the extension + and no free, realloc or resize on the mutator's stack, is now a harness race. Harness + races are located at the extension's access. +- **A bare source file name is never a file in the crate.** An uninstrumented library's + debug info can name a file with no directory; resolved against the working directory + (the crate root, in the image) it became a finding's primary location and merge key. +- **Races outside the extension are located at the plain access**, not at an atomic + helper from CPython's `pyatomic*.h` on the other side. + +### Changed + +- Race messages name the threads (`by thread T5 (ftm-refill)`), the text summary labels + each stack with its thread, and JSON stacks carry a `thread` field. +- A race whose access is in PyO3's own source says so, with the PyO3 version and line. + ## 0.1.0 — 2026-09-25 First release. From d4af3254605649ce8cd9f33938809f15e36c5c08 Mon Sep 17 00:00:00 2001 From: nimbrel <331679006+nimbrel@users.noreply.github.com> Date: Sat, 26 Sep 2026 10:02:46 +0000 Subject: [PATCH 7/7] test(tsan): say which data log is trimmed rather than unedited Signed-off-by: nimbrel <331679006+nimbrel@users.noreply.github.com> --- tests/test_tsan_parse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_tsan_parse.py b/tests/test_tsan_parse.py index ad40ac6..5d2f064 100644 --- a/tests/test_tsan_parse.py +++ b/tests/test_tsan_parse.py @@ -4,6 +4,8 @@ The logs in tests/data/tsan/ are unedited output of ThreadSanitizer running fixtures under ghcr.io/nascheme/cpython-tsan:3.14t with nightly-2026-01-15 (LLVM 21 on both sides). The paths in them are the container's. +mutator-numpy-copy.log is the exception: trimmed to two reports, with the +extension's names and lines invented. """ import pathlib