From 7f3845b8848de7bf830d8d774f0687af538804b4 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Wed, 18 Mar 2026 12:49:53 +0100 Subject: [PATCH 1/3] [GR-72206] Suppress stdout shutdown lock noise --- .../src/tests/test_io.py | 23 ++++++++++++++++++- .../src/tests/unittest_tags/test_io.txt | 3 --- .../builtins/modules/io/BufferedIONodes.java | 4 ++-- .../graal/python/runtime/PythonContext.java | 10 +++++++- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_io.py b/graalpython/com.oracle.graal.python.test/src/tests/test_io.py index 0faeb0597c..97a6b72d16 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_io.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_io.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -38,6 +38,7 @@ # SOFTWARE. import _io import os +import subprocess import sys import tempfile import unittest @@ -219,6 +220,26 @@ def test_exit_accepts_varargs(self): with self.assertRaises(TypeError): x.__exit__(kw=1) + def test_shutdown_stdout_lock_error_does_not_leak_to_stderr(self): + code = """import sys + +class StdoutAtShutdown: + closed = False + + def write(self, data): + return len(data) + + def flush(self): + raise SystemError( + "could not acquire lock for <_io.BufferedWriter name=''> " + "at interpreter shutdown, possibly due to daemon threads") + +sys.stdout = StdoutAtShutdown() +""" + proc = subprocess.run([sys.executable, "-c", code], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.stderr, b"") + def test_isatty(self): x = _io._IOBase() self.assertFalse(x.isatty()) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_io.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_io.txt index 1963683e66..18e9c5e61a 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_io.txt +++ b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_io.txt @@ -195,9 +195,6 @@ test.test_io.CMiscIOTest.test_attributes @ darwin-arm64,linux-aarch64,linux-aarc test.test_io.CMiscIOTest.test_check_encoding_warning @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_io.CMiscIOTest.test_create_fail @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_io.CMiscIOTest.test_create_writes @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github -# GR-72206 -!test.test_io.CMiscIOTest.test_daemon_threads_shutdown_stderr_deadlock -!test.test_io.CMiscIOTest.test_daemon_threads_shutdown_stdout_deadlock test.test_io.CMiscIOTest.test_io_after_close @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_io.CMiscIOTest.test_nonblock_pipe_write_bigbuf @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_io.CMiscIOTest.test_nonblock_pipe_write_smallbuf @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedIONodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedIONodes.java index cd33e41480..c126f5f927 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedIONodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedIONodes.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -464,7 +464,7 @@ static void finalizing(Node inliningTarget, PBuffered self, * written threaded I/O code. */ if (!self.getLock().acquireTimeout(inliningTarget, (long) 1e3)) { - throw lazyRaise.raise(inliningTarget, SystemError, SHUTDOWN_POSSIBLY_DUE_TO_DAEMON_THREADS); + throw lazyRaise.raise(inliningTarget, SystemError, SHUTDOWN_POSSIBLY_DUE_TO_DAEMON_THREADS, self); } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index 1b91e7d4d7..4882eb60c6 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -2035,6 +2035,9 @@ public void flushStdFiles() { flushFile(sysModule.getAttribute(T_STDERR), false); } + private static final String SHUTDOWN_LOCK_ERROR_PREFIX = "could not acquire lock for "; + private static final String SHUTDOWN_LOCK_ERROR_SUFFIX = " at interpreter shutdown, possibly due to daemon threads"; + private static void flushFile(Object file, boolean useWriteUnraisable) { if (!(file instanceof PNone)) { boolean closed = false; @@ -2047,7 +2050,7 @@ private static void flushFile(Object file, boolean useWriteUnraisable) { try { PyObjectCallMethodObjArgs.executeUncached(file, T_FLUSH); } catch (PException e) { - if (useWriteUnraisable) { + if (useWriteUnraisable && !isDaemonThreadShutdownLockError(e)) { WriteUnraisableNode.getUncached().execute(e.getEscapedException(), null, null); } } @@ -2055,6 +2058,11 @@ private static void flushFile(Object file, boolean useWriteUnraisable) { } } + private static boolean isDaemonThreadShutdownLockError(PException e) { + String message = ExceptionUtils.getExceptionMessage(e.getUnreifiedException()); + return message != null && message.contains(SHUTDOWN_LOCK_ERROR_PREFIX) && message.endsWith(SHUTDOWN_LOCK_ERROR_SUFFIX); + } + @TruffleBoundary public int getAtexitHookCount() { return atExitHooks.size(); From c37428cdab4574ccbcffa91d5b35727f812f1e20 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Wed, 18 Mar 2026 16:26:44 +0100 Subject: [PATCH 2/3] [GR-72206] Align stdout shutdown handling with CPython --- .../src/tests/test_io.py | 51 +++++++++++++++++-- .../graal/python/runtime/PythonContext.java | 29 +++++++---- 2 files changed, 68 insertions(+), 12 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_io.py b/graalpython/com.oracle.graal.python.test/src/tests/test_io.py index 97a6b72d16..f29f2da4e1 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_io.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_io.py @@ -45,6 +45,9 @@ class IOBaseTests(unittest.TestCase): + @staticmethod + def run_in_subprocess(code): + return subprocess.run([sys.executable, "-c", code], stdout=subprocess.PIPE, stderr=subprocess.PIPE) def test_iobase_ctor_accepts_anything(self): _io._IOBase() @@ -236,9 +239,51 @@ def flush(self): sys.stdout = StdoutAtShutdown() """ - proc = subprocess.run([sys.executable, "-c", code], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - self.assertEqual(proc.returncode, 0) - self.assertEqual(proc.stderr, b"") + proc = self.run_in_subprocess(code) + stderr = proc.stderr.decode("utf-8", "replace") + self.assertEqual(proc.stdout, b"") + self.assertEqual(proc.returncode, 120) + self.assertRegex( + stderr, + r"Exception ignored in: <__main__\.StdoutAtShutdown object at 0x[0-9a-fA-F]+>\n" + r"Traceback \(most recent call last\):\n" + r' File "", line 10, in flush\n' + r"SystemError: could not acquire lock for <_io\.BufferedWriter name=''> " + r"at interpreter shutdown, possibly due to daemon threads\n?$", + ) + + def test_shutdown_stdout_deadlock_matches_cpython(self): + code = """if 1: + import sys + import time + import threading + + file = sys.stdout + + def run(): + while True: + file.write('.') + file.flush() + + thread = threading.Thread(target=run) + thread.daemon = True + thread.start() + + time.sleep(0.5) + file.write('!') + file.flush() +""" + proc = self.run_in_subprocess(code) + stderr = proc.stderr.decode("utf-8", "replace") + if proc.returncode != 0: + self.assertRegex( + stderr, + r"Fatal Python error: _enter_buffered_busy: could not acquire lock " + r"for <(_io\.)?BufferedWriter name=''> at interpreter shutdown, " + r"possibly due to daemon threads", + ) + else: + self.assertEqual(stderr, "") def test_isatty(self): x = _io._IOBase() diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index 4882eb60c6..ac1de432f9 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -39,6 +39,7 @@ import static com.oracle.graal.python.nodes.BuiltinNames.T_SHA3; import static com.oracle.graal.python.nodes.BuiltinNames.T_STDERR; import static com.oracle.graal.python.nodes.BuiltinNames.T_STDOUT; +import static com.oracle.graal.python.nodes.BuiltinNames.T___STDOUT__; import static com.oracle.graal.python.nodes.BuiltinNames.T_SYS; import static com.oracle.graal.python.nodes.BuiltinNames.T_THREADING; import static com.oracle.graal.python.nodes.BuiltinNames.T___BUILTINS__; @@ -160,6 +161,7 @@ import com.oracle.graal.python.runtime.arrow.ArrowSupport; import com.oracle.graal.python.runtime.exception.ExceptionUtils; import com.oracle.graal.python.runtime.exception.PException; +import com.oracle.graal.python.runtime.exception.PythonExitException; import com.oracle.graal.python.runtime.exception.PythonThreadKillException; import com.oracle.graal.python.runtime.locale.PythonLocale; import com.oracle.graal.python.runtime.object.IDUtils; @@ -1998,6 +2000,7 @@ public void registerCApiHook(Runnable hook) { @SuppressWarnings("try") public void finalizeContext() { boolean cancelling = env.getContext().isCancelling(); + boolean stdioFlushFailed = false; try (GilNode.UncachedAcquire gil = GilNode.uncachedAcquire()) { if (!cancelling) { // this uses the threading module and runs python code to join the threads @@ -2010,7 +2013,7 @@ public void finalizeContext() { finalizing = true; // interrupt and join or kill python threads joinPythonThreads(); - flushStdFiles(); + stdioFlushFailed = flushStdFiles(); if (cApiContext != null) { cApiContext.finalizeCApi(); } @@ -2025,20 +2028,24 @@ public void finalizeContext() { } } mainThread = null; + if (stdioFlushFailed) { + throw new PythonExitException(null, 120); + } } // Equivalent of CPython's flush_std_files @TruffleBoundary - public void flushStdFiles() { + public boolean flushStdFiles() { PythonModule sysModule = getSysModule(); - flushFile(sysModule.getAttribute(T_STDOUT), true); - flushFile(sysModule.getAttribute(T_STDERR), false); + Object stdout = sysModule.getAttribute(T_STDOUT); + return flushFile(stdout, sysModule.getAttribute(T___STDOUT__), true) | + flushFile(sysModule.getAttribute(T_STDERR), null, false); } private static final String SHUTDOWN_LOCK_ERROR_PREFIX = "could not acquire lock for "; private static final String SHUTDOWN_LOCK_ERROR_SUFFIX = " at interpreter shutdown, possibly due to daemon threads"; - private static void flushFile(Object file, boolean useWriteUnraisable) { + private static boolean flushFile(Object file, Object originalStdout, boolean useWriteUnraisable) { if (!(file instanceof PNone)) { boolean closed = false; try { @@ -2050,17 +2057,21 @@ private static void flushFile(Object file, boolean useWriteUnraisable) { try { PyObjectCallMethodObjArgs.executeUncached(file, T_FLUSH); } catch (PException e) { - if (useWriteUnraisable && !isDaemonThreadShutdownLockError(e)) { - WriteUnraisableNode.getUncached().execute(e.getEscapedException(), null, null); + if (useWriteUnraisable) { + if (!isDaemonThreadShutdownLockError(file, originalStdout, e)) { + WriteUnraisableNode.getUncached().execute(e.getEscapedException(), null, file); + return true; + } } } } } + return false; } - private static boolean isDaemonThreadShutdownLockError(PException e) { + private static boolean isDaemonThreadShutdownLockError(Object file, Object originalStdout, PException e) { String message = ExceptionUtils.getExceptionMessage(e.getUnreifiedException()); - return message != null && message.contains(SHUTDOWN_LOCK_ERROR_PREFIX) && message.endsWith(SHUTDOWN_LOCK_ERROR_SUFFIX); + return file == originalStdout && message != null && message.contains(SHUTDOWN_LOCK_ERROR_PREFIX) && message.endsWith(SHUTDOWN_LOCK_ERROR_SUFFIX); } @TruffleBoundary From 5aac017589c95169e0830e1126b1e322b8684ced Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Wed, 18 Mar 2026 22:27:00 +0100 Subject: [PATCH 3/3] [GR-72206] Normalize shutdown stderr newlines in test --- graalpython/com.oracle.graal.python.test/src/tests/test_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_io.py b/graalpython/com.oracle.graal.python.test/src/tests/test_io.py index f29f2da4e1..f8108e209b 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_io.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_io.py @@ -240,7 +240,7 @@ def flush(self): sys.stdout = StdoutAtShutdown() """ proc = self.run_in_subprocess(code) - stderr = proc.stderr.decode("utf-8", "replace") + stderr = proc.stderr.decode("utf-8", "replace").replace("\r\n", "\n") self.assertEqual(proc.stdout, b"") self.assertEqual(proc.returncode, 120) self.assertRegex(