diff --git a/Doc/Makefile b/Doc/Makefile index 5b7fdf8ec08ed4..7bdabd8bf168fe 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -88,6 +88,7 @@ htmlhelp: build "build/htmlhelp/pydoc.hhp project file." .PHONY: latex +latex: _ensure-sphinxcontrib-svg2pdfconverter latex: BUILDER = latex latex: build @echo "Build finished; the LaTeX files are in build/latex." @@ -231,7 +232,7 @@ dist-text: @echo "Build finished and archived!" .PHONY: dist-pdf -dist-pdf: +dist-pdf: _ensure-sphinxcontrib-svg2pdfconverter # archive the A4 latex @echo "Building LaTeX (A4 paper)..." mkdir -p dist @@ -292,6 +293,10 @@ _ensure-pre-commit: _ensure-sphinx-autobuild: $(MAKE) _ensure-package PACKAGE=sphinx-autobuild +.PHONY: _ensure-sphinxcontrib-svg2pdfconverter +_ensure-sphinxcontrib-svg2pdfconverter: + $(MAKE) _ensure-package PACKAGE=sphinxcontrib-svg2pdfconverter + .PHONY: check check: _ensure-pre-commit $(VENVDIR)/bin/python3 -m pre_commit run --all-files diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 8cadf26cee3027..c9bb5c3f09ac18 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -274,6 +274,10 @@ Type Objects Return the module object associated with the given type when the type was created using :c:func:`PyType_FromModuleAndSpec`. + The returned reference is :term:`borrowed ` from *type*, + and will be valid as long as you hold a reference to *type*. + Do not release it with :c:func:`Py_DECREF` or similar. + If no module is associated with the given type, sets :py:class:`TypeError` and returns ``NULL``. diff --git a/Doc/conf.py b/Doc/conf.py index 545049bb460419..26497083d28e47 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -46,6 +46,7 @@ 'linklint.ext', 'notfound.extension', 'sphinxext.opengraph', + 'sphinxcontrib.rsvgconverter', ) for optional_ext in _OPTIONAL_EXTENSIONS: try: diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 64399f6ab1ff26..01b064f3e617ff 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -2427,6 +2427,9 @@ PyType_GetFlags:PyTypeObject*:type:0: PyType_GetName:PyObject*::+1: PyType_GetName:PyTypeObject*:type:0: +PyType_GetModule:PyObject*::0: +PyType_GetModule:PyTypeObject*:type:0: + PyType_GetModuleByToken:PyObject*::+1: PyType_GetModuleByToken:PyTypeObject*:type:0: PyType_GetModuleByToken:PyModuleDef*:def:: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 3ea24ea77004ad..a32c3828313454 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -156,7 +156,9 @@ And:: print(f.result()) executor = ThreadPoolExecutor(max_workers=1) - executor.submit(wait_on_future) + future = executor.submit(wait_on_future) + # Note: calling future.result() would also cause a deadlock because + # the single worker thread is already waiting for wait_on_future(). .. class:: ThreadPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=()) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index c23e81e29df0f5..fcbe2122d9f1a7 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -20,10 +20,6 @@ used to wrap these libraries in pure Python. ctypes tutorial --------------- -Note: The code samples in this tutorial use :mod:`doctest` to make sure that -they actually work. Since some code samples behave differently under Linux, -Windows, or macOS, they contain doctest directives in comments. - Note: Some code samples reference the ctypes :class:`c_int` type. On platforms where ``sizeof(long) == sizeof(int)`` it is an alias to :class:`c_long`. So, you should not be confused if :class:`c_long` is printed if you would expect @@ -34,13 +30,16 @@ So, you should not be confused if :class:`c_long` is printed if you would expect Loading dynamic link libraries ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`!ctypes` exports the *cdll*, and on Windows *windll* and *oledll* +:mod:`!ctypes` exports the :py:data:`~ctypes.cdll`, and on Windows +:py:data:`~ctypes.windll` and :py:data:`~ctypes.oledll` objects, for loading dynamic link libraries. -You load libraries by accessing them as attributes of these objects. *cdll* -loads libraries which export functions using the standard ``cdecl`` calling -convention, while *windll* libraries call functions using the ``stdcall`` -calling convention. *oledll* also uses the ``stdcall`` calling convention, and +You load libraries by accessing them as attributes of these objects. +:py:data:`!cdll` loads libraries which export functions using the +standard ``cdecl`` calling convention, while :py:data:`!windll` +libraries call functions using the ``stdcall`` +calling convention. +:py:data:`~oledll` also uses the ``stdcall`` calling convention, and assumes the functions return a Windows :c:type:`!HRESULT` error code. The error code is used to automatically raise an :class:`OSError` exception when the function call fails. @@ -70,11 +69,13 @@ Windows appends the usual ``.dll`` file suffix automatically. being used by Python. Where possible, use native Python functionality, or else import and use the ``msvcrt`` module. -On Linux, it is required to specify the filename *including* the extension to +Other systems require the filename *including* the extension to load a library, so attribute access can not be used to load libraries. Either the :meth:`~LibraryLoader.LoadLibrary` method of the dll loaders should be used, -or you should load the library by creating an instance of CDLL by calling -the constructor:: +or you should load the library by creating an instance of :py:class:`CDLL` +by calling the constructor. + +For example, on Linux:: >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX @@ -83,7 +84,14 @@ the constructor:: >>> -.. XXX Add section for macOS. +On macOS:: + + >>> cdll.LoadLibrary("libc.dylib") # doctest: +MACOS + + >>> libc = CDLL("libc.dylib") # doctest: +MACOS + >>> libc # doctest: +MACOS + + .. _ctypes-accessing-functions-from-loaded-dlls: @@ -1456,14 +1464,82 @@ Loading shared libraries ^^^^^^^^^^^^^^^^^^^^^^^^ There are several ways to load shared libraries into the Python process. One -way is to instantiate one of the following classes: +way is to instantiate :py:class:`CDLL` or one of its subclasses: .. class:: CDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - Instances of this class represent loaded shared libraries. Functions in these - libraries use the standard C calling convention, and are assumed to return - :c:expr:`int`. + Represents a loaded shared library. + + Functions in this library use the standard C calling convention, and are + assumed to return :c:expr:`int`. + The Python :term:`global interpreter lock` is released before calling any + function exported by these libraries, and reacquired afterwards. + For different function behavior, use a subclass: :py:class:`~ctypes.OleDLL`, + :py:class:`~ctypes.WinDLL`, or :py:class:`~ctypes.PyDLL`. + + If you have an existing :py:attr:`handle ` to an already + loaded shared library, it can be passed as the *handle* argument to wrap + the opened library in a new :py:class:`!CDLL` object. + In this case, *name* is only used to set the :py:attr:`~ctypes.CDLL._name` + attribute, but it may be adjusted and/or validated. + + If *handle* is ``None``, the underlying platform's :manpage:`dlopen(3)` or + :c:func:`!LoadLibrary` function is used to load the library into + the process, and to get a handle to it. + + *name* is the pathname of the shared library to open. + If *name* does not contain a path separator, the library is found + in a platform-specific way. + + On non-Windows systems, *name* can be ``None``. In this case, + :c:func:`!dlopen` is called with ``NULL``, which opens the main program + as a "library". + (Some systems do the same is *name* is empty; ``None``/``NULL`` is more + portable.) + + .. admonition:: CPython implementation detail + + Since CPython is linked to ``libc``, a ``None`` *name* is often used + to access the C standard library:: + + >>> printf = ctypes.CDLL(None).printf + >>> printf.argtypes = [ctypes.c_char_p] + >>> printf(b"hello\n") + hello + 6 + + To access the Python C API, prefer :py:data:`ctypes.pythonapi` which + works across platforms. + + The *mode* parameter can be used to specify how the library is loaded. For + details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is + ignored. On posix systems, RTLD_NOW is always added, and is not + configurable. + + The *use_errno* parameter, when set to true, enables a ctypes mechanism that + allows accessing the system :data:`errno` error number in a safe way. + :mod:`!ctypes` maintains a thread-local copy of the system's :data:`errno` + variable; if you call foreign functions created with ``use_errno=True`` then the + :data:`errno` value before the function call is swapped with the ctypes private + copy, the same happens immediately after the function call. + + The function :func:`ctypes.get_errno` returns the value of the ctypes private + copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy + to a new value and returns the former value. + + The *use_last_error* parameter, when set to true, enables the same mechanism for + the Windows error code which is managed by the :func:`GetLastError` and + :func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and + :func:`ctypes.set_last_error` are used to request and change the ctypes private + copy of the windows error code. + + The *winmode* parameter is used on Windows to specify how the library is loaded + (since *mode* is ignored). It takes any value that is valid for the Win32 API + ``LoadLibraryEx`` flags parameter. When omitted, the default is to use the + flags that result in the most secure DLL load, which avoids issues such as DLL + hijacking. Passing the full path to the DLL is the safest way to ensure the + correct library and dependencies are loaded. On Windows creating a :class:`CDLL` instance may fail even if the DLL name exists. When a dependent DLL of the loaded DLL is not found, a @@ -1475,20 +1551,47 @@ way is to instantiate one of the following classes: DLLs and determine which one is not found using Windows debugging and tracing tools. + .. seealso:: + + `Microsoft DUMPBIN tool `_ + -- A tool to find DLL dependents. + + .. versionchanged:: 3.8 + Added *winmode* parameter. + .. versionchanged:: 3.12 The *name* parameter can now be a :term:`path-like object`. -.. seealso:: + Instances of this class have no public methods. Functions exported by the + shared library can be accessed as attributes or by index. Please note that + accessing the function through an attribute caches the result and therefore + accessing it repeatedly returns the same object each time. On the other hand, + accessing it through an index returns a new object each time:: + + >>> from ctypes import CDLL + >>> libc = CDLL("libc.so.6") # On Linux + >>> libc.time == libc.time + True + >>> libc['time'] == libc['time'] + False + + The following public attributes are available. Their name starts with an + underscore to not clash with exported function names: + + .. attribute:: _handle + + The system handle used to access the library. - `Microsoft DUMPBIN tool `_ - -- A tool to find DLL dependents. + .. attribute:: _name + The name of the library passed in the constructor. -.. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) +.. class:: OleDLL - Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are + See :py:class:`~ctypes.CDLL`, the superclass, for common information. + + Functions in this library use the ``stdcall`` calling convention, and are assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT` values contain information specifying whether the function call failed or succeeded, together with additional error code. If the return value signals a @@ -1500,133 +1603,51 @@ way is to instantiate one of the following classes: :exc:`WindowsError` used to be raised, which is now an alias of :exc:`OSError`. - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. +.. class:: WinDLL -.. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) + See :py:class:`~ctypes.CDLL`, the superclass, for common information. - Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are + Functions in these libraries use the ``stdcall`` calling convention, and are assumed to return :c:expr:`int` by default. .. availability:: Windows - .. versionchanged:: 3.12 +.. class:: PyDLL - The *name* parameter can now be a :term:`path-like object`. + See :py:class:`~ctypes.CDLL`, the superclass, for common information. - -The Python :term:`global interpreter lock` is released before calling any -function exported by these libraries, and reacquired afterwards. - - -.. class:: PyDLL(name, mode=DEFAULT_MODE, handle=None) - - Instances of this class behave like :class:`CDLL` instances, except that the + When functions in this library are called, the Python GIL is *not* released during the function call, and after the function execution the Python error flag is checked. If the error flag is set, a Python exception is raised. - Thus, this is only useful to call Python C api functions directly. - - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. - -All these classes can be instantiated by calling them with at least one -argument, the pathname of the shared library. If you have an existing handle to -an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platform's :c:func:`!dlopen` or -:c:func:`!LoadLibrary` function is used to load the library into -the process, and to get a handle to it. - -The *mode* parameter can be used to specify how the library is loaded. For -details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is -ignored. On posix systems, RTLD_NOW is always added, and is not -configurable. - -The *use_errno* parameter, when set to true, enables a ctypes mechanism that -allows accessing the system :data:`errno` error number in a safe way. -:mod:`!ctypes` maintains a thread-local copy of the system's :data:`errno` -variable; if you call foreign functions created with ``use_errno=True`` then the -:data:`errno` value before the function call is swapped with the ctypes private -copy, the same happens immediately after the function call. - -The function :func:`ctypes.get_errno` returns the value of the ctypes private -copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy -to a new value and returns the former value. - -The *use_last_error* parameter, when set to true, enables the same mechanism for -the Windows error code which is managed by the :func:`GetLastError` and -:func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and -:func:`ctypes.set_last_error` are used to request and change the ctypes private -copy of the windows error code. - -The *winmode* parameter is used on Windows to specify how the library is loaded -(since *mode* is ignored). It takes any value that is valid for the Win32 API -``LoadLibraryEx`` flags parameter. When omitted, the default is to use the -flags that result in the most secure DLL load, which avoids issues such as DLL -hijacking. Passing the full path to the DLL is the safest way to ensure the -correct library and dependencies are loaded. - -.. versionchanged:: 3.8 - Added *winmode* parameter. + Thus, this is only useful to call Python C API functions directly. .. data:: RTLD_GLOBAL - :noindex: Flag to use as *mode* parameter. On platforms where this flag is not available, it is defined as the integer zero. .. data:: RTLD_LOCAL - :noindex: Flag to use as *mode* parameter. On platforms where this is not available, it is the same as *RTLD_GLOBAL*. .. data:: DEFAULT_MODE - :noindex: The default mode which is used to load shared libraries. On OSX 10.3, this is *RTLD_GLOBAL*, otherwise it is the same as *RTLD_LOCAL*. -Instances of these classes have no public methods. Functions exported by the -shared library can be accessed as attributes or by index. Please note that -accessing the function through an attribute caches the result and therefore -accessing it repeatedly returns the same object each time. On the other hand, -accessing it through an index returns a new object each time:: - - >>> from ctypes import CDLL - >>> libc = CDLL("libc.so.6") # On Linux - >>> libc.time == libc.time - True - >>> libc['time'] == libc['time'] - False - -The following public attributes are available, their name starts with an -underscore to not clash with exported function names: - - -.. attribute:: PyDLL._handle - - The system handle used to access the library. - - -.. attribute:: PyDLL._name - - The name of the library passed in the constructor. Shared libraries can also be loaded by using one of the prefabricated objects, which are instances of the :class:`LibraryLoader` class, either by calling the :meth:`~LibraryLoader.LoadLibrary` method, or by retrieving the library as attribute of the loader instance. - .. class:: LibraryLoader(dlltype) Class which loads shared libraries. *dlltype* should be one of the @@ -1645,13 +1666,11 @@ attribute of the loader instance. These prefabricated library loaders are available: .. data:: cdll - :noindex: Creates :class:`CDLL` instances. .. data:: windll - :noindex: Creates :class:`WinDLL` instances. @@ -1659,7 +1678,6 @@ These prefabricated library loaders are available: .. data:: oledll - :noindex: Creates :class:`OleDLL` instances. @@ -1667,7 +1685,6 @@ These prefabricated library loaders are available: .. data:: pydll - :noindex: Creates :class:`PyDLL` instances. @@ -1676,7 +1693,6 @@ For accessing the C Python api directly, a ready-to-use Python shared library object is available: .. data:: pythonapi - :noindex: An instance of :class:`PyDLL` that exposes Python C API functions as attributes. Note that all these functions are assumed to return C diff --git a/Doc/library/profiling.sampling.rst b/Doc/library/profiling.sampling.rst index 078062c08c6020..d2b7d9669ab07e 100644 --- a/Doc/library/profiling.sampling.rst +++ b/Doc/library/profiling.sampling.rst @@ -1194,10 +1194,12 @@ data, similar to the ``top`` command for system processes:: python -m profiling.sampling run --live script.py python -m profiling.sampling attach --live 12345 -.. figure:: tachyon-live-mode-2.gif - :alt: Tachyon live mode showing all threads - :align: center - :width: 100% +.. only:: not latex + + .. figure:: tachyon-live-mode-2.gif + :alt: Tachyon live mode showing all threads + :align: center + :width: 100% Live mode displays real-time profiling statistics, showing combined data from multiple threads in a multi-threaded application. @@ -1217,10 +1219,12 @@ main table, showing instruction-level statistics for the currently selected function. This panel displays which bytecode instructions are executing most frequently, including specialized variants and their base opcodes. -.. figure:: tachyon-live-mode-1.gif - :alt: Tachyon live mode with opcode panel - :align: center - :width: 100% +.. only:: not latex + + .. figure:: tachyon-live-mode-1.gif + :alt: Tachyon live mode with opcode panel + :align: center + :width: 100% Live mode with ``--opcodes`` enabled shows an opcode panel with a bytecode instruction breakdown for the selected function. diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 321a4e5d5d18fb..224b1883808a41 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -265,7 +265,7 @@ def _try_finish(self): # to avoid hanging forever in self._wait as otherwise _exit_waiters # would never be woken up, we wake them up here. for waiter in self._exit_waiters: - if not waiter.cancelled(): + if not waiter.done(): waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): @@ -278,7 +278,7 @@ def _call_connection_lost(self, exc): finally: # wake up futures waiting for wait() for waiter in self._exit_waiters: - if not waiter.cancelled(): + if not waiter.done(): waiter.set_result(self._returncode) self._exit_waiters = None self._loop = None diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index bf301740741ae7..c08eb7cf261568 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -111,6 +111,37 @@ def test_subprocess_repr(self): ) transport.close() + def test_proc_exited_no_invalid_state_error_on_exit_waiters(self): + # gh-145541: when _connect_pipes hasn't completed (so + # _pipes_connected is False) and the process exits, _try_finish() + # sets the result on exit waiters. Then _call_connection_lost() must + # not call set_result() again on the same waiters. + self.loop.set_exception_handler( + lambda loop, context: self.fail( + f"unexpected exception: {context}") + ) + waiter = self.loop.create_future() + transport, protocol = self.create_transport(waiter) + + # Simulate a waiter registered via _wait() before the process exits. + exit_waiter = self.loop.create_future() + transport._exit_waiters.append(exit_waiter) + + # _connect_pipes hasn't completed, so _pipes_connected is False. + self.assertFalse(transport._pipes_connected) + + # Simulate process exit. _try_finish() will set the result on + # exit_waiter because _pipes_connected is False, and then schedule + # _call_connection_lost() because _pipes is empty (vacuously all + # disconnected). _call_connection_lost() must skip exit_waiter + # because it's already done. + transport._process_exited(6) + self.loop.run_until_complete(waiter) + + self.assertEqual(exit_waiter.result(), 6) + + transport.close() + class SubprocessMixin: diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index 5bdf74ef73ab54..cd46fea5476ca6 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -619,6 +619,16 @@ def test_frozendict_new(self): self.assertEqual(dct, frozendict(x=1, y=2)) self.assertIs(type(dct), frozendict) + # PyFrozenDict_New(frozendict) returns the same object unmodified + fd = frozendict(a=1, b=2, c=3) + fd2 = frozendict_new(fd) + self.assertIs(fd2, fd) + + fd = FrozenDictSubclass(a=1, b=2, c=3) + fd2 = frozendict_new(fd) + self.assertIsNot(fd2, fd) + self.assertEqual(fd2, fd) + # PyFrozenDict_New(NULL) creates an empty dictionary dct = frozendict_new(NULL) self.assertEqual(dct, frozendict()) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 45448d1264a53e..b2f4363b23e748 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1829,6 +1829,13 @@ def test_constructor(self): with self.assertRaises(TypeError): dict.__init__(d, x=1) + # Avoid copy if it's frozendict type + d2 = frozendict(d) + self.assertIs(d2, d) + d2 = FrozenDict(d) + self.assertIsNot(d2, d) + self.assertEqual(d2, d) + def test_copy(self): d = frozendict(x=1, y=2) d2 = d.copy() diff --git a/Makefile.pre.in b/Makefile.pre.in index 120a6add38507f..4c2426ed283d43 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1729,6 +1729,9 @@ FROZEN_FILES_IN = \ Lib/zipimport.py \ Lib/abc.py \ Lib/codecs.py \ + Lib/encodings/__init__.py \ + Lib/encodings/aliases.py \ + Lib/encodings/utf_8.py \ Lib/io.py \ Lib/_collections_abc.py \ Lib/_sitebuiltins.py \ @@ -1738,6 +1741,7 @@ FROZEN_FILES_IN = \ Lib/os.py \ Lib/site.py \ Lib/stat.py \ + Lib/linecache.py \ Lib/importlib/util.py \ Lib/importlib/machinery.py \ Lib/runpy.py \ @@ -1754,6 +1758,9 @@ FROZEN_FILES_OUT = \ Python/frozen_modules/zipimport.h \ Python/frozen_modules/abc.h \ Python/frozen_modules/codecs.h \ + Python/frozen_modules/encodings.h \ + Python/frozen_modules/encodings.aliases.h \ + Python/frozen_modules/encodings.utf_8.h \ Python/frozen_modules/io.h \ Python/frozen_modules/_collections_abc.h \ Python/frozen_modules/_sitebuiltins.h \ @@ -1763,6 +1770,7 @@ FROZEN_FILES_OUT = \ Python/frozen_modules/os.h \ Python/frozen_modules/site.h \ Python/frozen_modules/stat.h \ + Python/frozen_modules/linecache.h \ Python/frozen_modules/importlib.util.h \ Python/frozen_modules/importlib.machinery.h \ Python/frozen_modules/runpy.h \ @@ -1802,6 +1810,15 @@ Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS) Python/frozen_modules/codecs.h: Lib/codecs.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h +Python/frozen_modules/encodings.h: Lib/encodings/__init__.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) encodings $(srcdir)/Lib/encodings/__init__.py Python/frozen_modules/encodings.h + +Python/frozen_modules/encodings.aliases.h: Lib/encodings/aliases.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) encodings.aliases $(srcdir)/Lib/encodings/aliases.py Python/frozen_modules/encodings.aliases.h + +Python/frozen_modules/encodings.utf_8.h: Lib/encodings/utf_8.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) encodings.utf_8 $(srcdir)/Lib/encodings/utf_8.py Python/frozen_modules/encodings.utf_8.h + Python/frozen_modules/io.h: Lib/io.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h @@ -1829,6 +1846,9 @@ Python/frozen_modules/site.h: Lib/site.py $(FREEZE_MODULE_DEPS) Python/frozen_modules/stat.h: Lib/stat.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h +Python/frozen_modules/linecache.h: Lib/linecache.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) linecache $(srcdir)/Lib/linecache.py Python/frozen_modules/linecache.h + Python/frozen_modules/importlib.util.h: Lib/importlib/util.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-02-26-21-22-34.gh-issue-145278.DHkYqt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-26-21-22-34.gh-issue-145278.DHkYqt.rst new file mode 100644 index 00000000000000..6b6a9eb9813663 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-26-21-22-34.gh-issue-145278.DHkYqt.rst @@ -0,0 +1,4 @@ +The :mod:`encodings` is now partially frozen, including +the ``aliases`` and ``utf_8`` submodules. + +The :mod:`linecache` is now frozen. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-06-21-05-05.gh-issue-145615.NKXXZgDW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-06-21-05-05.gh-issue-145615.NKXXZgDW.rst new file mode 100644 index 00000000000000..2183eef618daae --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-06-21-05-05.gh-issue-145615.NKXXZgDW.rst @@ -0,0 +1,2 @@ +Fixed a memory leak in the :term:`free-threaded build` where mimalloc pages +could become permanently unreclaimable until the owning thread exited. diff --git a/Misc/NEWS.d/next/Library/2026-03-05-19-01-28.gh-issue-145551.gItPRl.rst b/Misc/NEWS.d/next/Library/2026-03-05-19-01-28.gh-issue-145551.gItPRl.rst new file mode 100644 index 00000000000000..15b70d734ca3b9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-05-19-01-28.gh-issue-145551.gItPRl.rst @@ -0,0 +1 @@ +Fix InvalidStateError when cancelling process created by :func:`asyncio.create_subprocess_exec` or :func:`asyncio.create_subprocess_shell`. Patch by Daan De Meyer. diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 61464348d6fab8..fe9d6fe2763f36 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -1112,11 +1112,13 @@ _curses_window_addstr_impl(PyCursesWindowObject *self, int group_left_1, attr_old = getattrs(self->win); if (curses_wattrset(self, attr, "addstr") < 0) { curses_release_wstr(strtype, wstr); + Py_XDECREF(bytesobj); return NULL; } } #ifdef HAVE_NCURSESW if (strtype == 2) { + assert(bytesobj == NULL); if (use_xy) { rtn = mvwaddwstr(self->win,y,x,wstr); funcname = "mvwaddwstr"; @@ -1130,6 +1132,9 @@ _curses_window_addstr_impl(PyCursesWindowObject *self, int group_left_1, else #endif { +#ifdef HAVE_NCURSESW + assert(wstr == NULL); +#endif const char *str = PyBytes_AS_STRING(bytesobj); if (use_xy) { rtn = mvwaddstr(self->win,y,x,str); @@ -1210,6 +1215,7 @@ _curses_window_addnstr_impl(PyCursesWindowObject *self, int group_left_1, attr_old = getattrs(self->win); if (curses_wattrset(self, attr, "addnstr") < 0) { curses_release_wstr(strtype, wstr); + Py_XDECREF(bytesobj); return NULL; } } @@ -2212,6 +2218,7 @@ _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, attr_old = getattrs(self->win); if (curses_wattrset(self, attr, "insstr") < 0) { curses_release_wstr(strtype, wstr); + Py_XDECREF(bytesobj); return NULL; } } diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 025a3fac46e59b..a2d1aefb1611b3 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -702,6 +702,7 @@ PyObject* get_cfunc_from_callable(PyObject* callable, PyObject* self_arg, PyObje if (PyCFunction_Check(meth)) { return (PyObject*)((PyCFunctionObject *)meth); } + Py_DECREF(meth); } return NULL; } @@ -961,6 +962,8 @@ profiler_traverse(PyObject *op, visitproc visit, void *arg) ProfilerObject *self = ProfilerObject_CAST(op); Py_VISIT(Py_TYPE(op)); Py_VISIT(self->externalTimer); + Py_VISIT(self->missing); + return 0; } @@ -979,6 +982,7 @@ profiler_dealloc(PyObject *op) flush_unmatched(self); clearEntries(self); + Py_XDECREF(self->missing); Py_XDECREF(self->externalTimer); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1017,7 +1021,7 @@ profiler_init_impl(ProfilerObject *self, PyObject *timer, double timeunit, if (!monitoring) { return -1; } - self->missing = PyObject_GetAttrString(monitoring, "MISSING"); + Py_XSETREF(self->missing, PyObject_GetAttrString(monitoring, "MISSING")); if (!self->missing) { Py_DECREF(monitoring); return -1; diff --git a/Modules/binascii.c b/Modules/binascii.c index e6cd64338064b3..3f3695d50f2754 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -924,7 +924,7 @@ binascii_a2b_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, } unsigned char *bin_data = PyBytesWriter_GetData(writer); if (bin_data == NULL) { - return NULL; + goto error; } uint32_t leftchar = 0; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ff0e2fd2b3569d..bc25bf6bfc1bd2 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -3531,23 +3531,26 @@ count_traverse(PyObject *op, visitproc visit, void *arg) static PyObject * count_nextlong(countobject *lz) { - PyObject *long_cnt; - PyObject *stepped_up; - - long_cnt = lz->long_cnt; - if (long_cnt == NULL) { + if (lz->long_cnt == NULL) { /* Switch to slow_mode */ - long_cnt = PyLong_FromSsize_t(PY_SSIZE_T_MAX); - if (long_cnt == NULL) + lz->long_cnt = PyLong_FromSsize_t(PY_SSIZE_T_MAX); + if (lz->long_cnt == NULL) { return NULL; + } } - assert(lz->cnt == PY_SSIZE_T_MAX && long_cnt != NULL); + assert(lz->cnt == PY_SSIZE_T_MAX && lz->long_cnt != NULL); + + // We hold one reference to "result" (a.k.a. the old value of + // lz->long_cnt); we'll either return it or keep it in lz->long_cnt. + PyObject *result = lz->long_cnt; - stepped_up = PyNumber_Add(long_cnt, lz->long_step); - if (stepped_up == NULL) + PyObject *stepped_up = PyNumber_Add(result, lz->long_step); + if (stepped_up == NULL) { return NULL; + } lz->long_cnt = stepped_up; - return long_cnt; + + return result; } static PyObject * diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 61fde37f8d4fff..b5f2a682c54982 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5169,15 +5169,47 @@ dict_vectorcall(PyObject *type, PyObject * const*args, return NULL; } - PyObject *self; - if (Py_Is((PyTypeObject*)type, &PyFrozenDict_Type) - || PyType_IsSubtype((PyTypeObject*)type, &PyFrozenDict_Type)) - { - self = frozendict_new(_PyType_CAST(type), NULL, NULL); + PyObject *self = dict_new(_PyType_CAST(type), NULL, NULL); + if (self == NULL) { + return NULL; } - else { - self = dict_new(_PyType_CAST(type), NULL, NULL); + if (nargs == 1) { + if (dict_update_arg(self, args[0]) < 0) { + Py_DECREF(self); + return NULL; + } + args++; } + if (kwnames != NULL) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(kwnames); i++) { + PyObject *key = PyTuple_GET_ITEM(kwnames, i); // borrowed + if (PyDict_SetItem(self, key, args[i]) < 0) { + Py_DECREF(self); + return NULL; + } + } + } + return self; +} + +static PyObject * +frozendict_vectorcall(PyObject *type, PyObject * const*args, + size_t nargsf, PyObject *kwnames) +{ + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + if (!_PyArg_CheckPositional("frozendict", nargs, 0, 1)) { + return NULL; + } + + if (nargs == 1 && kwnames == NULL + && PyFrozenDict_CheckExact(args[0]) + && Py_Is((PyTypeObject*)type, &PyFrozenDict_Type)) + { + // frozendict(frozendict) returns the same object unmodified + return Py_NewRef(args[0]); + } + + PyObject *self = frozendict_new(_PyType_CAST(type), NULL, NULL); if (self == NULL) { return NULL; } @@ -8171,6 +8203,11 @@ PyObject* PyFrozenDict_New(PyObject *iterable) { if (iterable != NULL) { + if (PyFrozenDict_CheckExact(iterable)) { + // PyFrozenDict_New(frozendict) returns the same object unmodified + return Py_NewRef(iterable); + } + PyObject *args = PyTuple_Pack(1, iterable); if (args == NULL) { return NULL; @@ -8228,6 +8265,6 @@ PyTypeObject PyFrozenDict_Type = { .tp_alloc = _PyType_AllocNoTrack, .tp_new = frozendict_new, .tp_free = PyObject_GC_Del, - .tp_vectorcall = dict_vectorcall, + .tp_vectorcall = frozendict_vectorcall, .tp_version_tag = _Py_TYPE_VERSION_FROZENDICT, }; diff --git a/Objects/enumobject.c b/Objects/enumobject.c index 814ce4f919514b..70e7cce6aba008 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -178,14 +178,16 @@ enum_traverse(PyObject *op, visitproc visit, void *arg) static inline PyObject * increment_longindex_lock_held(enumobject *en) { - PyObject *next_index = en->en_longindex; - if (next_index == NULL) { - next_index = PyLong_FromSsize_t(PY_SSIZE_T_MAX); - if (next_index == NULL) { + if (en->en_longindex == NULL) { + en->en_longindex = PyLong_FromSsize_t(PY_SSIZE_T_MAX); + if (en->en_longindex == NULL) { return NULL; } } - assert(next_index != NULL); + assert(en->en_longindex != NULL); + // We hold one reference to "next_index" (a.k.a. the old value of + // en->en_longindex); we'll either return it or keep it in en->en_longindex + PyObject *next_index = en->en_longindex; PyObject *stepped_up = PyNumber_Add(next_index, en->one); if (stepped_up == NULL) { return NULL; diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 119dd4b5c2dd00..7aef56cf4e93b8 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -299,6 +299,8 @@ subs_tvars(PyObject *obj, PyObject *params, &PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg)); if (j < 0) { + Py_DECREF(subparams); + Py_DECREF(subargs); return NULL; } continue; @@ -455,6 +457,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje if (is_args_list) { args = tuple_args = PySequence_Tuple(args); if (args == NULL) { + Py_DECREF(item); return NULL; } } diff --git a/Objects/listobject.c b/Objects/listobject.c index 3921b7cd7b69bc..7fc21907fefd31 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -4294,7 +4294,9 @@ listiter_reduce_general(void *_it, int forward) } /* empty iterator, create an empty list */ list = PyList_New(0); - if (list == NULL) + if (list == NULL) { + Py_DECREF(iter); return NULL; + } return Py_BuildValue("N(N)", iter, list); } diff --git a/Objects/mimalloc/heap.c b/Objects/mimalloc/heap.c index d92dc768e5ec28..5fbfb82baa0204 100644 --- a/Objects/mimalloc/heap.c +++ b/Objects/mimalloc/heap.c @@ -100,7 +100,10 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t // note: this will free retired pages as well. bool freed = _PyMem_mi_page_maybe_free(page, pq, collect >= MI_FORCE); if (!freed && collect == MI_ABANDON) { - _mi_page_abandon(page, pq); + // _PyMem_mi_page_maybe_free may have moved the page to a different + // page queue, so we need to re-fetch the correct queue. + uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : _mi_bin(page->xblock_size)); + _mi_page_abandon(page, &heap->pages[bin]); } } else if (collect == MI_ABANDON) { diff --git a/Objects/mimalloc/page.c b/Objects/mimalloc/page.c index ff7444cce10923..ded59f8eb1ccaa 100644 --- a/Objects/mimalloc/page.c +++ b/Objects/mimalloc/page.c @@ -213,6 +213,13 @@ static void _mi_page_thread_free_collect(mi_page_t* page) // update counts now page->used -= count; + + if (page->used == 0) { + // The page may have had a QSBR goal set from a previous point when it + // was all-free. That goal is no longer valid because the page was + // allocated from and then freed again by other threads. + _PyMem_mi_page_clear_qsbr(page); + } } void _mi_page_free_collect(mi_page_t* page, bool force) { @@ -225,9 +232,6 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { // and the local free list if (page->local_free != NULL) { - // any previous QSBR goals are no longer valid because we reused the page - _PyMem_mi_page_clear_qsbr(page); - if mi_likely(page->free == NULL) { // usual case page->free = page->local_free; diff --git a/Objects/mimalloc/segment.c b/Objects/mimalloc/segment.c index 9b092b9b734d4c..9dad69c995e7a0 100644 --- a/Objects/mimalloc/segment.c +++ b/Objects/mimalloc/segment.c @@ -1286,6 +1286,7 @@ static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, s _mi_stat_decrease(&tld->stats->pages_abandoned, 1); #ifdef Py_GIL_DISABLED page->qsbr_goal = 0; + mi_assert_internal(page->qsbr_node.next == NULL); #endif segment->abandoned--; slice = mi_segment_page_clear(page, tld); // re-assign slice due to coalesce! @@ -1361,6 +1362,7 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, // if everything free by now, free the page #ifdef Py_GIL_DISABLED page->qsbr_goal = 0; + mi_assert_internal(page->qsbr_node.next == NULL); #endif slice = mi_segment_page_clear(page, tld); // set slice again due to coalesceing } diff --git a/Objects/object.c b/Objects/object.c index b537c0d104e58c..e405963614689f 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1295,6 +1295,7 @@ _PyObject_SetAttributeErrorContext(PyObject* v, PyObject* name) // Augment the exception with the name and object if (PyObject_SetAttr(exc, &_Py_ID(name), name) || PyObject_SetAttr(exc, &_Py_ID(obj), v)) { + Py_DECREF(exc); return 1; } restore: @@ -3077,9 +3078,9 @@ Py_ReprEnter(PyObject *obj) list = PyList_New(0); if (list == NULL) return -1; - if (PyDict_SetItem(dict, &_Py_ID(Py_Repr), list) < 0) + if (_PyDict_SetItem_Take2((PyDictObject *)dict, &_Py_ID(Py_Repr), list) < 0) { return -1; - Py_DECREF(list); + } } i = PyList_GET_SIZE(list); while (--i >= 0) { diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index b59ebdfbda3897..983bdddbf026a8 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -151,6 +151,12 @@ should_advance_qsbr_for_page(struct _qsbr_thread_state *qsbr, mi_page_t *page) } return false; } + +static _PyThreadStateImpl * +tstate_from_heap(mi_heap_t *heap) +{ + return _Py_CONTAINER_OF(heap->tld, _PyThreadStateImpl, mimalloc.tld); +} #endif static bool @@ -159,23 +165,35 @@ _PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force) #ifdef Py_GIL_DISABLED assert(mi_page_all_free(page)); if (page->use_qsbr) { - _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); - if (page->qsbr_goal != 0 && _Py_qbsr_goal_reached(tstate->qsbr, page->qsbr_goal)) { + struct _qsbr_thread_state *qsbr = ((_PyThreadStateImpl *)PyThreadState_GET())->qsbr; + if (page->qsbr_goal != 0 && _Py_qbsr_goal_reached(qsbr, page->qsbr_goal)) { _PyMem_mi_page_clear_qsbr(page); _mi_page_free(page, pq, force); return true; } + // gh-145615: since we are not freeing this page yet, we want to + // make it available for allocations. Note that the QSBR goal and + // linked list node remain set even if the page is later used for + // an allocation. We only detect and clear the QSBR goal when the + // page becomes empty again (used == 0). + if (mi_page_is_in_full(page)) { + _mi_page_unfull(page); + } + _PyMem_mi_page_clear_qsbr(page); page->retire_expire = 0; - if (should_advance_qsbr_for_page(tstate->qsbr, page)) { - page->qsbr_goal = _Py_qsbr_advance(tstate->qsbr->shared); + if (should_advance_qsbr_for_page(qsbr, page)) { + page->qsbr_goal = _Py_qsbr_advance(qsbr->shared); } else { - page->qsbr_goal = _Py_qsbr_shared_next(tstate->qsbr->shared); + page->qsbr_goal = _Py_qsbr_shared_next(qsbr->shared); } + // We may be freeing a page belonging to a different thread during a + // stop-the-world event. Find the _PyThreadStateImpl for the page. + _PyThreadStateImpl *tstate = tstate_from_heap(mi_page_heap(page)); llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); return false; } @@ -192,7 +210,8 @@ _PyMem_mi_page_reclaimed(mi_page_t *page) if (page->qsbr_goal != 0) { if (mi_page_all_free(page)) { assert(page->qsbr_node.next == NULL); - _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); + _PyThreadStateImpl *tstate = tstate_from_heap(mi_page_heap(page)); + assert(tstate == (_PyThreadStateImpl *)_PyThreadState_GET()); page->retire_expire = 0; llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); } diff --git a/Objects/structseq.c b/Objects/structseq.c index 7a159338b9ba8a..b8bb041f0cff21 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -515,7 +515,8 @@ initialize_structseq_dict(PyStructSequence_Desc *desc, PyObject* dict, } if (_PyTuple_Resize(&keys, k) == -1) { - goto error; + assert(keys == NULL); + return -1; } if (PyDict_SetItemString(dict, match_args_key, keys) < 0) { diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 2ec546aff52c0a..0a260f4c10278c 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -2305,13 +2305,12 @@ generic_class_getitem(PyObject *cls, PyObject *args, PyObject *kwargs) PyObject * _Py_subscript_generic(PyThreadState* unused, PyObject *params) { - params = unpack_typevartuples(params); - PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->cached_objects.generic_type == NULL) { PyErr_SetString(PyExc_SystemError, "Cannot find Generic type"); return NULL; } + params = unpack_typevartuples(params); PyObject *args[2] = {(PyObject *)interp->cached_objects.generic_type, params}; PyObject *result = call_typing_func_object("_GenericAlias", args, 2); Py_DECREF(params); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7aa85a942e449e..7756f1a8482477 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5220,7 +5220,7 @@ unicode_decode_utf8_impl(_PyUnicodeWriter *writer, } if (_PyUnicodeWriter_Prepare(writer, end - s, 127) < 0) { - return -1; + goto onError; } } } diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index cb806459596084..40946cafd16b75 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -310,6 +310,21 @@ $(IntDir)codecs.g.h $(GeneratedFrozenModulesDir)Python\frozen_modules\codecs.h + + encodings + $(IntDir)encodings.g.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\encodings.h + + + encodings.aliases + $(IntDir)encodings.aliases.g.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\encodings.aliases.h + + + encodings.utf_8 + $(IntDir)encodings.utf_8.g.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\encodings.utf_8.h + io $(IntDir)io.g.h @@ -355,6 +370,11 @@ $(IntDir)stat.g.h $(GeneratedFrozenModulesDir)Python\frozen_modules\stat.h + + linecache + $(IntDir)linecache.g.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\linecache.h + importlib.util $(IntDir)importlib.util.g.h diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 6dcf0e8712903a..a0e36a4c526ede 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -537,6 +537,15 @@ Python Files + + Python Files + + + Python Files + + + Python Files + Python Files @@ -564,6 +573,9 @@ Python Files + + Python Files + Python Files diff --git a/Python/frozen.c b/Python/frozen.c index 15d256b6743e0a..72a291d0d43f1d 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -46,6 +46,9 @@ #include "frozen_modules/zipimport.h" #include "frozen_modules/abc.h" #include "frozen_modules/codecs.h" +#include "frozen_modules/encodings.h" +#include "frozen_modules/encodings.aliases.h" +#include "frozen_modules/encodings.utf_8.h" #include "frozen_modules/io.h" #include "frozen_modules/_collections_abc.h" #include "frozen_modules/_sitebuiltins.h" @@ -55,6 +58,7 @@ #include "frozen_modules/os.h" #include "frozen_modules/site.h" #include "frozen_modules/stat.h" +#include "frozen_modules/linecache.h" #include "frozen_modules/importlib.util.h" #include "frozen_modules/importlib.machinery.h" #include "frozen_modules/runpy.h" @@ -76,6 +80,9 @@ static const struct _frozen stdlib_modules[] = { /* stdlib - startup, without site (python -S) */ {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), false}, {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), false}, + {"encodings", _Py_M__encodings, (int)sizeof(_Py_M__encodings), true}, + {"encodings.aliases", _Py_M__encodings_aliases, (int)sizeof(_Py_M__encodings_aliases), false}, + {"encodings.utf_8", _Py_M__encodings_utf_8, (int)sizeof(_Py_M__encodings_utf_8), false}, {"io", _Py_M__io, (int)sizeof(_Py_M__io), false}, /* stdlib - startup, with site */ @@ -88,6 +95,9 @@ static const struct _frozen stdlib_modules[] = { {"site", _Py_M__site, (int)sizeof(_Py_M__site), false}, {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), false}, + /* pythonrun - interactive */ + {"linecache", _Py_M__linecache, (int)sizeof(_Py_M__linecache), false}, + /* runpy - run module with -m */ {"importlib.util", _Py_M__importlib_util, (int)sizeof(_Py_M__importlib_util), false}, {"importlib.machinery", _Py_M__importlib_machinery, (int)sizeof(_Py_M__importlib_machinery), false}, diff --git a/Python/optimizer.c b/Python/optimizer.c index f485c27bca2a4f..4387bcb0d67832 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1076,12 +1076,19 @@ _PyJit_FinalizeTracing(PyThreadState *tstate, int err) exit->temperature = initial_temperature_backoff_counter(&tstate->interp->opt_config); } } + // Clear all recorded values + _PyJitUopBuffer *buffer = &tracer->code_buffer; + for (_PyUOpInstruction *inst = buffer->start; inst < buffer->next; inst++) { + if (_PyUop_Flags[inst->opcode] & HAS_RECORDS_VALUE_FLAG) { + Py_XDECREF((PyObject *)(uintptr_t)inst->operand0); + } + } Py_CLEAR(tracer->initial_state.code); Py_CLEAR(tracer->initial_state.func); Py_CLEAR(tracer->initial_state.executor); Py_CLEAR(tracer->prev_state.instr_code); Py_CLEAR(tracer->prev_state.recorded_value); - uop_buffer_init(&tracer->code_buffer, &tracer->uop_array[0], UOP_MAX_TRACE_LENGTH); + uop_buffer_init(buffer, &tracer->uop_array[0], UOP_MAX_TRACE_LENGTH); tracer->is_tracing = false; } @@ -1521,6 +1528,11 @@ uop_optimize( } assert(_PyOpcode_uop_name[buffer[pc].opcode]); } + // We've cleaned up the references in the buffer, so discard the code buffer + // to avoid doing it again during tracer cleanup + _PyJitUopBuffer *code_buffer = &_tstate->jit_tracer_state->code_buffer; + code_buffer->next = code_buffer->start; + OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); _PyUOpInstruction *output = &_tstate->jit_tracer_state->uop_array[0]; length = stack_allocate(buffer, output, length); diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index 3c43f7e3bbe8ca..0d1f5968d2a9e9 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -50,10 +50,9 @@ ('stdlib - startup, without site (python -S)', [ 'abc', 'codecs', - # For now we do not freeze the encodings, due # to the noise all - # those extra modules add to the text printed during the build. - # (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.) - #'', + '', + 'encodings.aliases', + 'encodings.utf_8', 'io', ]), ('stdlib - startup, with site', [ @@ -66,6 +65,9 @@ 'site', 'stat', ]), + ('pythonrun - interactive', [ + 'linecache', + ]), ('runpy - run module with -m', [ "importlib.util", "importlib.machinery",