diff --git a/Doc/conf.py b/Doc/conf.py index 26497083d28e47..6f66ed68c52e83 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -557,6 +557,7 @@ # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html extlinks = { + "oss-fuzz": ("https://issues.oss-fuzz.com/issues/%s", "#%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), "source": (SOURCE_URI, "%s"), } diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 7a6f88d90a9ea5..8bd2bc99d74b83 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -8,11 +8,11 @@ Programming FAQ .. contents:: -General Questions +General questions ================= -Is there a source code level debugger with breakpoints, single-stepping, etc.? ------------------------------------------------------------------------------- +Is there a source code-level debugger with breakpoints and single-stepping? +--------------------------------------------------------------------------- Yes. @@ -25,8 +25,7 @@ Reference Manual `. You can also write your own debugger by using the code for pdb as an example. The IDLE interactive development environment, which is part of the standard -Python distribution (normally available as -`Tools/scripts/idle3 `_), +Python distribution (normally available as :mod:`idlelib`), includes a graphical debugger. PythonWin is a Python IDE that includes a GUI debugger based on pdb. The @@ -48,7 +47,6 @@ There are a number of commercial Python IDEs that include graphical debuggers. They include: * `Wing IDE `_ -* `Komodo IDE `_ * `PyCharm `_ @@ -57,13 +55,15 @@ Are there tools to help find bugs or perform static analysis? Yes. -`Pylint `_ and -`Pyflakes `_ do basic checking that will +`Ruff `__, +`Pylint `__ and +`Pyflakes `__ do basic checking that will help you catch bugs sooner. -Static type checkers such as `Mypy `_, -`Pyre `_, and -`Pytype `_ can check type hints in Python +Static type checkers such as `mypy `__, +`ty `__, +`Pyrefly `__, and +`pytype `__ can check type hints in Python source code. @@ -79,7 +79,7 @@ set of modules required by a program and bind these modules together with a Python binary to produce a single executable. One is to use the freeze tool, which is included in the Python source tree as -`Tools/freeze `_. +:source:`Tools/freeze`. It converts Python byte code to C arrays; with a C compiler you can embed all your modules into a new program, which is then linked with the standard Python modules. @@ -103,6 +103,7 @@ executables: * `py2app `_ (macOS only) * `py2exe `_ (Windows only) + Are there coding standards or a style guide for Python programs? ---------------------------------------------------------------- @@ -110,7 +111,7 @@ Yes. The coding style required for standard library modules is documented as :pep:`8`. -Core Language +Core language ============= .. _faq-unboundlocalerror: @@ -143,7 +144,7 @@ results in an :exc:`!UnboundLocalError`: >>> foo() Traceback (most recent call last): ... - UnboundLocalError: local variable 'x' referenced before assignment + UnboundLocalError: cannot access local variable 'x' where it is not associated with a value This is because when you make an assignment to a variable in a scope, that variable becomes local to that scope and shadows any similarly named variable @@ -208,7 +209,7 @@ Why do lambdas defined in a loop with different values all return the same resul ---------------------------------------------------------------------------------- Assume you use a for loop to define a few different lambdas (or even plain -functions), e.g.:: +functions), for example:: >>> squares = [] >>> for x in range(5): @@ -227,7 +228,7 @@ they all return ``16``:: This happens because ``x`` is not local to the lambdas, but is defined in the outer scope, and it is accessed when the lambda is called --- not when it is defined. At the end of the loop, the value of ``x`` is ``4``, so all the -functions now return ``4**2``, i.e. ``16``. You can also verify this by +functions now return ``4**2``, that is ``16``. You can also verify this by changing the value of ``x`` and see how the results of the lambdas change:: >>> x = 8 @@ -298,9 +299,9 @@ using multiple imports per line uses less screen space. It's good practice if you import modules in the following order: -1. standard library modules -- e.g. :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` +1. standard library modules -- such as :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` 2. third-party library modules (anything installed in Python's site-packages - directory) -- e.g. :mod:`!dateutil`, :mod:`!requests`, :mod:`!PIL.Image` + directory) -- such as :pypi:`dateutil`, :pypi:`requests`, :pypi:`tzdata` 3. locally developed modules It is sometimes necessary to move imports to a function or class to avoid @@ -494,11 +495,11 @@ new objects). In other words: -* If we have a mutable object (:class:`list`, :class:`dict`, :class:`set`, - etc.), we can use some specific operations to mutate it and all the variables +* If we have a mutable object (such as :class:`list`, :class:`dict`, :class:`set`), + we can use some specific operations to mutate it and all the variables that refer to it will see the change. -* If we have an immutable object (:class:`str`, :class:`int`, :class:`tuple`, - etc.), all the variables that refer to it will always see the same value, +* If we have an immutable object (such as :class:`str`, :class:`int`, :class:`tuple`), + all the variables that refer to it will always see the same value, but operations that transform that value into a new value always return a new object. @@ -511,7 +512,7 @@ How do I write a function with output parameters (call by reference)? Remember that arguments are passed by assignment in Python. Since assignment just creates references to objects, there's no alias between an argument name in -the caller and callee, and so no call-by-reference per se. You can achieve the +the caller and callee, and consequently no call-by-reference. You can achieve the desired effect in a number of ways. 1) By returning a tuple of the results:: @@ -714,8 +715,8 @@ not:: "a" in ("b", "a") -The same is true of the various assignment operators (``=``, ``+=`` etc). They -are not truly operators but syntactic delimiters in assignment statements. +The same is true of the various assignment operators (``=``, ``+=``, and so on). +They are not truly operators but syntactic delimiters in assignment statements. Is there an equivalent of C's "?:" ternary operator? @@ -868,9 +869,9 @@ with either a space or parentheses. How do I convert a string to a number? -------------------------------------- -For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') +For integers, use the built-in :func:`int` type constructor, for example, ``int('144') == 144``. Similarly, :func:`float` converts to a floating-point number, -e.g. ``float('144') == 144.0``. +for example, ``float('144') == 144.0``. By default, these interpret the number as decimal, so that ``int('0144') == 144`` holds true, and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, @@ -887,18 +888,18 @@ unwanted side effects. For example, someone could pass directory. :func:`eval` also has the effect of interpreting numbers as Python expressions, -so that e.g. ``eval('09')`` gives a syntax error because Python does not allow +so that, for example, ``eval('09')`` gives a syntax error because Python does not allow leading '0' in a decimal number (except '0'). How do I convert a number to a string? -------------------------------------- -To convert, e.g., the number ``144`` to the string ``'144'``, use the built-in type +For example, to convert the number ``144`` to the string ``'144'``, use the built-in type constructor :func:`str`. If you want a hexadecimal or octal representation, use the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see -the :ref:`f-strings` and :ref:`formatstrings` sections, -e.g. ``"{:04d}".format(144)`` yields +the :ref:`f-strings` and :ref:`formatstrings` sections. +For example, ``"{:04d}".format(144)`` yields ``'0144'`` and ``"{:.3f}".format(1.0/3.0)`` yields ``'0.333'``. @@ -908,7 +909,7 @@ How do I modify a string in place? You can't, because strings are immutable. In most situations, you should simply construct a new string from the various parts you want to assemble it from. However, if you need an object with the ability to modify in-place -unicode data, try using an :class:`io.StringIO` object or the :mod:`array` +Unicode data, try using an :class:`io.StringIO` object or the :mod:`array` module:: >>> import io @@ -1066,13 +1067,14 @@ the raw string:: Also see the specification in the :ref:`language reference `. + Performance =========== My program is too slow. How do I speed it up? --------------------------------------------- -That's a tough one, in general. First, here are a list of things to +That's a tough one, in general. First, here is list of things to remember before diving further: * Performance characteristics vary across Python implementations. This FAQ @@ -1125,6 +1127,7 @@ yourself. The wiki page devoted to `performance tips `_. + .. _efficient_string_concatenation: What is the most efficient way to concatenate many strings together? @@ -1143,7 +1146,7 @@ them into a list and call :meth:`str.join` at the end:: chunks.append(s) result = ''.join(chunks) -(another reasonably efficient idiom is to use :class:`io.StringIO`) +(Another reasonably efficient idiom is to use :class:`io.StringIO`.) To accumulate many :class:`bytes` objects, the recommended idiom is to extend a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: @@ -1153,7 +1156,7 @@ a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: result += b -Sequences (Tuples/Lists) +Sequences (tuples/lists) ======================== How do I convert between tuples and lists? @@ -1217,8 +1220,8 @@ list, deleting duplicates as you go:: else: last = mylist[i] -If all elements of the list may be used as set keys (i.e. they are all -:term:`hashable`) this is often faster :: +If all elements of the list may be used as set keys (that is, they are all +:term:`hashable`) this is often faster:: mylist = list(set(mylist)) @@ -1254,7 +1257,7 @@ difference is that a Python list can contain objects of many different types. The ``array`` module also provides methods for creating arrays of fixed types with compact representations, but they are slower to index than lists. Also note that `NumPy `_ -and other third party packages define array-like structures with +and other third-party packages define array-like structures with various characteristics as well. To get Lisp-style linked lists, you can emulate *cons cells* using tuples:: @@ -1324,7 +1327,7 @@ Or, you can use an extension that provides a matrix datatype; `NumPy How do I apply a method or function to a sequence of objects? ------------------------------------------------------------- -To call a method or function and accumulate the return values is a list, +To call a method or function and accumulate the return values in a list, a :term:`list comprehension` is an elegant solution:: result = [obj.method() for obj in mylist] @@ -1340,6 +1343,7 @@ a plain :keyword:`for` loop will suffice:: for obj in mylist: function(obj) + .. _faq-augmented-assignment-tuple-error: Why does a_tuple[i] += ['item'] raise an exception when the addition works? @@ -1444,7 +1448,7 @@ How can I sort one list by values from another list? ---------------------------------------------------- Merge them into an iterator of tuples, sort the resulting list, and then pick -out the element you want. :: +out the element you want. >>> list1 = ["what", "I'm", "sorting", "by"] >>> list2 = ["something", "else", "to", "sort"] @@ -1504,14 +1508,15 @@ How do I check if an object is an instance of a given class or of a subclass of Use the built-in function :func:`isinstance(obj, cls) `. You can check if an object is an instance of any of a number of classes by providing a tuple instead of a -single class, e.g. ``isinstance(obj, (class1, class2, ...))``, and can also -check whether an object is one of Python's built-in types, e.g. +single class, for example, ``isinstance(obj, (class1, class2, ...))``, and can also +check whether an object is one of Python's built-in types, for example, ``isinstance(obj, str)`` or ``isinstance(obj, (int, float, complex))``. Note that :func:`isinstance` also checks for virtual inheritance from an :term:`abstract base class`. So, the test will return ``True`` for a registered class even if hasn't directly or indirectly inherited from it. To -test for "true inheritance", scan the :term:`MRO` of the class: +test for "true inheritance", scan the :term:`method resolution order` (MRO) of +the class: .. testcode:: @@ -1574,7 +1579,7 @@ call it:: What is delegation? ------------------- -Delegation is an object oriented technique (also called a design pattern). +Delegation is an object-oriented technique (also called a design pattern). Let's say you have an object ``x`` and want to change the behaviour of just one of its methods. You can create a new class that provides a new implementation of the method you're interested in changing and delegates all other methods to @@ -1645,7 +1650,7 @@ How can I organize my code to make it easier to change the base class? You could assign the base class to an alias and derive from the alias. Then all you have to change is the value assigned to the alias. Incidentally, this trick -is also handy if you want to decide dynamically (e.g. depending on availability +is also handy if you want to decide dynamically (such as depending on availability of resources) which base class to use. Example:: class Base: @@ -1710,9 +1715,9 @@ How can I overload constructors (or methods) in Python? This answer actually applies to all methods, but the question usually comes up first in the context of constructors. -In C++ you'd write +In C++ you'd write: -.. code-block:: c +.. code-block:: c++ class C { C() { cout << "No arguments\n"; } @@ -1731,7 +1736,7 @@ default arguments. For example:: This is not entirely equivalent, but close enough in practice. -You could also try a variable-length argument list, e.g. :: +You could also try a variable-length argument list, for example:: def __init__(self, *args): ... @@ -1774,6 +1779,7 @@ to use private variable names at all. The :ref:`private name mangling specifications ` for details and special cases. + My class defines __del__ but it is not called when I delete the object. ----------------------------------------------------------------------- @@ -1783,7 +1789,7 @@ The :keyword:`del` statement does not necessarily call :meth:`~object.__del__` - decrements the object's reference count, and if this reaches zero :meth:`!__del__` is called. -If your data structures contain circular links (e.g. a tree where each child has +If your data structures contain circular links (for example, a tree where each child has a parent reference and each parent has a list of children) the reference counts will never go back to zero. Once in a while Python runs an algorithm to detect such cycles, but the garbage collector might run some time after the last @@ -1885,9 +1891,9 @@ are preferred. In particular, identity tests should not be used to check constants such as :class:`int` and :class:`str` which aren't guaranteed to be singletons:: - >>> a = 1000 - >>> b = 500 - >>> c = b + 500 + >>> a = 10_000_000 + >>> b = 5_000_000 + >>> c = b + 5_000_000 >>> a is c False @@ -1956,9 +1962,9 @@ parent class: .. testcode:: - from datetime import date + import datetime as dt - class FirstOfMonthDate(date): + class FirstOfMonthDate(dt.date): "Always choose the first day of the month" def __new__(cls, year, month, day): return super().__new__(cls, year, month, 1) @@ -2001,7 +2007,7 @@ The two principal tools for caching methods are former stores results at the instance level and the latter at the class level. -The *cached_property* approach only works with methods that do not take +The ``cached_property`` approach only works with methods that do not take any arguments. It does not create a reference to the instance. The cached method result will be kept only as long as the instance is alive. @@ -2010,7 +2016,7 @@ method result will be released right away. The disadvantage is that if instances accumulate, so too will the accumulated method results. They can grow without bound. -The *lru_cache* approach works with methods that have :term:`hashable` +The ``lru_cache`` approach works with methods that have :term:`hashable` arguments. It creates a reference to the instance unless special efforts are made to pass in weak references. @@ -2044,11 +2050,11 @@ This example shows the various techniques:: # Depends on the station_id, date, and units. The above example assumes that the *station_id* never changes. If the -relevant instance attributes are mutable, the *cached_property* approach +relevant instance attributes are mutable, the ``cached_property`` approach can't be made to work because it cannot detect changes to the attributes. -To make the *lru_cache* approach work when the *station_id* is mutable, +To make the ``lru_cache`` approach work when the *station_id* is mutable, the class needs to define the :meth:`~object.__eq__` and :meth:`~object.__hash__` methods so that the cache can detect relevant attribute updates:: @@ -2094,10 +2100,10 @@ one user but run as another, such as if you are testing with a web server. Unless the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable is set, creation of a .pyc file is automatic if you're importing a module and Python -has the ability (permissions, free space, etc...) to create a ``__pycache__`` +has the ability (permissions, free space, and so on) to create a ``__pycache__`` subdirectory and write the compiled module to that subdirectory. -Running Python on a top level script is not considered an import and no +Running Python on a top-level script is not considered an import and no ``.pyc`` will be created. For example, if you have a top-level module ``foo.py`` that imports another module ``xyz.py``, when you run ``foo`` (by typing ``python foo.py`` as a shell command), a ``.pyc`` will be created for @@ -2116,7 +2122,7 @@ the ``compile()`` function in that module interactively:: This will write the ``.pyc`` to a ``__pycache__`` subdirectory in the same location as ``foo.py`` (or you can override that with the optional parameter -``cfile``). +*cfile*). You can also automatically compile all files in a directory or directories using the :mod:`compileall` module. You can do it from the shell prompt by running @@ -2221,7 +2227,7 @@ changed module, do this:: importlib.reload(modname) Warning: this technique is not 100% fool-proof. In particular, modules -containing statements like :: +containing statements like:: from modname import some_objects diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 3a485a43a5a751..412005f3ec82f4 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -6,8 +6,6 @@ Installing Python Modules ************************* -:Email: distutils-sig@python.org - As a popular open source development project, Python has an active supporting community of contributors and users that also make their software available for other Python developers to use under open source license terms. diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py index 0c3b1d3a9108a3..916bb25d74dee9 100644 --- a/Lib/_ast_unparse.py +++ b/Lib/_ast_unparse.py @@ -738,9 +738,13 @@ def visit_SetComp(self, node): def visit_DictComp(self, node): with self.delimit("{", "}"): - self.traverse(node.key) - self.write(": ") - self.traverse(node.value) + if node.value: + self.traverse(node.key) + self.write(": ") + self.traverse(node.value) + else: + self.write("**") + self.traverse(node.key) for gen in node.generators: self.traverse(gen) diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 71f1e616116d81..faa3a2bfe121dc 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -349,6 +349,8 @@ def test_annotations(self): eq("(i ** 2 + j for i in (1, 2, 3) for j in (1, 2, 3))") eq("{i: 0 for i in (1, 2, 3)}") eq("{i: j for i, j in ((1, 'a'), (2, 'b'), (3, 'c'))}") + eq("{**x for x in ()}") + eq("[*x for x in ()]") eq("[(x, y) for x, y in (a, b)]") eq("[(x,) for x, in (a,)]") eq("Python3 > Python2 > COBOL") diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 70148dc30fc957..cee528722b85aa 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -180,6 +180,18 @@ def test_references___class___defined(self): code, outputs={"res": [2]}, scopes=["module", "function"]) self._check_in_scopes(code, raises=NameError, scopes=["class"]) + def test_references___classdict__(self): + code = """ + class i: [__classdict__ for x in y] + """ + self._check_in_scopes(code, raises=NameError) + + def test_references___conditional_annotations__(self): + code = """ + class i: [__conditional_annotations__ for x in y] + """ + self._check_in_scopes(code, raises=NameError) + def test_references___class___enclosing(self): code = """ __class__ = 2 diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 35e4652a87b423..dcaad49ffab5d2 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -403,6 +403,11 @@ def test_set_comprehension(self): def test_dict_comprehension(self): self.check_ast_roundtrip("{x: x*x for x in range(10)}") + def test_dict_comprehension_unpacking(self): + self.check_ast_roundtrip("{**x for x in ()}") + self.check_ast_roundtrip("{**x for x in range(10)}") + self.check_ast_roundtrip("[*x for x in ()]") + def test_class_decorators(self): self.check_ast_roundtrip(class_decorator) diff --git a/Makefile.pre.in b/Makefile.pre.in index 4c2426ed283d43..120a6add38507f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1729,9 +1729,6 @@ FROZEN_FILES_IN = \ Lib/zipimport.py \ Lib/abc.py \ Lib/codecs.py \ - Lib/encodings/__init__.py \ - Lib/encodings/aliases.py \ - Lib/encodings/utf_8.py \ Lib/io.py \ Lib/_collections_abc.py \ Lib/_sitebuiltins.py \ @@ -1741,7 +1738,6 @@ FROZEN_FILES_IN = \ Lib/os.py \ Lib/site.py \ Lib/stat.py \ - Lib/linecache.py \ Lib/importlib/util.py \ Lib/importlib/machinery.py \ Lib/runpy.py \ @@ -1758,9 +1754,6 @@ FROZEN_FILES_OUT = \ Python/frozen_modules/zipimport.h \ Python/frozen_modules/abc.h \ Python/frozen_modules/codecs.h \ - Python/frozen_modules/encodings.h \ - Python/frozen_modules/encodings.aliases.h \ - Python/frozen_modules/encodings.utf_8.h \ Python/frozen_modules/io.h \ Python/frozen_modules/_collections_abc.h \ Python/frozen_modules/_sitebuiltins.h \ @@ -1770,7 +1763,6 @@ FROZEN_FILES_OUT = \ Python/frozen_modules/os.h \ Python/frozen_modules/site.h \ Python/frozen_modules/stat.h \ - Python/frozen_modules/linecache.h \ Python/frozen_modules/importlib.util.h \ Python/frozen_modules/importlib.machinery.h \ Python/frozen_modules/runpy.h \ @@ -1810,15 +1802,6 @@ Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS) Python/frozen_modules/codecs.h: Lib/codecs.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h -Python/frozen_modules/encodings.h: Lib/encodings/__init__.py $(FREEZE_MODULE_DEPS) - $(FREEZE_MODULE) encodings $(srcdir)/Lib/encodings/__init__.py Python/frozen_modules/encodings.h - -Python/frozen_modules/encodings.aliases.h: Lib/encodings/aliases.py $(FREEZE_MODULE_DEPS) - $(FREEZE_MODULE) encodings.aliases $(srcdir)/Lib/encodings/aliases.py Python/frozen_modules/encodings.aliases.h - -Python/frozen_modules/encodings.utf_8.h: Lib/encodings/utf_8.py $(FREEZE_MODULE_DEPS) - $(FREEZE_MODULE) encodings.utf_8 $(srcdir)/Lib/encodings/utf_8.py Python/frozen_modules/encodings.utf_8.h - Python/frozen_modules/io.h: Lib/io.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h @@ -1846,9 +1829,6 @@ Python/frozen_modules/site.h: Lib/site.py $(FREEZE_MODULE_DEPS) Python/frozen_modules/stat.h: Lib/stat.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h -Python/frozen_modules/linecache.h: Lib/linecache.py $(FREEZE_MODULE_DEPS) - $(FREEZE_MODULE) linecache $(srcdir)/Lib/linecache.py Python/frozen_modules/linecache.h - Python/frozen_modules/importlib.util.h: Lib/importlib/util.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-02-26-21-22-34.gh-issue-145278.DHkYqt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-26-21-22-34.gh-issue-145278.DHkYqt.rst deleted file mode 100644 index 6b6a9eb9813663..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-02-26-21-22-34.gh-issue-145278.DHkYqt.rst +++ /dev/null @@ -1,4 +0,0 @@ -The :mod:`encodings` is now partially frozen, including -the ``aliases`` and ``utf_8`` submodules. - -The :mod:`linecache` is now frozen. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-05-16-16-17.gh-issue-143055.qDUFlY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-05-16-16-17.gh-issue-143055.qDUFlY.rst new file mode 100644 index 00000000000000..9b55459bffdea8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-05-16-16-17.gh-issue-143055.qDUFlY.rst @@ -0,0 +1,2 @@ +Fix crash in AST unparser when unparsing dict comprehension unpacking. +Found by OSS Fuzz in :oss-fuzz:`489790200`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-09-18-52-03.gh-issue-145701.79KQyO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-09-18-52-03.gh-issue-145701.79KQyO.rst new file mode 100644 index 00000000000000..23796082fb616f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-09-18-52-03.gh-issue-145701.79KQyO.rst @@ -0,0 +1,3 @@ +Fix :exc:`SystemError` when ``__classdict__`` or +``__conditional_annotations__`` is in a class-scope inlined comprehension. +Found by OSS Fuzz in :oss-fuzz:`491105000`. diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index 14da472c1bb110..f3a22f3f6a87cb 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -517,8 +517,8 @@ static int fuzz_pycompile(const char* data, size_t size) { return 0; } - // Need 2 bytes for parameter selection - if (size < 2) { + // Need 3 bytes for parameter selection + if (size < 3) { return 0; } @@ -530,25 +530,39 @@ static int fuzz_pycompile(const char* data, size_t size) { unsigned char optimize_idx = (unsigned char) data[1]; int optimize = optimize_vals[optimize_idx % NUM_OPTIMIZE_VALS]; + // Use third byte to determine compiler flags to use. + unsigned char flags_byte = (unsigned char) data[2]; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + if (flags_byte & 0x01) { + flags.cf_flags |= PyCF_DONT_IMPLY_DEDENT; + } + if (flags_byte & 0x02) { + flags.cf_flags |= PyCF_ONLY_AST; + } + if (flags_byte & 0x04) { + flags.cf_flags |= PyCF_IGNORE_COOKIE; + } + if (flags_byte & 0x08) { + flags.cf_flags |= PyCF_TYPE_COMMENTS; + } + if (flags_byte & 0x10) { + flags.cf_flags |= PyCF_ALLOW_TOP_LEVEL_AWAIT; + } + if (flags_byte & 0x20) { + flags.cf_flags |= PyCF_ALLOW_INCOMPLETE_INPUT; + } + if (flags_byte & 0x40) { + flags.cf_flags |= PyCF_OPTIMIZED_AST; + } + char pycompile_scratch[MAX_PYCOMPILE_TEST_SIZE]; // Create a NUL-terminated C string from the remaining input - memcpy(pycompile_scratch, data + 2, size - 2); + memcpy(pycompile_scratch, data + 3, size - 3); // Put a NUL terminator just after the copied data. (Space was reserved already.) - pycompile_scratch[size - 2] = '\0'; - - // XXX: instead of always using NULL for the `flags` value to - // `Py_CompileStringExFlags`, there are many flags that conditionally - // change parser behavior: - // - // #define PyCF_TYPE_COMMENTS 0x1000 - // #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 - // #define PyCF_ONLY_AST 0x0400 - // - // It would be good to test various combinations of these, too. - PyCompilerFlags *flags = NULL; - - PyObject *result = Py_CompileStringExFlags(pycompile_scratch, "", start, flags, optimize); + pycompile_scratch[size - 3] = '\0'; + + PyObject *result = Py_CompileStringExFlags(pycompile_scratch, "", start, &flags, optimize); if (result == NULL) { /* Compilation failed, most likely from a syntax error. If it was a SystemError we abort. There's no non-bug reason to raise a diff --git a/Modules/getpath.py b/Modules/getpath.py index 2f4d635a29585c..4dceb5cdc8dfcf 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -776,9 +776,9 @@ def search_up(prefix, *landmarks, test=isfile): # Warn if the standard library is missing, unless pythonpath_was_set was set, as # that skips parts of the stdlib directories calculation — assume the provided # pythonpath is correct. This is how subinterpreters initialize the path for eg. -if not py_setpath and not pythonpath_was_set and (not stdlib_zip or not isfile(stdlib_zip)): +if not py_setpath and not pythonpath_was_set: home_hint = f"The Python 'home' directory was set to {home!r}, is this correct?" - if not stdlib_dir or not isdir(stdlib_dir): + if (not stdlib_zip or not isfile(stdlib_zip)) and (not stdlib_dir or not isdir(stdlib_dir)): hint = home_hint if home else f'sys.prefix is set to {prefix}, is this correct?' warn('WARN: Could not find the standard library directory! ' + hint) elif not platstdlib_dir or not isdir(platstdlib_dir): diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 27ec8bb40a929f..81abb0990eebe9 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -647,7 +647,6 @@ clear_tp_bases(PyTypeObject *self, int final) static inline PyObject * lookup_tp_mro(PyTypeObject *self) { - ASSERT_NEW_TYPE_OR_LOCKED(self); return self->tp_mro; } @@ -664,8 +663,19 @@ set_tp_mro(PyTypeObject *self, PyObject *mro, int initial) /* Other checks are done via set_tp_bases. */ _Py_SetImmortal(mro); } + else { + PyUnstable_Object_EnableDeferredRefcount(mro); + } + } + if (!initial) { + type_lock_prevent_release(); + types_stop_world(); } self->tp_mro = mro; + if (!initial) { + types_start_world(); + type_lock_allow_release(); + } } static inline void @@ -1728,18 +1738,11 @@ static PyObject * type_get_mro(PyObject *tp, void *Py_UNUSED(closure)) { PyTypeObject *type = PyTypeObject_CAST(tp); - PyObject *mro; - - BEGIN_TYPE_LOCK(); - mro = lookup_tp_mro(type); + PyObject *mro = lookup_tp_mro(type); if (mro == NULL) { - mro = Py_None; - } else { - Py_INCREF(mro); + Py_RETURN_NONE; } - - END_TYPE_LOCK(); - return mro; + return Py_NewRef(mro); } static PyTypeObject *find_best_base(PyObject *); diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 40946cafd16b75..cb806459596084 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -310,21 +310,6 @@ $(IntDir)codecs.g.h $(GeneratedFrozenModulesDir)Python\frozen_modules\codecs.h - - encodings - $(IntDir)encodings.g.h - $(GeneratedFrozenModulesDir)Python\frozen_modules\encodings.h - - - encodings.aliases - $(IntDir)encodings.aliases.g.h - $(GeneratedFrozenModulesDir)Python\frozen_modules\encodings.aliases.h - - - encodings.utf_8 - $(IntDir)encodings.utf_8.g.h - $(GeneratedFrozenModulesDir)Python\frozen_modules\encodings.utf_8.h - io $(IntDir)io.g.h @@ -370,11 +355,6 @@ $(IntDir)stat.g.h $(GeneratedFrozenModulesDir)Python\frozen_modules\stat.h - - linecache - $(IntDir)linecache.g.h - $(GeneratedFrozenModulesDir)Python\frozen_modules\linecache.h - importlib.util $(IntDir)importlib.util.g.h diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index a0e36a4c526ede..6dcf0e8712903a 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -537,15 +537,6 @@ Python Files - - Python Files - - - Python Files - - - Python Files - Python Files @@ -573,9 +564,6 @@ Python Files - - Python Files - Python Files diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index c25699978cf651..6050c351cff68f 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -464,9 +464,15 @@ static int append_ast_dictcomp(PyUnicodeWriter *writer, expr_ty e) { APPEND_CHAR('{'); - APPEND_EXPR(e->v.DictComp.key, PR_TEST); - APPEND_STR(": "); - APPEND_EXPR(e->v.DictComp.value, PR_TEST); + if (e->v.DictComp.value) { + APPEND_EXPR(e->v.DictComp.key, PR_TEST); + APPEND_STR(": "); + APPEND_EXPR(e->v.DictComp.value, PR_TEST); + } + else { + APPEND_STR("**"); + APPEND_EXPR(e->v.DictComp.key, PR_TEST); + } APPEND(comprehensions, e->v.DictComp.generators); APPEND_CHAR_FINISH('}'); } diff --git a/Python/frozen.c b/Python/frozen.c index 72a291d0d43f1d..15d256b6743e0a 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -46,9 +46,6 @@ #include "frozen_modules/zipimport.h" #include "frozen_modules/abc.h" #include "frozen_modules/codecs.h" -#include "frozen_modules/encodings.h" -#include "frozen_modules/encodings.aliases.h" -#include "frozen_modules/encodings.utf_8.h" #include "frozen_modules/io.h" #include "frozen_modules/_collections_abc.h" #include "frozen_modules/_sitebuiltins.h" @@ -58,7 +55,6 @@ #include "frozen_modules/os.h" #include "frozen_modules/site.h" #include "frozen_modules/stat.h" -#include "frozen_modules/linecache.h" #include "frozen_modules/importlib.util.h" #include "frozen_modules/importlib.machinery.h" #include "frozen_modules/runpy.h" @@ -80,9 +76,6 @@ static const struct _frozen stdlib_modules[] = { /* stdlib - startup, without site (python -S) */ {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), false}, {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), false}, - {"encodings", _Py_M__encodings, (int)sizeof(_Py_M__encodings), true}, - {"encodings.aliases", _Py_M__encodings_aliases, (int)sizeof(_Py_M__encodings_aliases), false}, - {"encodings.utf_8", _Py_M__encodings_utf_8, (int)sizeof(_Py_M__encodings_utf_8), false}, {"io", _Py_M__io, (int)sizeof(_Py_M__io), false}, /* stdlib - startup, with site */ @@ -95,9 +88,6 @@ static const struct _frozen stdlib_modules[] = { {"site", _Py_M__site, (int)sizeof(_Py_M__site), false}, {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), false}, - /* pythonrun - interactive */ - {"linecache", _Py_M__linecache, (int)sizeof(_Py_M__linecache), false}, - /* runpy - run module with -m */ {"importlib.util", _Py_M__importlib_util, (int)sizeof(_Py_M__importlib_util), false}, {"importlib.machinery", _Py_M__importlib_machinery, (int)sizeof(_Py_M__importlib_machinery), false}, diff --git a/Python/symtable.c b/Python/symtable.c index beb6df88d097e3..4b695e4b2588d8 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -807,6 +807,8 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, PyObject *k, *v; Py_ssize_t pos = 0; int remove_dunder_class = 0; + int remove_dunder_classdict = 0; + int remove_dunder_cond_annotations = 0; while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { // skip comprehension parameter @@ -829,15 +831,27 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (existing == NULL && PyErr_Occurred()) { return 0; } - // __class__ is never allowed to be free through a class scope (see + // __class__, __classdict__ and __conditional_annotations__ are + // never allowed to be free through a class scope (see // drop_class_free) if (scope == FREE && ste->ste_type == ClassBlock && - _PyUnicode_EqualToASCIIString(k, "__class__")) { + (_PyUnicode_EqualToASCIIString(k, "__class__") || + _PyUnicode_EqualToASCIIString(k, "__classdict__") || + _PyUnicode_EqualToASCIIString(k, "__conditional_annotations__"))) { scope = GLOBAL_IMPLICIT; if (PySet_Discard(comp_free, k) < 0) { return 0; } - remove_dunder_class = 1; + + if (_PyUnicode_EqualToASCIIString(k, "__class__")) { + remove_dunder_class = 1; + } + else if (_PyUnicode_EqualToASCIIString(k, "__conditional_annotations__")) { + remove_dunder_cond_annotations = 1; + } + else { + remove_dunder_classdict = 1; + } } if (!existing) { // name does not exist in scope, copy from comprehension @@ -877,6 +891,12 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) { return 0; } + if (remove_dunder_classdict && PyDict_DelItemString(comp->ste_symbols, "__classdict__") < 0) { + return 0; + } + if (remove_dunder_cond_annotations && PyDict_DelItemString(comp->ste_symbols, "__conditional_annotations__") < 0) { + return 0; + } return 1; } diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index 0d1f5968d2a9e9..3c43f7e3bbe8ca 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -50,9 +50,10 @@ ('stdlib - startup, without site (python -S)', [ 'abc', 'codecs', - '', - 'encodings.aliases', - 'encodings.utf_8', + # For now we do not freeze the encodings, due # to the noise all + # those extra modules add to the text printed during the build. + # (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.) + #'', 'io', ]), ('stdlib - startup, with site', [ @@ -65,9 +66,6 @@ 'site', 'stat', ]), - ('pythonrun - interactive', [ - 'linecache', - ]), ('runpy - run module with -m', [ "importlib.util", "importlib.machinery",