Merge branch 'main' into gh-135736-taskgroup-generatorexit

Author: sreehariannam

Doc/library/difflib.rst

@@ -40,13 +40,18 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module.
    complicated way on how many elements the sequences have in common; best case
    time is linear.
 
-   **Automatic junk heuristic:** :class:`SequenceMatcher` supports a heuristic that
-   automatically treats certain sequence items as junk. The heuristic counts how many
-   times each individual item appears in the sequence. If an item's duplicates (after
-   the first one) account for more than 1% of the sequence and the sequence is at least
-   200 items long, this item is marked as "popular" and is treated as junk for
-   the purpose of sequence matching. This heuristic can be turned off by setting
-   the ``autojunk`` argument to ``False`` when creating the :class:`SequenceMatcher`.
+   **Junk**: :class:`SequenceMatcher` accepts an ``isjunk`` predicate and an
+   ``autojunk`` flag. Items that are considered as junk will not be considered
+   to find similar content blocks. This can produce better results for humans
+   (typically breaking on whitespace) and faster (because it reduces the number
+   of possible combinations). But it can also cause pathological cases where
+   too many items considered junk cause an unexpectedly large (but correct)
+   diff result.
+   You should consider tuning them or turning them off depending on your data.
+   Moreover, only the second sequence is inspected for junk. This causes the diff
+   output to not be symmetrical.
+   When ``autojunk=True``, it will consider as junk the items that account for more
+   than 1% of the sequence, if it is at least 200 items long.
 
    .. versionchanged:: 3.2
       Added the *autojunk* parameter.
@@ -558,16 +563,6 @@ The :class:`SequenceMatcher` class has this constructor:
       to try :meth:`quick_ratio` or :meth:`real_quick_ratio` first to get an
       upper bound.
 
-      .. note::
-
-         Caution: The result of a :meth:`ratio` call may depend on the order of
-         the arguments. For instance::
-
-            >>> SequenceMatcher(None, 'tide', 'diet').ratio()
-            0.25
-            >>> SequenceMatcher(None, 'diet', 'tide').ratio()
-            0.5
-
 
    .. method:: quick_ratio()
 

Doc/library/logging.handlers.rst

@@ -631,7 +631,8 @@ supports sending logging messages to a remote or local Unix syslog.
    the form of a ``(host, port)`` tuple.  If *address* is not specified,
    ``('localhost', 514)`` is used.  The address is used to open a socket.  An
    alternative to providing a ``(host, port)`` tuple is providing an address as a
-   string, for example '/dev/log'. In this case, a Unix domain socket is used to
+   string or a :class:`bytes` object, for example '/dev/log'.
+   In this case, a Unix domain socket is used to
    send the message to the syslog. If *facility* is not specified,
    :const:`LOG_USER` is used. The type of socket opened depends on the
    *socktype* argument, which defaults to :const:`socket.SOCK_DGRAM` and thus
@@ -664,6 +665,9 @@ supports sending logging messages to a remote or local Unix syslog.
    .. versionchanged:: 3.14
       *timeout* was added.
 
+   .. versionchanged:: next
+      *address* can now be a :class:`bytes` object.
+
    .. method:: close()
 
       Closes the socket to the remote host.

Lib/argparse.py

@@ -87,12 +87,17 @@
 
 
 import os as _os
-import re as _re
 import sys as _sys
-from gettext import gettext as _
-from gettext import ngettext
 
 lazy import _colorize
+lazy import copy
+lazy import difflib
+lazy import re as _re
+lazy import shutil
+lazy import textwrap
+lazy import warnings
+lazy from gettext import gettext as _
+lazy from gettext import ngettext
 
 SUPPRESS = '==SUPPRESS=='
 
@@ -143,10 +148,8 @@ def _copy_items(items):
         return []
     # The copy module is used only in the 'append' and 'append_const'
     # actions, and it is needed only when the default value isn't a list.
-    # Delay its import for speeding up the common case.
     if type(items) is list:
         return items[:]
-    import copy
     return copy.copy(items)
 
 
@@ -186,7 +189,6 @@ def __init__(
     ):
         # default setting for width
         if width is None:
-            import shutil
             width = shutil.get_terminal_size().columns
             width -= 2
 
@@ -773,14 +775,38 @@ def _iter_indented_subactions(self, action):
 
     def _split_lines(self, text, width):
         text = self._whitespace_matcher.sub(' ', text).strip()
-        # The textwrap module is used only for formatting help.
-        # Delay its import for speeding up the common usage of argparse.
-        import textwrap
-        return textwrap.wrap(text, width)
+        decolored = self._decolor(text)
+        if decolored == text:
+            return textwrap.wrap(text, width)
+
+        # gh-142035: colors inflate textwrap's length counts, so wrap
+        # the decolored text and re-apply colors per word; if textwrap
+        # split a word, keep the plain lines (colors can't be mapped).
+        plain = self._whitespace_matcher.sub(' ', decolored).strip()
+        if not plain:
+            # nothing visible to wrap (e.g. an empty interpolated value)
+            return [text]
+        plain_lines = textwrap.wrap(plain, width)
+        plain_words = plain.split()
+        colored_words = text.split()
+        # Drop escape-only tokens (e.g. an empty interpolated value).
+        if len(colored_words) != len(plain_words):
+            colored_words = [
+                word for word in colored_words if self._decolor(word)
+            ]
+        colored_lines = []
+        start = 0
+        for plain_line in plain_lines:
+            plain_line_words = plain_line.split()
+            end = start + len(plain_line_words)
+            if plain_words[start:end] != plain_line_words:
+                return plain_lines
+            colored_lines.append(' '.join(colored_words[start:end]))
+            start = end
+        return colored_lines
 
     def _fill_text(self, text, width, indent):
         text = self._whitespace_matcher.sub(' ', text).strip()
-        import textwrap
         return textwrap.fill(text, width,
                              initial_indent=indent,
                              subsequent_indent=indent)
@@ -1458,7 +1484,6 @@ class FileType(object):
     """
 
     def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None):
-        import warnings
         warnings.warn(
             "FileType is deprecated. Simply open files after parsing arguments.",
             category=PendingDeprecationWarning,
@@ -1865,7 +1890,6 @@ class _ArgumentGroup(_ActionsContainer):
 
     def __init__(self, container, title=None, description=None, **kwargs):
         if 'prefix_chars' in kwargs:
-            import warnings
             depr_msg = (
                 "The use of the undocumented 'prefix_chars' parameter in "
                 "ArgumentParser.add_argument_group() is deprecated."
@@ -2796,7 +2820,6 @@ def _check_value(self, action, value):
 
             if self.suggest_on_error and isinstance(value, str):
                 if all(isinstance(choice, str) for choice in action.choices):
-                    import difflib
                     suggestions = difflib.get_close_matches(value, action.choices, 1)
                     if suggestions:
                         args['closest'] = suggestions[0]
@@ -2936,8 +2959,6 @@ def _warning(self, message):
 
 def __getattr__(name):
     if name == "__version__":
-        from warnings import _deprecated
-
-        _deprecated("__version__", remove=(3, 20))
+        warnings._deprecated("__version__", remove=(3, 20))
         return "1.1"  # Do not change
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

Lib/logging/handlers.py

@@ -886,8 +886,9 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT),
         """
         Initialize a handler.
 
-        If address is specified as a string, a UNIX socket is used. To log to a
-        local syslogd, "SysLogHandler(address="/dev/log")" can be used.
+        If address is specified as a string or bytes, a UNIX socket is used.
+        To log to a local syslogd, "SysLogHandler(address="/dev/log")" can be
+        used.
         If facility is not specified, LOG_USER is used. If socktype is
         specified as socket.SOCK_DGRAM or socket.SOCK_STREAM, that specific
         socket type will be used. For Unix sockets, you can also specify a
@@ -938,7 +939,7 @@ def createSocket(self):
         address = self.address
         socktype = self.socktype
 
-        if isinstance(address, str):
+        if not isinstance(address, (list, tuple)):
             self.unixsocket = True
             # Syslog server may be unavailable during handler initialisation.
             # C's openlog() function also ignores connection errors.

Lib/test/test_argparse.py

@@ -3,7 +3,6 @@
 import _colorize
 import contextlib
 import functools
-import inspect
 import io
 import operator
 import os
@@ -12,6 +11,7 @@
 import sys
 import textwrap
 import tempfile
+import types
 import unittest
 import argparse
 import warnings
@@ -85,6 +85,8 @@ class TestLazyImports(unittest.TestCase):
         "_colorize",
         "copy",
         "difflib",
+        "gettext",
+        "re",
         "shutil",
         "textwrap",
         "warnings",
@@ -99,7 +101,7 @@ def test_create_parser(self):
         # Test imports are still unused after
         # creating a parser
         create_parser = "argparse.ArgumentParser()"
-        imported_modules = {"shutil"}
+        imported_modules = {"gettext", "re", "shutil"}
 
         import_helper.ensure_lazy_imports(
             "argparse",
@@ -114,7 +116,7 @@ def test_add_subparser(self):
             parser.add_subparsers(dest='command', required=False)
             """
         )
-        imported_modules = {"shutil"}
+        imported_modules = {"gettext", "re", "shutil"}
 
         import_helper.ensure_lazy_imports(
             "argparse",
@@ -132,7 +134,7 @@ def test_parse_args(self):
             parser.parse_args(['BAR', '--foo', 'FOO'])
             """
         )
-        imported_modules = {"shutil"}
+        imported_modules = {"gettext", "re", "shutil"}
         import_helper.ensure_lazy_imports(
             "argparse",
             self.LAZY_IMPORTS - imported_modules,
@@ -7098,7 +7100,10 @@ def test_all_exports_everything_but_modules(self):
             name
             for name, value in vars(argparse).items()
             if not (name.startswith("_") or name == 'ngettext')
-            if not inspect.ismodule(value)
+            if not isinstance(
+                value,
+                (types.ModuleType, types.LazyImportType),
+            )
         ]
         self.assertEqual(sorted(items), sorted(argparse.__all__))
 
@@ -7580,6 +7585,62 @@ def test_argparse_color_custom_usage(self):
             ),
         )
 
+    def test_argparse_color_wrapping_matches_uncolored(self):
+        # gh-142035: color codes must not affect where help text wraps.
+        # Stripping the escapes from colored help must yield exactly the
+        # same text as the uncolored help across representative widths.
+        def build(color, path="output.txt"):
+            parser = argparse.ArgumentParser(prog="PROG", color=color)
+            parser.add_argument(
+                "--mode",
+                default="auto",
+                choices=("auto", "fast", "slow"),
+                help="select the operating mode from the available choices "
+                     "%(choices)s and note the default is %(default)s here",
+            )
+            parser.add_argument(
+                "--path",
+                default=path,
+                help="write output to %(default)s and continue processing",
+            )
+            return parser
+
+        env = self.enterContext(os_helper.EnvironmentVarGuard())
+        paths = (
+            "output.txt",
+            "/var/lib/application/cache/unusually_long_generated_filename",
+            "production-read-only-replica",
+        )
+        for path in paths:
+            for columns in ("80", "60", "45", "30", "20"):
+                with self.subTest(path=path, columns=columns):
+                    env["COLUMNS"] = columns
+                    colored = build(color=True, path=path).format_help()
+                    plain = build(color=False, path=path).format_help()
+                    self.assertIn(
+                        f"{self.theme.interpolated_value}auto"
+                        f"{self.theme.reset}",
+                        colored,
+                    )
+                    self.assertEqual(_colorize.decolor(colored), plain)
+
+    def test_argparse_color_preserved_when_wrapping_between_words(self):
+        parser = argparse.ArgumentParser(prog="PROG", color=True)
+        parser.add_argument(
+            "--mode", default="auto",
+            help="select the %(default)s operating mode from the available "
+                 "options and continue with several more words",
+        )
+
+        env = self.enterContext(os_helper.EnvironmentVarGuard())
+        env["COLUMNS"] = "40"
+        help_text = parser.format_help()
+
+        self.assertIn(
+            f"{self.theme.interpolated_value}auto{self.theme.reset}",
+            help_text,
+        )
+
     def test_custom_formatter_function(self):
         def custom_formatter(prog):
             return argparse.RawTextHelpFormatter(prog, indent_increment=5)

Lib/test/test_functools.py

@@ -579,6 +579,40 @@ def f(**kwargs):
         with self.assertRaises(RuntimeError):
             result = p(**{BadStr("poison"): "new_value"})
 
+    def test_call_safety_against_reentrant_mutation(self):
+        def old_function(*args, **kwargs):
+            return "old_function", args, kwargs
+
+        def new_function(*args, **kwargs):
+            return "new_function", args, kwargs
+
+        g_partial = None
+
+        class EvilKey(str):
+            armed = False
+            def __hash__(self):
+                if EvilKey.armed and g_partial is not None:
+                    EvilKey.armed = False
+                    new_args_tuple = ("new_arg",)
+                    new_keywords_dict = {"new_keyword": None}
+                    new_tuple_state = (new_function, new_args_tuple, new_keywords_dict, None)
+                    g_partial.__setstate__(new_tuple_state)
+                    gc.collect()
+                return str.__hash__(self)
+
+        g_partial = functools.partial(old_function, "old_arg", old_keyword=None)
+
+        kwargs = {EvilKey("evil_key"): None}
+        EvilKey.armed = True
+
+        result = g_partial(**kwargs)
+        expected = ("old_function", ("old_arg",), {"old_keyword": None, "evil_key": None})
+        self.assertEqual(result, expected)
+
+        result = g_partial()
+        expected = ("new_function", ("new_arg",), {"new_keyword": None})
+        self.assertEqual(result, expected)
+
 @unittest.skipUnless(c_functools, 'requires the C _functools module')
 class TestPartialC(TestPartial, unittest.TestCase):
     if c_functools: