Merge branch 'main' into gh-150942-csv-appendtakeref

Author: omkar-334

Doc/library/collections.rst

@@ -1233,7 +1233,7 @@ variants of :deco:`functools.lru_cache`:
 .. testcode::
 
     from collections import OrderedDict
-    from time import time
+    from time import monotonic
 
     class TimeBoundedLRU:
         "LRU Cache that invalidates and refreshes old entries."
@@ -1248,10 +1248,10 @@ variants of :deco:`functools.lru_cache`:
             if args in self.cache:
                 self.cache.move_to_end(args)
                 timestamp, result = self.cache[args]
-                if time() - timestamp <= self.maxage:
+                if monotonic() - timestamp <= self.maxage:
                     return result
             result = self.func(*args)
-            self.cache[args] = time(), result
+            self.cache[args] = monotonic(), result
             if len(self.cache) > self.maxsize:
                 self.cache.popitem(last=False)
             return result

Doc/library/difflib.rst

@@ -724,7 +724,7 @@ Finally, we compare the two:
 
    >>> result = list(d.compare(text1, text2))
 
-``result`` is a list of strings, so let's pretty-print it:
+``result`` is a list of strings, so let's pretty-print it::
 
    >>> from pprint import pprint
    >>> pprint(result)
@@ -739,7 +739,7 @@ Finally, we compare the two:
     '?           ++++ ^                      ^\n',
     '+   5. Flat is better than nested.\n']
 
-As a single multi-line string it looks like this:
+As a single multi-line string it looks like this::
 
    >>> import sys
    >>> sys.stdout.writelines(result)

Doc/library/io.rst

@@ -38,6 +38,7 @@ will raise a :exc:`TypeError`.  So will giving a :class:`bytes` object to the
    Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since
    :exc:`IOError` is now an alias of :exc:`OSError`.
 
+.. _text-io:
 
 Text I/O
 ^^^^^^^^
@@ -65,6 +66,7 @@ In-memory text streams are also available as :class:`StringIO` objects::
 The text stream API is described in detail in the documentation of
 :class:`TextIOBase`.
 
+.. _binary-io:
 
 Binary I/O
 ^^^^^^^^^^
@@ -103,6 +105,13 @@ stream by opening a file in binary mode with buffering disabled::
 
 The raw stream API is described in detail in the docs of :class:`RawIOBase`.
 
+.. warning::
+   Raw I/O is a low-level interface and methods generally must have their return
+   values checked and be explicitly retried to ensure an operation completes.
+   For instance :meth:`~RawIOBase.write` returns the number of bytes written
+   which may be less than the number of bytes provided (a partial write).
+   High-level I/O objects like :ref:`binary-io` and :ref:`text-io` implement
+   retry behavior.
 
 .. _io-text-encoding:
 
@@ -478,8 +487,11 @@ I/O Base Classes
 
       Read up to *size* bytes from the object and return them.  As a convenience,
       if *size* is unspecified or -1, all bytes until EOF are returned.
-      Otherwise, only one system call is ever made.  Fewer than *size* bytes may
-      be returned if the operating system call returns fewer than *size* bytes.
+
+      Attempts to make only one system call but will retry if interrupted and
+      the signal handler does not raise an exception (see :pep:`475` for the
+      rationale). This means fewer than *size* bytes may be returned if the
+      operating system call returns fewer than *size* bytes.
 
       If 0 bytes are returned, and *size* was not 0, this indicates end of file.
       If the object is in non-blocking mode and no bytes are available,
@@ -493,13 +505,19 @@ I/O Base Classes
       Read and return all the bytes from the stream until EOF, using multiple
       calls to the stream if necessary.
 
+      If ``0`` bytes are returned this indicates end of file. If the object is in
+      non-blocking mode and the underlying :meth:`read` returns ``None``
+      indicating no bytes are available, ``None`` is returned.
+
    .. method:: readinto(b, /)
 
       Read bytes into a pre-allocated, writable
       :term:`bytes-like object` *b*, and return the
       number of bytes read.  For example, *b* might be a :class:`bytearray`.
-      If the object is in non-blocking mode and no bytes
-      are available, ``None`` is returned.
+
+      If ``0`` is returned and ``len(b)`` is not ``0``, this indicates end of file. If
+      the object is in non-blocking mode and no bytes are available, ``None`` is
+      returned.
 
    .. method:: write(b, /)
 
@@ -513,6 +531,13 @@ I/O Base Classes
       this method returns, so the implementation should only access *b*
       during the method call.
 
+      .. warning::
+
+         This function does not ensure all bytes are written or an exception is
+         thrown. Callers may implement that behavior by checking the return
+         value and, if it is less than the length of *b*, looping with additional
+         write calls until all unwritten bytes are written. High-level I/O
+         objects like :ref:`binary-io` and :ref:`text-io` implement retry behavior.
 
 .. class:: BufferedIOBase
 
@@ -641,7 +666,11 @@ Raw File I/O
 .. class:: FileIO(name, mode='r', closefd=True, opener=None)
 
    A raw binary stream representing an OS-level file containing bytes data.  It
-   inherits from :class:`RawIOBase`.
+   inherits from :class:`RawIOBase` and implements its low-level access design.
+   This means :meth:`~RawIOBase.write` does not guarantee all bytes are written
+   and :meth:`~RawIOBase.read` may read less bytes than requested even when more
+   bytes may be present in the underlying file. To get "write all" and
+   "read at least" behavior, use :ref:`binary-io`.
 
    The *name* can be one of two things:
 
@@ -661,10 +690,6 @@ Raw File I/O
    implies writing, so this mode behaves in a similar way to ``'w'``. Add a
    ``'+'`` to the mode to allow simultaneous reading and writing.
 
-   The :meth:`~RawIOBase.read` (when called with a positive argument),
-   :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this
-   class will only make one system call.
-
    A custom opener can be used by passing a callable as *opener*. The underlying
    file descriptor for the file object is then obtained by calling *opener* with
    (*name*, *flags*). *opener* must return an open file descriptor (passing
@@ -676,6 +701,13 @@ Raw File I/O
    See the :func:`open` built-in function for examples on using the *opener*
    parameter.
 
+   .. warning::
+      :class:`FileIO` is a low-level I/O object and members, such as
+      :meth:`~RawIOBase.read` and :meth:`~RawIOBase.write`, need to have their
+      return values checked explicitly in a retry loop to implement "write all"
+      and "read at least" behavior. High-level I/O objects :ref:`binary-io` and
+      :ref:`text-io` implement retry behavior.
+
    .. versionchanged:: 3.3
       The *opener* parameter was added.
       The ``'x'`` mode was added.

Include/dynamic_annotations.h

@@ -461,6 +461,7 @@ int RunningOnValgrind(void);
 
 #if DYNAMIC_ANNOTATIONS_ENABLED != 0 && defined(__cplusplus)
 
+extern "C++" {
   /* _Py_ANNOTATE_UNPROTECTED_READ is the preferred way to annotate racey reads.
 
      Instead of doing
@@ -476,6 +477,8 @@ int RunningOnValgrind(void);
     _Py_ANNOTATE_IGNORE_READS_END();
     return res;
   }
+}
+
   /* Apply _Py_ANNOTATE_BENIGN_RACE_SIZED to a static variable. */
 #define _Py_ANNOTATE_BENIGN_RACE_STATIC(static_var, description)        \
     namespace {                                                       \

Lib/email/charset.py

@@ -9,6 +9,7 @@
     'add_codec',
     ]
 
+import codecs
 from functools import partial
 
 import email.base64mime
@@ -58,37 +59,71 @@
     'shift_jis':   (BASE64,    None,    'iso-2022-jp'),
     'iso-2022-jp': (BASE64,    None,    None),
     'koi8-r':      (BASE64,    BASE64,  None),
-    'utf-8':       (SHORTEST,  BASE64, 'utf-8'),
     }
 
-# Aliases for other commonly-used names for character sets.  Map
-# them to the real ones used in email.
+# Map Python codec names to their corresponding MIME/IANA names.
 ALIASES = {
-    'latin_1': 'iso-8859-1',
-    'latin-1': 'iso-8859-1',
-    'latin_2': 'iso-8859-2',
-    'latin-2': 'iso-8859-2',
-    'latin_3': 'iso-8859-3',
-    'latin-3': 'iso-8859-3',
-    'latin_4': 'iso-8859-4',
-    'latin-4': 'iso-8859-4',
-    'latin_5': 'iso-8859-9',
-    'latin-5': 'iso-8859-9',
-    'latin_6': 'iso-8859-10',
-    'latin-6': 'iso-8859-10',
-    'latin_7': 'iso-8859-13',
-    'latin-7': 'iso-8859-13',
-    'latin_8': 'iso-8859-14',
-    'latin-8': 'iso-8859-14',
-    'latin_9': 'iso-8859-15',
-    'latin-9': 'iso-8859-15',
-    'latin_10':'iso-8859-16',
-    'latin-10':'iso-8859-16',
-    'cp949':   'ks_c_5601-1987',
-    'euc_jp':  'euc-jp',
-    'euc_kr':  'euc-kr',
-    'ascii':   'us-ascii',
-    }
+    'ascii': 'us-ascii',
+    'big5hkscs': 'big5-hkscs',
+    'cp037': 'ibm037',
+    'cp1026': 'ibm1026',
+    'cp1140': 'ibm01140',
+    'cp1250': 'windows-1250',
+    'cp1251': 'windows-1251',
+    'cp1252': 'windows-1252',
+    'cp1253': 'windows-1253',
+    'cp1254': 'windows-1254',
+    'cp1255': 'windows-1255',
+    'cp1256': 'windows-1256',
+    'cp1257': 'windows-1257',
+    'cp1258': 'windows-1258',
+    'cp273': 'ibm273',
+    'cp424': 'ibm424',
+    'cp437': 'ibm437',
+    'cp500': 'ibm500',
+    'cp775': 'ibm775',
+    'cp850': 'ibm850',
+    'cp852': 'ibm852',
+    'cp855': 'ibm855',
+    'cp857': 'ibm857',
+    'cp858': 'ibm00858',
+    'cp860': 'ibm860',
+    'cp861': 'ibm861',
+    'cp862': 'ibm862',
+    'cp863': 'ibm863',
+    'cp864': 'ibm864',
+    'cp865': 'ibm865',
+    'cp866': 'ibm866',
+    'cp869': 'ibm869',
+    'cp874': 'windows-874',
+    'euc_jp': 'euc-jp',
+    'euc_kr': 'euc-kr',
+    'hz': 'hz-gb-2312',
+    'iso2022_jp': 'iso-2022-jp',
+    'iso2022_jp_2': 'iso-2022-jp-2',
+    'iso2022_kr': 'iso-2022-kr',
+    'iso8859-1': 'iso-8859-1',
+    'iso8859-10': 'iso-8859-10',
+    'iso8859-11': 'iso-8859-11',
+    'iso8859-13': 'iso-8859-13',
+    'iso8859-14': 'iso-8859-14',
+    'iso8859-15': 'iso-8859-15',
+    'iso8859-16': 'iso-8859-16',
+    'iso8859-2': 'iso-8859-2',
+    'iso8859-3': 'iso-8859-3',
+    'iso8859-4': 'iso-8859-4',
+    'iso8859-5': 'iso-8859-5',
+    'iso8859-6': 'iso-8859-6',
+    'iso8859-7': 'iso-8859-7',
+    'iso8859-8': 'iso-8859-8-i',
+    'iso8859-9': 'iso-8859-9',
+    'kz1048': 'kz-1048',
+    'mac-roman': 'macintosh',
+
+    # CP949 is not registered in IANA. KS_C_5601-1987 is not the same,
+    # but the closest registered option.
+    'cp949': 'ks_c_5601-1987',
+}
 
 
 # Map charsets to their Unicode codec strings.
@@ -215,7 +250,18 @@ def __init__(self, input_charset=DEFAULT_CHARSET):
             raise errors.CharsetError(input_charset)
         input_charset = input_charset.lower()
         # Set the input charset after filtering through the aliases
-        self.input_charset = ALIASES.get(input_charset, input_charset)
+        # For backward compatibility, try ALIASES first to let the user
+        # override it.
+        if input_charset in ALIASES:
+            input_charset = ALIASES[input_charset]
+        else:
+            try:
+                input_codec = codecs.lookup(input_charset).name
+            except LookupError:
+                pass
+            else:
+                input_charset = ALIASES.get(input_codec, input_codec)
+        self.input_charset = input_charset
         # We can try to guess which encoding and conversion to use by the
         # charset_map dictionary.  Try that first, but let the user override
         # it.

Lib/email/contentmanager.py

@@ -173,11 +173,11 @@ def set_text_content(msg, string, subtype="plain", charset='utf-8', cte=None,
                      disposition=None, filename=None, cid=None,
                      params=None, headers=None):
     _prepare_set(msg, 'text', subtype, headers)
+
+    charset = email.charset.Charset(charset).input_charset
     cte, payload = _encode_text(string, charset, cte, msg.policy)
     msg.set_payload(payload)
-    msg.set_param('charset',
-                  email.charset.ALIASES.get(charset, charset),
-                  replace=True)
+    msg.set_param('charset', charset, replace=True)
     msg['Content-Transfer-Encoding'] = cte
     _finalize_set(msg, disposition, filename, cid, params)
 raw_data_manager.add_set_handler(str, set_text_content)