[3.15] gh-88336: Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList (GH-26328) (#161116)

* bpo-44170: Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList
* Round up the allocated size instead of rejecting the exact fit

Allocating one extra alignment block for a value whose encoded size is
a multiple of the alignment made a copy of the list wider than the
original.  Round the size up instead, so that it is the same for the
same value, and keep allowing a value which fills the slot exactly:
rejecting it broke setting an 8-byte string in a slot of an integer.

---------



(cherry picked from commit 9348e13)

Co-authored-by: Jun <junnplus@gmail.com>

Author: gpshead

Lib/multiprocessing/shared_memory.py

@@ -305,14 +305,21 @@ def _extract_recreation_code(value):
         else:
             return 3  # NoneType
 
+    @staticmethod
+    def _encode_value(value):
+        if not isinstance(value, str):
+            return value
+        else:
+            return value.encode(_encoding)
+
     def __init__(self, sequence=None, *, name=None):
         if name is None or sequence is not None:
             sequence = sequence or ()
             _formats = [
                 self._types_mapping[type(item)]
                     if not isinstance(item, (str, bytes))
                     else self._types_mapping[type(item)] % (
-                        self._alignment * (len(item) // self._alignment + 1),
+                        self._alignment * ((len(self._encode_value(item)) - 1) // self._alignment + 1),
                     )
                 for item in sequence
             ]
@@ -353,7 +360,7 @@ def __init__(self, sequence=None, *, name=None):
                 "".join(_formats),
                 self.shm.buf,
                 self._offset_data_start,
-                *(v.encode(_enc) if isinstance(v, str) else v for v in sequence)
+                *(self._encode_value(v) for v in sequence)
             )
             struct.pack_into(
                 self._format_packing_metainfo,
@@ -463,8 +470,7 @@ def __setitem__(self, position, value):
         else:
             allocated_length = self._allocated_offsets[position + 1] - item_offset
 
-            encoded_value = (value.encode(_encoding)
-                             if isinstance(value, str) else value)
+            encoded_value = self._encode_value(value)
             if len(encoded_value) > allocated_length:
                 raise ValueError("bytes/str item exceeds available storage")
             if current_format[-1] == "s":

Lib/test/_test_multiprocessing.py

@@ -4810,7 +4810,7 @@ def test_shared_memory_SharedMemoryManager_basics(self):
 
     def test_shared_memory_ShareableList_basics(self):
         sl = shared_memory.ShareableList(
-            ['howdy', b'HoWdY', -273.154, 100, None, True, 42]
+            ['howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥']
         )
         self.addCleanup(sl.shm.unlink)
 
@@ -4820,22 +4820,22 @@ def test_shared_memory_ShareableList_basics(self):
 
         # Index Out of Range (get)
         with self.assertRaises(IndexError):
-            sl[7]
+            sl[8]
 
         # Index Out of Range (set)
         with self.assertRaises(IndexError):
-            sl[7] = 2
+            sl[8] = 2
 
         # Assign value without format change (str -> str)
         current_format = sl._get_packing_format(0)
         sl[0] = 'howdy'
         self.assertEqual(current_format, sl._get_packing_format(0))
 
         # Verify attributes are readable.
-        self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q')
+        self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q16s')
 
         # Exercise len().
-        self.assertEqual(len(sl), 7)
+        self.assertEqual(len(sl), 8)
 
         # Exercise index().
         with warnings.catch_warnings():
@@ -4847,30 +4847,31 @@ def test_shared_memory_ShareableList_basics(self):
 
         # Exercise retrieving individual values.
         self.assertEqual(sl[0], 'howdy')
-        self.assertEqual(sl[-2], True)
+        self.assertEqual(sl[-3], True)
+        self.assertEqual(sl[-1], '💥 💥')
 
         # Exercise iterability.
         self.assertEqual(
             tuple(sl),
-            ('howdy', b'HoWdY', -273.154, 100, None, True, 42)
+            ('howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥')
         )
 
         # Exercise modifying individual values.
         sl[3] = 42
         self.assertEqual(sl[3], 42)
         sl[4] = 'some'  # Change type at a given position.
         self.assertEqual(sl[4], 'some')
-        self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q')
+        self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q16s')
         with self.assertRaisesRegex(ValueError,
                                     "exceeds available storage"):
             sl[4] = 'far too many'
         self.assertEqual(sl[4], 'some')
-        sl[0] = 'encodés'  # Exactly 8 bytes of UTF-8 data
-        self.assertEqual(sl[0], 'encodés')
+        sl[0] = 'éncodé'  # Exactly 8 bytes of UTF-8 data
+        self.assertEqual(sl[0], 'éncodé')
         self.assertEqual(sl[1], b'HoWdY')  # no spillage
         with self.assertRaisesRegex(ValueError,
                                     "exceeds available storage"):
-            sl[0] = 'encodées'  # Exactly 9 bytes of UTF-8 data
+            sl[0] = 'éncodés'  # Exactly 9 bytes of UTF-8 data
         self.assertEqual(sl[1], b'HoWdY')
         with self.assertRaisesRegex(ValueError,
                                     "exceeds available storage"):

Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst

@@ -0,0 +1 @@
+Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList.