gh-159184: Make the marshal order of set elements with equal dumps stable (GH-159197)

Set elements with the same dump (NaNs which are distinct objects) were
written in set iteration order, which depends on their addresses.  When
one of them is also referenced elsewhere in the dump, the position of
the reference among them changed from run to run.

Elements with equal dumps are now ordered by when they were first
written in this dump; floats and complex numbers not written yet are
interchangeable, and which one is which is decided when one of them is
referenced again.  This relies on the writer knowing exactly which
objects it has written (gh-129724).

The elements are sorted with qsort() on the bytes of their dumps,
in an array instead of a list of tuples.

Author: gpshead

Lib/test/test_marshal.py

@@ -617,7 +617,6 @@ def test_unorderable_set_items(self):
                     self.assertEqual(sorted(map(repr, new)),
                                      sorted(map(repr, sample)))
 
-
     def test_unorderable_set_items_in_code(self):
         # A constant set containing NaNs which are distinct objects.
         # Run in a subprocess: constants of code objects are immortal
@@ -662,7 +661,6 @@ def shrink(sample):
                                                 "set changed size"):
                         marshal.dumps(sample, version)
 
-
     def test_error_in_buffer(self):
         # The BufferError raised for a non-contiguous buffer is not replaced
         # with a generic error.
@@ -1238,7 +1236,8 @@ def test_nan_in_set(self):
                     nan1 = float('nan')
                     nan2 = float('nan')
                     nan3 = float('nan')
-                    s = kind([nan1, 1.5, nan2, nan3])
+                    s = kind([nan1, 1.5, nan2, nan3,
+                              complex('nan'), complex('nan')])
                     for sample in (s, [s], [nan1, s], [s, nan2],
                                    [nan3, s, nan1], [nan1, nan2, nan3, s],
                                    [s, s], (nan2, [s, nan2], s)):
@@ -1247,6 +1246,194 @@ def test_nan_in_set(self):
                         new = marshal.loads(data)
                         self.check_same_structure(new, sample, version >= 3)
 
+    @support.cpython_only
+    def test_nan_in_set_order(self):
+        # The output does not depend on the order of the elements of a set
+        # which have the same marshalled representation, even if some of
+        # them are also referenced elsewhere.
+        version = self.VERSION
+        def flagged(data):
+            data = bytearray(data)
+            data[0] |= self.FLAG_REF
+            return bytes(data)
+        def ref(index):
+            return b'r' + struct.pack('<i', index)
+        keep = []
+        for make_nan in (lambda: float('nan'), lambda: complex('nan')):
+            data = marshal.dumps(make_nan(), version)
+            fdata = flagged(data)
+            data15 = marshal.dumps(1.5, version)
+            for kind, typecode in (set, b'<'), (frozenset, b'>'):
+                head = typecode + b'\4\0\0\0' + data15
+                for i in range(20):
+                    # Different objects at different addresses every time.
+                    keep.append(bytes(i * 8))
+                    nans = [make_nan() for j in range(3)]
+                    keep.append(nans)
+                    s = kind([1.5, *nans])
+                    a, b, c = nans[i % 3:] + nans[:i % 3]
+                    for sample, expected in [
+                        (s, head + data * 3),
+                        ([s, a],
+                         b'[\2\0\0\0' + head + fdata + data * 2 + ref(0)),
+                        ([a, s],
+                         b'[\2\0\0\0' + fdata + head + ref(0) + data * 2),
+                        ([s, a, b, a],
+                         b'[\4\0\0\0' + head + fdata * 2 + data +
+                         ref(0) + ref(1) + ref(0)),
+                        ([a, b, s, c],
+                         b'[\4\0\0\0' + fdata * 2 + head + ref(0) + ref(1) +
+                         flagged(data) + ref(2)),
+                        ([b, s, c, a, kind([a, b, c])],
+                         b'[\5\0\0\0' + fdata + head + ref(0) + fdata * 2 +
+                         ref(1) + ref(2) + typecode + b'\3\0\0\0' +
+                         ref(0) + ref(1) + ref(2)),
+                        ([s, kind([b, a]), kind([c, make_nan()]), b, a, c],
+                         b'[\6\0\0\0' + head + fdata * 3 +
+                         typecode + b'\2\0\0\0' + ref(0) + ref(1) +
+                         typecode + b'\2\0\0\0' + ref(2) + data +
+                         ref(0) + ref(1) + ref(2)),
+                    ]:
+                        self.assertEqual(marshal.dumps(sample, version),
+                                         expected)
+                        new = marshal.loads(expected)
+                        self.check_same_structure(new, sample, True)
+
+    @support.cpython_only
+    def test_nan_constants_in_code(self):
+        # Python has no literal for NaN, but constant folding produces one
+        # from arithmetic on 1e999 (inf).  Such NaNs end up as a constant
+        # of the code object, in a tuple constant, or in a frozenset
+        # constant (the only kind of set the compiler creates, from a set
+        # display used as the right operand of "in").  Every fold creates
+        # a new object, so the compiler cannot produce a NaN which is an
+        # element of a set and is also referenced elsewhere; the ordering
+        # of such elements (test_nan_in_set_order) is not observable here.
+        # What is observable is that the code can be marshalled, and that
+        # the output, so the .pyc file, is the same in every process.
+        sources = {
+            'const': 'x = 1e999 - 1e999',
+            'tuple': 'x = (1e999 - 1e999, 2.0)',
+            'repeat': 'x = (1e999 - 1e999,) * 2',
+            'set': 'x in {1e999 - 1e999, 1e999 - 1e999, 2.0}',
+            'complex': 'x in {(1e999 - 1e999) * 1j, (1e999 - 1e999) * 2j}',
+            'tuples': 'x in {(1e999 - 1e999,) * 2, (1e999 - 1e999,) * 2}',
+            'nested': 'def f(x):\n    return x in {1e999 - 1e999, 2j}',
+        }
+        def nan_places(code):
+            # Where the NaN constants of a code object are, with the
+            # elements of sets in any order.
+            places = []
+            def visit(x, where):
+                if isinstance(x, (float, complex)):
+                    if x != x:
+                        places.append((where, repr(x)))
+                elif isinstance(x, (tuple, frozenset)):
+                    for y in sorted(x, key=repr):
+                        visit(y, type(x).__name__)
+                elif isinstance(x, types.CodeType):
+                    for y in x.co_consts:
+                        visit(y, 'consts')
+            visit(code, None)
+            return places
+        expected = {
+            'const': [('consts', 'nan')],
+            'tuple': [('tuple', 'nan')],
+            'repeat': [('tuple', 'nan'), ('tuple', 'nan')],
+            'set': [('frozenset', 'nan'), ('frozenset', 'nan')],
+            'complex': [('frozenset', '(nan+nanj)'),
+                        ('frozenset', '(nan+nanj)')],
+            'tuples': [('tuple', 'nan')] * 4,
+            'nested': [('frozenset', 'nan')],
+        }
+        dumps = {}
+        for name, source in sources.items():
+            code = compile(source, '<marshal test>', 'exec')
+            self.assertEqual(nan_places(code), expected[name], name)
+            for version in range(marshal.version + 1):
+                data = marshal.dumps(code, version)
+                dumps[name, version] = data
+                new = marshal.loads(data)
+                self.assertEqual(nan_places(new), expected[name], name)
+                self.assertEqual(new.co_code, code.co_code)
+                # Compiling again gives other NaN objects at other
+                # addresses, and the same bytes.
+                code2 = compile(source, '<marshal test>', 'exec')
+                self.assertEqual(marshal.dumps(code2, version), data)
+
+        # The same bytes in other processes.
+        script = textwrap.dedent("""
+            import marshal, sys
+            sources = %r
+            for name, source in sources.items():
+                code = compile(source, '<marshal test>', 'exec')
+                for version in range(marshal.version + 1):
+                    print(name, version, marshal.dumps(code, version).hex())
+            """) % sources
+        for seed in '1', '2', 'random':
+            with self.subTest(hashseed=seed):
+                rc, out, err = assert_python_ok('-c', script,
+                                                PYTHONHASHSEED=seed)
+                for line in out.decode().splitlines():
+                    name, version, data = line.split()
+                    self.assertEqual(bytes.fromhex(data),
+                                     dumps[name, int(version)],
+                                     (name, version))
+
+    def test_nan_in_set_changed(self):
+        # An element which is removed from the set, and another one added,
+        # while a later element is dumped (here by its __buffer__() method)
+        # is still written, since it was seen; but it is then referenced
+        # only by the writer, which only keeps track of objects that have
+        # several references.  The set code must cope with a NaN which is
+        # written but not tracked.
+        class Item:
+            def __init__(self, action, h):
+                self.action = action
+                self.h = h
+            def __hash__(self):
+                return self.h
+            def __buffer__(self, flags):
+                # Called twice: to dump the element, then to write it.
+                action, self.action = self.action, None
+                if action is not None:
+                    action()
+                return memoryview(b'abc')
+
+        def layout(nans, extra):
+            # A hash for the item and a replacement element such that the
+            # NaNs are iterated before the item, and the replacement lands
+            # in a slot which has been iterated already: the writer sees
+            # as many elements as the set has, and does not notice.
+            for h in range(128):
+                for k in range(128, 256):
+                    s = set(nans) | set(extra)
+                    item = Item(None, h)
+                    s.add(item)
+                    seen = []
+                    for x in s:
+                        seen.append(x)
+                        if x is item:
+                            s.discard(nans[0])
+                            s.add(k)
+                    if (len(seen) == len(s) and seen[-1] is item
+                            and all(any(n is x for x in seen) for n in nans)):
+                        return h, k
+            self.skipTest('no suitable set layout found')
+
+        for version in range(marshal.version + 1):
+            with self.subTest(version=version):
+                nans = [float('nan') for i in range(3)]
+                h, k = layout(nans, [1.5])
+                s = set(nans) | {1.5}
+                def remove_one():
+                    s.discard(nans.pop(0))
+                    s.add(k)
+                s.add(Item(remove_one, h))
+                new = marshal.loads(marshal.dumps(s, version))
+                self.assertEqual(sorted(map(repr, new)),
+                                 ['1.5', "b'abc'", 'nan', 'nan', 'nan'])
+
     def check_same_structure(self, new, sample, shared):
         # Both object graphs have the same values and, if shared is true,
         # share the same objects.  Elements of sets are matched in any
@@ -1261,7 +1448,7 @@ def known(x):
                 if shared:
                     return seen.get(id(x), -1)
             def visit(x):
-                if isinstance(x, float):
+                if isinstance(x, (float, complex)):
                     result.append((repr(x), number(x)))
                 elif isinstance(x, (set, frozenset)):
                     # Elements are only numbered if they occur elsewhere.

Misc/NEWS.d/next/Library/2026-10-11-08-42-00.gh-issue-159184.Tq2nSe.rst

@@ -0,0 +1,3 @@
+The output of :func:`marshal.dumps` no longer depends on the order of the
+elements of a set which have the same marshalled form, such as NaNs which
+are distinct objects.