gh-88300: Support .xz stream padding in lzma.decompress() and LZMAFile

Null bytes in multiples of four after an .xz stream are now skipped, so
that a stream following them is decoded instead of ignored.

Author: gpshead

Lib/compression/_common/_streams.py

@@ -37,7 +37,7 @@ def readable(self):
         return True
 
     def __init__(self, fp, decomp_factory, trailing_error=(), *,
-                 stream_magic=(), **decomp_args):
+                 stream_magic=(), stream_padding=False, **decomp_args):
         self._fp = fp
         self._eof = False
         self._pos = 0  # Current offset in decompressed stream
@@ -62,6 +62,12 @@ def __init__(self, fp, decomp_factory, trailing_error=(), *,
         # stream rather than trailing data, so its error is not ignored.
         self._stream_magic = stream_magic
 
+        # Whether a stream that starts with stream_magic (bytes) may be
+        # followed by Stream Padding, as in the .xz format.  _head holds the
+        # first bytes of the current stream.
+        self._stream_padding = stream_padding and bool(stream_magic)
+        self._head = b""
+
     def close(self):
         self._decompressor = None
         return super().close()
@@ -88,11 +94,15 @@ def read(self, size=-1):
             if self._decompressor.eof:
                 rawblock = (self._decompressor.unused_data or
                             self._fp.read(BUFFER_SIZE))
+                if self._stream_padding and self._head == self._stream_magic:
+                    rawblock = self._skip_padding(rawblock)
                 if not rawblock:
                     break
                 # Continue to next stream.
                 self._decompressor = self._decomp_factory(
                     **self._decomp_args)
+                if self._stream_padding:
+                    self._head = rawblock[:len(self._stream_magic)]
                 try:
                     data = self._decompressor.decompress(rawblock, size)
                 except self._trailing_error:
@@ -109,6 +119,10 @@ def read(self, size=-1):
                                        "end-of-stream marker was reached")
                 else:
                     rawblock = b""
+                if (self._stream_padding
+                        and len(self._head) < len(self._stream_magic)):
+                    self._head += rawblock[:len(self._stream_magic)
+                                           - len(self._head)]
                 data = self._decompressor.decompress(rawblock, size)
             if data:
                 break
@@ -129,12 +143,30 @@ def readall(self):
 
         return b"".join(chunks)
 
+    def _skip_padding(self, rawblock):
+        # Skip the null bytes at the start of the data following a stream
+        # if they are Stream Padding, whose size is a multiple of four.
+        size = 0
+        while True:
+            rest = rawblock.lstrip(b"\0")
+            size += len(rawblock) - len(rest)
+            if rest:
+                break
+            rawblock = self._fp.read(BUFFER_SIZE)
+            if not rawblock:
+                break
+        if size % 4:
+            # Not Stream Padding: leave the null bytes to the decompressor.
+            return bytes(size) + rest
+        return rest
+
     # Rewind the file to the beginning of the data stream.
     def _rewind(self):
         self._fp.seek(0)
         self._eof = False
         self._pos = 0
         self._decompressor = self._decomp_factory(**self._decomp_args)
+        self._head = b""
 
     def seek(self, offset, whence=io.SEEK_SET):
         # Recalculate offset as an absolute file position.
@@ -171,15 +203,37 @@ def tell(self):
         return self._pos
 
 
+def _padding_size(data, pos):
+    """Return the size of the Stream Padding at *data[pos:]*, or 0.
+
+    Stream Padding is a run of null bytes whose size is a multiple of four.
+    """
+    end = pos
+    chunk_size = 4
+    while end < len(data):
+        with data[end:end + chunk_size] as chunk:
+            block = chunk.tobytes()
+        rest = len(block.lstrip(b"\0"))
+        end += len(block) - rest
+        if rest:
+            break
+        chunk_size *= 2
+    size = end - pos
+    return 0 if size % 4 else size
+
+
 def _decompress_all(data, decomp_factory, trailing_error=(), *,
-                    stream_magic=(), truncated_error, **decomp_args):
+                    stream_magic=(), stream_padding=False, truncated_error,
+                    **decomp_args):
     """Decompress all of the concatenated streams in *data*.
 
     Each stream is decompressed by a new decompressor, made by calling
     *decomp_factory* with *decomp_args*.  As in DecompressReader, an error
     of class *trailing_error* in a stream after the first one ends the
     data, unless that stream starts with *stream_magic*.  A stream without
-    an end-of-stream marker raises *truncated_error*.
+    an end-of-stream marker raises *truncated_error*.  If *stream_padding*
+    is true, a stream that starts with *stream_magic* (bytes) may be
+    followed by Stream Padding, as in the .xz format.
     """
     results = []
     # The views are released on exit, so that the caller can resize a
@@ -220,6 +274,10 @@ def _decompress_all(data, decomp_factory, trailing_error=(), *,
                 raise truncated_error("Compressed data ended before the "
                                       "end-of-stream marker was reached")
             pos -= len(decomp.unused_data)
+            if stream_padding and stream_magic:
+                with data[start:start + len(stream_magic)] as head:
+                    if head == stream_magic:
+                        pos += _padding_size(data, pos)
             if pos == len(data):
                 break
             # Later streams start small: a larger first chunk reads further

Lib/lzma.py

@@ -138,7 +138,7 @@ def __init__(self, filename=None, mode="r", *,
         if self._mode == _MODE_READ:
             raw = _streams.DecompressReader(self._fp, LZMADecompressor,
                 trailing_error=LZMAError, stream_magic=_stream_magic(format),
-                format=format, filters=filters)
+                stream_padding=True, format=format, filters=filters)
             self._buffer = io.BufferedReader(raw)
 
     def close(self):
@@ -356,6 +356,7 @@ def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None):
     """
     return _streams._decompress_all(data, LZMADecompressor, LZMAError,
                                     stream_magic=_stream_magic(format),
+                                    stream_padding=True,
                                     truncated_error=LZMAError,
                                     format=format, memlimit=memlimit,
                                     filters=filters)

Lib/test/test_lzma.py

@@ -585,6 +585,28 @@ def test_decompress_multistream_damaged_stream(self):
                     self.assertRaises(LZMAError, lzma.decompress,
                                       COMPRESSED_XZ + damaged, format)
 
+    def test_decompress_stream_padding(self):
+        # Null bytes in multiples of four can follow each .xz stream.
+        cdata = (COMPRESSED_XZ + bytes(4) + COMPRESSED_XZ + bytes(20) +
+                 COMPRESSED_XZ + bytes(100_000) + COMPRESSED_XZ + bytes(8))
+        for format in lzma.FORMAT_AUTO, lzma.FORMAT_XZ:
+            with self.subTest(format=format):
+                self.assertEqual(lzma.decompress(COMPRESSED_XZ + bytes(4),
+                                                 format), INPUT)
+                self.assertEqual(lzma.decompress(cdata, format), INPUT * 4)
+                # A damaged stream after padding is not trailing junk.
+                damaged = bytearray(COMPRESSED_XZ)
+                damaged[-20] ^= 1
+                self.assertRaises(LZMAError, lzma.decompress,
+                                  COMPRESSED_XZ + bytes(4) + damaged, format)
+                self.assertRaises(LZMAError, lzma.decompress,
+                                  bytes(4) + COMPRESSED_XZ, format)
+        # Not after an .lzma stream.
+        for format in lzma.FORMAT_AUTO, lzma.FORMAT_ALONE:
+            with self.subTest(format=format):
+                self.assertRaises(LZMAError, lzma.decompress,
+                                  COMPRESSED_ALONE + bytes(4), format)
+
 
 class TempFile:
     """Context manager - creates a file, and deletes it on __exit__."""
@@ -965,6 +987,39 @@ def test_read_multistream_damaged_stream(self):
                     with LZMAFile(BytesIO(cdata), format=format) as f:
                         self.assertRaises(LZMAError, f.seek, 0, 2)
 
+    def test_read_stream_padding(self):
+        # Null bytes in multiples of four can follow each .xz stream,
+        # whatever the size of the reads.
+        cdata = (COMPRESSED_XZ + bytes(4) + COMPRESSED_XZ + bytes(20) +
+                 COMPRESSED_XZ + bytes(10_000) + COMPRESSED_XZ + bytes(8))
+        damaged = bytearray(COMPRESSED_XZ)
+        damaged[-20] ^= 1
+        damaged = COMPRESSED_XZ + bytes(4) + damaged
+        for buffer_size in _streams.BUFFER_SIZE, len(COMPRESSED_XZ) + 2, 3:
+            for format in lzma.FORMAT_AUTO, lzma.FORMAT_XZ:
+                with (self.subTest(buffer_size=buffer_size, format=format),
+                      support.swap_attr(_streams, "BUFFER_SIZE",
+                                        buffer_size)):
+                    with LZMAFile(BytesIO(COMPRESSED_XZ + bytes(4)),
+                                  format=format) as f:
+                        self.assertEqual(f.read(), INPUT)
+                    with LZMAFile(BytesIO(cdata), format=format) as f:
+                        self.assertEqual(f.read(), INPUT * 4)
+                        self.assertEqual(f.seek(0), 0)
+                        chunks = []
+                        while result := f.read(10):
+                            chunks.append(result)
+                        self.assertEqual(b"".join(chunks), INPUT * 4)
+                    # A damaged stream after padding is not trailing junk.
+                    with LZMAFile(BytesIO(damaged), format=format) as f:
+                        self.assertRaises(LZMAError, f.read)
+        # Not after an .lzma stream.
+        for format in lzma.FORMAT_AUTO, lzma.FORMAT_ALONE:
+            with self.subTest(format=format):
+                with LZMAFile(BytesIO(COMPRESSED_ALONE + bytes(4)),
+                              format=format) as f:
+                    self.assertRaises(EOFError, f.read)
+
     def test_read_from_file(self):
         with TempFile(TESTFN, COMPRESSED_XZ):
             with LZMAFile(TESTFN) as f:

Misc/NEWS.d/next/Library/2026-11-01-03-34-23.gh-issue-88300.GsD4W1.rst

@@ -0,0 +1,3 @@
+Fix :func:`lzma.decompress` and :class:`lzma.LZMAFile` raising an error for
+``.xz`` stream padding (null bytes in multiples of four) at the end of the
+data, and ignoring any ``.xz`` streams that follow such padding.