From d701ba394565c544401050906235fde7c23f2887 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 22 Oct 2026 08:39:45 +0000 Subject: [PATCH 1/3] gh-160310: Fix quadratic one-shot decompress() with many members gzip.decompress(), bz2.decompress(), lzma.decompress() and compression.zstd.decompress() passed all of the remaining input to a new decompressor for every member and copied the rest back via unused_data. Feed each member in chunks of increasing size instead. --- Lib/bz2.py | 52 ++++++++++++++----- Lib/compression/zstd/__init__.py | 36 +++++++++---- Lib/gzip.py | 42 +++++++++++---- Lib/lzma.py | 52 +++++++++++++------ Lib/test/test_bz2.py | 43 +++++++++++++++ Lib/test/test_gzip.py | 44 ++++++++++++++++ Lib/test/test_lzma.py | 44 ++++++++++++++++ Lib/test/test_zstd.py | 46 ++++++++++++++++ ...-10-22-08-34-22.gh-issue-160310.q7Zm2K.rst | 4 ++ 9 files changed, 315 insertions(+), 48 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-22-08-34-22.gh-issue-160310.q7Zm2K.rst diff --git a/Lib/bz2.py b/Lib/bz2.py index 79305669efc38c9..f6f6223b65f4186 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -337,18 +337,42 @@ def decompress(data): For incremental decompression, use a BZ2Decompressor object instead. """ results = [] - while data: - decomp = BZ2Decompressor() - try: - res = decomp.decompress(data) - except OSError: - if results: - break # Leftover data is not a valid bzip2 stream; ignore it. - else: - raise # Error on the first iteration; bail out. - results.append(res) - if not decomp.eof: - raise ValueError("Compressed data ended before the " - "end-of-stream marker was reached") - data = decomp.unused_data + if not data: + return b"" + # The views are released on exit, so that the caller can resize a + # bytearray argument again, also while handling an exception. + with memoryview(data) as view, view.cast("B") as data: + pos = 0 + # There is usually a single stream: the first chunk is all of the + # data, to decompress it in one go. + chunk_size = len(data) + while True: + decomp = BZ2Decompressor() + res = [] + try: + # Feed the stream in chunks of increasing size, so that what + # is read past its end (and copied into unused_data) is + # bounded by the size of the stream, not by the size of all + # the streams that follow it. + while not decomp.eof and pos < len(data): + with data[pos:pos + chunk_size] as chunk: + res.append(decomp.decompress(chunk)) + pos += len(chunk) + chunk_size *= 2 + except OSError: + if results: + # Leftover data is not a valid bzip2 stream; ignore it. + break + else: + raise # Error on the first iteration; bail out. + results.extend(res) + if not decomp.eof: + raise ValueError("Compressed data ended before the " + "end-of-stream marker was reached") + pos -= len(decomp.unused_data) + if pos == len(data): + break + # Later streams start small: a larger first chunk reads further + # past the end of a short stream, a smaller one needs more calls. + chunk_size = 1024 return b"".join(results) diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index 22ce028c435caa8..757c11a82833b83 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -161,15 +161,33 @@ def decompress(data, zstd_dict=None, options=None): For incremental decompression, use a ZstdDecompressor instead. """ results = [] - while True: - decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict) - results.append(decomp.decompress(data)) - if not decomp.eof: - raise ZstdError('Compressed data ended before the ' - 'end-of-stream marker was reached') - data = decomp.unused_data - if not data: - break + # The views are released on exit, so that the caller can resize a + # bytearray argument again, also while handling an exception. + with memoryview(data) as view, view.cast('B') as data: + pos = 0 + # There is usually a single frame: the first chunk is all of the + # data, to decompress it in one go. + chunk_size = len(data) + while True: + decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict) + # Feed the frame in chunks of increasing size, so that what is + # read past its end (and copied into unused_data) is bounded by + # the size of the frame, not by the size of all the frames that + # follow it. + while not decomp.eof and pos < len(data): + with data[pos:pos + chunk_size] as chunk: + results.append(decomp.decompress(chunk)) + pos += len(chunk) + chunk_size *= 2 + if not decomp.eof: + raise ZstdError('Compressed data ended before the ' + 'end-of-stream marker was reached') + pos -= len(decomp.unused_data) + if pos == len(data): + break + # Later frames start small: a larger first chunk reads further + # past the end of a short frame, a smaller one needs more calls. + chunk_size = 1024 return b''.join(results) diff --git a/Lib/gzip.py b/Lib/gzip.py index 5b6ebcfef45c3ad..ca2834827cffaea 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -682,24 +682,48 @@ def decompress(data): Return the decompressed string. """ decompressed_members = [] + fp = io.BytesIO(data) + # There is usually a single member: the first chunk is all of the data, + # to decompress it in one go. + chunk_size = fp.seek(0, io.SEEK_END) + fp.seek(0) while True: - fp = io.BytesIO(data) if _read_gzip_header(fp) is None: return b"".join(decompressed_members) # Use a zlib raw deflate compressor do = zlib.decompressobj(wbits=-zlib.MAX_WBITS) - # Read all the data except the header - decompressed = do.decompress(data[fp.tell():]) - if not do.eof or len(do.unused_data) < 8: + crc = zlib.crc32(b"") + length = 0 + # Feed the member in chunks of increasing size, so that what is read + # past its end (and copied into unused_data) is bounded by the size + # of the member, not by the size of all the members that follow it. + while not do.eof and (chunk := fp.read(chunk_size)): + decompressed = do.decompress(chunk) + crc = zlib.crc32(decompressed, crc) + length += len(decompressed) + decompressed_members.append(decompressed) + chunk_size *= 2 + # Later members start small: a larger first chunk reads further past + # the end of a short member, a smaller one needs more calls. + chunk_size = 1024 + # Rewind to the end of the deflate stream and read the trailer + fp.seek(-len(do.unused_data), io.SEEK_CUR) + trailer = fp.read(8) + if not do.eof or len(trailer) < 8: raise EOFError("Compressed file ended before the end-of-stream " "marker was reached") - crc, length = struct.unpack(" Date: Sat, 31 Oct 2026 20:40:30 +0000 Subject: [PATCH 2/3] gh-160310: Share the one-shot decompress loop in compression._common bz2.decompress(), lzma.decompress() and compression.zstd.decompress() had three copies of the chunked multi-stream loop. Move it into compression._common._streams._decompress_all(), which takes the decompressor factory, the trailing-data error, the stream magic and the truncation error. gzip keeps its own loop. --- Lib/bz2.py | 44 ++-------------------- Lib/compression/_common/_streams.py | 57 +++++++++++++++++++++++++++++ Lib/compression/zstd/__init__.py | 33 ++--------------- Lib/lzma.py | 47 +++--------------------- Lib/test/test_bz2.py | 4 ++ Lib/test/test_lzma.py | 7 ++++ 6 files changed, 80 insertions(+), 112 deletions(-) diff --git a/Lib/bz2.py b/Lib/bz2.py index ba6501f0e47dd51..28d339f0a2ccb1a 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -341,49 +341,11 @@ def decompress(data): For incremental decompression, use a BZ2Decompressor object instead. """ - results = [] if not data: return b"" - # The views are released on exit, so that the caller can resize a - # bytearray argument again, also while handling an exception. - with memoryview(data) as view, view.cast("B") as data: - pos = 0 - # There is usually a single stream: the first chunk is all of the - # data, to decompress it in one go. - chunk_size = len(data) - while True: - decomp = BZ2Decompressor() - start = pos - res = [] - try: - # Feed the stream in chunks of increasing size, so that what - # is read past its end (and copied into unused_data) is - # bounded by the size of the stream, not by the size of all - # the streams that follow it. - while not decomp.eof and pos < len(data): - with data[pos:pos + chunk_size] as chunk: - res.append(decomp.decompress(chunk)) - pos += len(chunk) - chunk_size *= 2 - except OSError: - if not results: - raise # Error on the first iteration; bail out. - with data[start:start + 4] as head: - if head.tobytes().startswith(_STREAM_MAGIC): - # Leftover data is a damaged bzip2 stream; bail out. - raise - break # Leftover data is not a valid bzip2 stream; ignore it. - results.extend(res) - if not decomp.eof: - raise ValueError("Compressed data ended before the " - "end-of-stream marker was reached") - pos -= len(decomp.unused_data) - if pos == len(data): - break - # Later streams start small: a larger first chunk reads further - # past the end of a short stream, a smaller one needs more calls. - chunk_size = 1024 - return b"".join(results) + return _streams._decompress_all(data, BZ2Decompressor, OSError, + stream_magic=_STREAM_MAGIC, + truncated_error=ValueError) def __getattr__(name): diff --git a/Lib/compression/_common/_streams.py b/Lib/compression/_common/_streams.py index 09c65507297886b..0e3308c0c6e5fdc 100644 --- a/Lib/compression/_common/_streams.py +++ b/Lib/compression/_common/_streams.py @@ -169,3 +169,60 @@ def seek(self, offset, whence=io.SEEK_SET): def tell(self): """Return the current file position.""" return self._pos + + +def _decompress_all(data, decomp_factory, trailing_error=(), *, + stream_magic=(), truncated_error, **decomp_args): + """Decompress all of the concatenated streams in *data*. + + Each stream is decompressed by a new decompressor, made by calling + *decomp_factory* with *decomp_args*. As in DecompressReader, an error + of class *trailing_error* in a stream after the first one ends the + data, unless that stream starts with *stream_magic*. A stream without + an end-of-stream marker raises *truncated_error*. + """ + results = [] + # The views are released on exit, so that the caller can resize a + # bytearray argument again, also while handling an exception. + with memoryview(data) as view, view.cast("B") as data: + pos = 0 + # There is usually a single stream: the first chunk is all of the + # data, to decompress it in one go. + chunk_size = len(data) + while True: + decomp = decomp_factory(**decomp_args) + start = pos + res = [] + try: + # Feed the stream in chunks of increasing size, so that what + # is read past its end (and copied into unused_data) is + # bounded by the size of the stream, not by the size of all + # the streams that follow it. + while not decomp.eof and pos < len(data): + with data[pos:pos + chunk_size] as chunk: + res.append(decomp.decompress(chunk)) + pos += len(chunk) + chunk_size *= 2 + except trailing_error: + if not results: + raise # Error in the first stream; bail out. + if stream_magic: + if isinstance(stream_magic, bytes): + stream_magic = (stream_magic,) + size = max(map(len, stream_magic)) + with data[start:start + size] as head: + if head.tobytes().startswith(stream_magic): + raise # Damaged stream, not trailing data. + # Trailing data isn't a valid compressed stream; ignore it. + break + results.extend(res) + if not decomp.eof: + raise truncated_error("Compressed data ended before the " + "end-of-stream marker was reached") + pos -= len(decomp.unused_data) + if pos == len(data): + break + # Later streams start small: a larger first chunk reads further + # past the end of a short stream, a smaller one needs more calls. + chunk_size = 1024 + return b"".join(results) diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index 757c11a82833b83..c87cac26691cf1a 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -33,6 +33,7 @@ from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError, get_frame_size, zstd_version, ZSTD_VERSION, zstd_version_info, ZSTD_VERSION_INFO) +from compression._common import _streams from compression.zstd._zstdfile import ZstdFile, open, _nbytes @@ -160,35 +161,9 @@ def decompress(data, zstd_dict=None, options=None): For incremental decompression, use a ZstdDecompressor instead. """ - results = [] - # The views are released on exit, so that the caller can resize a - # bytearray argument again, also while handling an exception. - with memoryview(data) as view, view.cast('B') as data: - pos = 0 - # There is usually a single frame: the first chunk is all of the - # data, to decompress it in one go. - chunk_size = len(data) - while True: - decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict) - # Feed the frame in chunks of increasing size, so that what is - # read past its end (and copied into unused_data) is bounded by - # the size of the frame, not by the size of all the frames that - # follow it. - while not decomp.eof and pos < len(data): - with data[pos:pos + chunk_size] as chunk: - results.append(decomp.decompress(chunk)) - pos += len(chunk) - chunk_size *= 2 - if not decomp.eof: - raise ZstdError('Compressed data ended before the ' - 'end-of-stream marker was reached') - pos -= len(decomp.unused_data) - if pos == len(data): - break - # Later frames start small: a larger first chunk reads further - # past the end of a short frame, a smaller one needs more calls. - chunk_size = 1024 - return b''.join(results) + return _streams._decompress_all(data, ZstdDecompressor, + truncated_error=ZstdError, + options=options, zstd_dict=zstd_dict) class CompressionParameter(enum.IntEnum): diff --git a/Lib/lzma.py b/Lib/lzma.py index 4562ee749378cd1..df749fc119a5843 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -354,45 +354,8 @@ def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None): For incremental decompression, use an LZMADecompressor instead. """ - results = [] - magic = _stream_magic(format) - # The views are released on exit, so that the caller can resize a - # bytearray argument again, also while handling an exception. - with memoryview(data) as view, view.cast("B") as data: - pos = 0 - # There is usually a single stream: the first chunk is all of the - # data, to decompress it in one go. - chunk_size = len(data) - while True: - decomp = LZMADecompressor(format, memlimit, filters) - start = pos - res = [] - try: - # Feed the stream in chunks of increasing size, so that what - # is read past its end (and copied into unused_data) is - # bounded by the size of the stream, not by the size of all - # the streams that follow it. - while not decomp.eof and pos < len(data): - with data[pos:pos + chunk_size] as chunk: - res.append(decomp.decompress(chunk)) - pos += len(chunk) - chunk_size *= 2 - except LZMAError: - if not results: - raise # Error on the first iteration; bail out. - with data[start:start + len(magic)] as head: - if magic and head.tobytes().startswith(magic): - # Leftover data is a damaged XZ stream; bail out. - raise - break # Leftover data is not a valid LZMA/XZ stream; ignore it. - results.extend(res) - if not decomp.eof: - raise LZMAError("Compressed data ended before the " - "end-of-stream marker was reached") - pos -= len(decomp.unused_data) - if pos == len(data): - break - # Later streams start small: a larger first chunk reads further - # past the end of a short stream, a smaller one needs more calls. - chunk_size = 1024 - return b"".join(results) + return _streams._decompress_all(data, LZMADecompressor, LZMAError, + stream_magic=_stream_magic(format), + truncated_error=LZMAError, + format=format, memlimit=memlimit, + filters=filters) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 0273fc2f03e83c3..6e0a1207ddfc186 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -1126,6 +1126,10 @@ def testDecompressMultiStreamBig(self): self.assertEqual(bz2.decompress(memoryview(data)), expected) self.assertEqual(bz2.decompress(data + self.BAD_DATA), expected) self.assertRaises(ValueError, bz2.decompress, data[:-50_000]) + # The damage is past the first chunk fed for the second stream. + damaged = bytearray(bz2.compress(big)) + damaged[-3] ^= 1 + self.assertRaises(OSError, bz2.decompress, self.DATA + damaged) def testDecompressErrorReleasesBuffer(self): # The argument can be resized while the exception is handled. diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index a0e0d0147ff5e08..c14b3ebf586c96f 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -524,6 +524,10 @@ def test_decompress_multistream_big(self): self.assertEqual(lzma.decompress(memoryview(cdata)), expected) self.assertEqual(lzma.decompress(cdata + COMPRESSED_BOGUS), expected) self.assertRaises(LZMAError, lzma.decompress, cdata[:-50_000]) + # The damage is past the first chunk fed for the second stream. + damaged = bytearray(lzma.compress(big)) + damaged[-1] ^= 1 + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_XZ + damaged) def test_decompress_error_releases_buffer(self): # The argument can be resized while the exception is handled. @@ -563,6 +567,9 @@ def decompress(self, data): def test_decompress_trailing_junk(self): ddata = lzma.decompress(COMPRESSED_XZ + COMPRESSED_BOGUS) self.assertEqual(ddata, INPUT) + ddata = lzma.decompress(COMPRESSED_ALONE + COMPRESSED_BOGUS, + lzma.FORMAT_ALONE) + self.assertEqual(ddata, INPUT) def test_decompress_multistream_trailing_junk(self): ddata = lzma.decompress(COMPRESSED_XZ * 3 + COMPRESSED_BOGUS) From 1cde995989bab214e4e3823d44904b844a191f32 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 31 Oct 2026 21:15:23 +0000 Subject: [PATCH 3/3] gh-160310: Use len(data) for the first gzip chunk size --- Lib/gzip.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/gzip.py b/Lib/gzip.py index ca2834827cffaea..95c05fb98e47e71 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -685,8 +685,7 @@ def decompress(data): fp = io.BytesIO(data) # There is usually a single member: the first chunk is all of the data, # to decompress it in one go. - chunk_size = fp.seek(0, io.SEEK_END) - fp.seek(0) + chunk_size = len(data) while True: if _read_gzip_header(fp) is None: return b"".join(decompressed_members)