From 9097fc1390a2a2de7eff573f2a79be8f28b8d56c Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Tue, 23 Jun 2026 09:34:44 +0800 Subject: [PATCH 1/7] GHSA-p384-rgv5-vhc2: Bound zipfile decompression for bzip2/LZMA/Zstandard zipfile.ZipExtFile._read1() bounds the output of each decompress() call for DEFLATE members by passing a max_length to zlib, but for bzip2, LZMA, and Zstandard members it called decompress() with no bound. A whole compressed chunk was therefore expanded into a single allocation before the data[:self._left] clip ran, so a consumer that deliberately reads in small chunks to limit memory (for example zf.open(name).read(8192)) was silently unprotected for non-DEFLATE members. A small, spec-conformant archive member declaring a large uncompressed size could drive multi-GB peak memory. _read1() now passes a per-call bound to the non-DEFLATE decompress() (mirroring the DEFLATE branch) and drains the decompressor's internal buffer across calls by checking needs_input before reading more compressed input. zipfile's LZMADecompressor wrapper forwards max_length and exposes needs_input so the bound also holds for LZMA members. --- Lib/test/test_zipfile/test_core.py | 41 +++++++++++++++++++ Lib/zipfile/__init__.py | 31 +++++++++++--- ...22-09-45-00.GHSA-p384-rgv5-vhc2.zKb3Lm.rst | 4 ++ 3 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-06-22-09-45-00.GHSA-p384-rgv5-vhc2.zKb3Lm.rst diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 4f20209927e7b3d..29bdbab6555b2af 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -4832,6 +4832,47 @@ def tearDown(self): unlink(TESTFN2) +@requires_subprocess() +class BoundedDecompressTests(unittest.TestCase): + # gh-151857: ZipExtFile._read1() bounds the output of each decompress() + # call for DEFLATE members, but historically did not for bzip2/lzma/zstd. + # A small, spec-conformant member that declares a large uncompressed size + # could therefore expand into one unbounded allocation even when the + # consumer deliberately reads in small chunks. Verify the bound now holds + # for every non-DEFLATE compression method. + @unittest.skipUnless(sys.platform.startswith("linux"), + "RLIMIT_AS is only reliably enforced on Linux") + def test_decompress_is_chunk_bounded(self): + for comp in ("ZIP_BZIP2", "ZIP_LZMA", "ZIP_ZSTANDARD"): + with self.subTest(compression=comp): + child = f"""if True: + import resource, io, zipfile + comp = getattr(zipfile, {comp!r}, None) + if comp is None: + print("SKIP"); raise SystemExit + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compression=comp) as z: + z.writestr("big", b"\\0" * (256 * 1024 * 1024)) + data = buf.getvalue() + # 512 MiB address-space cap: ample for a bounded streaming + # read, far below the 256 MiB single-shot expansion the bug + # would attempt on top of the interpreter's own footprint. + soft, hard = resource.getrlimit(resource.RLIMIT_AS) + resource.setrlimit(resource.RLIMIT_AS, (512*1024*1024, hard)) + with zipfile.ZipFile(io.BytesIO(data)) as z: + got = z.open("big").read(8192) + assert len(got) == 8192, len(got) + print("OK") + """ + r = subprocess.run([sys.executable, "-c", child], + capture_output=True, text=True) + out = (r.stdout + r.stderr).strip() + if "SKIP" in out: + self.skipTest(f"{comp} unavailable") + self.assertEqual(r.returncode, 0, out) + self.assertIn("OK", out) + + class AbstractBadCrcTests: def test_testzip_with_bad_crc(self): """Tests that files with bad CRCs return their name from testzip.""" diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 084a47518a935cc..b8e1329cc60ad4e 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -792,7 +792,16 @@ def unused_data(self): except AttributeError: return b'' - def decompress(self, data): + @property + def needs_input(self): + # While the LZMA properties header is still being buffered, more input + # is required; afterwards defer to the wrapped decompressor so a bounded + # decompress() call can be drained across reads. + if self._decomp is None: + return True + return self._decomp.needs_input + + def decompress(self, data, max_length=-1): if self._decomp is None: self._unconsumed += data if len(self._unconsumed) <= 4: @@ -808,7 +817,7 @@ def decompress(self, data): data = self._unconsumed[4 + psize:] del self._unconsumed - result = self._decomp.decompress(data) + result = self._decomp.decompress(data, max_length) self.eof = self._decomp.eof return result @@ -1177,8 +1186,15 @@ def _read1(self, n): data = self._decompressor.unconsumed_tail if n > len(data): data += self._read2(n - len(data)) - else: + elif self._compress_type == ZIP_STORED: data = self._read2(n) + else: + # bzip2/lzma/zstd: a bounded decompress() call may leave input + # buffered inside the decompressor; drain that before reading more. + if self._decompressor.needs_input: + data = self._read2(n) + else: + data = b'' if self._compress_type == ZIP_STORED: self._eof = self._compress_left <= 0 @@ -1191,8 +1207,13 @@ def _read1(self, n): if self._eof: data += self._decompressor.flush() else: - data = self._decompressor.decompress(data) - self._eof = self._decompressor.eof or self._compress_left <= 0 + # Bound the output of a single decompress() call (mirroring the + # DEFLATE path above) so that a small compressed member cannot + # expand into one unbounded allocation (decompression bomb). + data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE)) + self._eof = (self._decompressor.eof or + self._compress_left <= 0 and + self._decompressor.needs_input) data = data[:self._left] self._left -= len(data) diff --git a/Misc/NEWS.d/next/Security/2026-06-22-09-45-00.GHSA-p384-rgv5-vhc2.zKb3Lm.rst b/Misc/NEWS.d/next/Security/2026-06-22-09-45-00.GHSA-p384-rgv5-vhc2.zKb3Lm.rst new file mode 100644 index 000000000000000..4e49ad5ce8fa00a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-22-09-45-00.GHSA-p384-rgv5-vhc2.zKb3Lm.rst @@ -0,0 +1,4 @@ +Bound the amount of data :mod:`zipfile` decompresses per read for members +compressed with bzip2, LZMA, or Zstandard, matching the existing limit for +deflate. A small archive member could previously expand into an unbounded +allocation even when read in small chunks. From d61a324322cffc841e6baa8dca80fd0c577d445c Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Fri, 10 Jul 2026 09:38:09 +0800 Subject: [PATCH 2/7] GHSA-p384-rgv5-vhc2: Address review: private _needs_input and reword bound comment --- Lib/zipfile/__init__.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index b8e1329cc60ad4e..9511294c761876b 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -793,7 +793,7 @@ def unused_data(self): return b'' @property - def needs_input(self): + def _needs_input(self): # While the LZMA properties header is still being buffered, more input # is required; afterwards defer to the wrapped decompressor so a bounded # decompress() call can be drained across reads. @@ -884,6 +884,13 @@ def _get_compressor(compress_type, compresslevel=None): return None +def _decompressor_needs_input(decompressor): + # bz2/zstd expose the stdlib decompressor's public needs_input; the LZMA + # wrapper keeps it private (_needs_input) to avoid adding public API. + needs_input = getattr(decompressor, "needs_input", None) + return decompressor._needs_input if needs_input is None else needs_input + + def _get_decompressor(compress_type): _check_compression(compress_type) if compress_type == ZIP_STORED: @@ -1191,7 +1198,7 @@ def _read1(self, n): else: # bzip2/lzma/zstd: a bounded decompress() call may leave input # buffered inside the decompressor; drain that before reading more. - if self._decompressor.needs_input: + if _decompressor_needs_input(self._decompressor): data = self._read2(n) else: data = b'' @@ -1209,11 +1216,11 @@ def _read1(self, n): else: # Bound the output of a single decompress() call (mirroring the # DEFLATE path above) so that a small compressed member cannot - # expand into one unbounded allocation (decompression bomb). + # expand into one unbounded read. data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE)) self._eof = (self._decompressor.eof or self._compress_left <= 0 and - self._decompressor.needs_input) + _decompressor_needs_input(self._decompressor)) data = data[:self._left] self._left -= len(data) From 660522baa233fd6bfdf9802a9bf6258ee6b51341 Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Tue, 28 Jul 2026 20:08:12 +0800 Subject: [PATCH 3/7] GHSA-p384-rgv5-vhc2: Rewrite the bound test to call _read1 directly Replace the Linux-only subprocess RSS test with a cross-platform check that _read1() output is bounded by MIN_READ_SIZE for bzip2/LZMA/Zstandard. --- Lib/test/test_zipfile/test_core.py | 68 +++++++++++++----------------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 29bdbab6555b2af..56f554c317650ae 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -4832,45 +4832,35 @@ def tearDown(self): unlink(TESTFN2) -@requires_subprocess() -class BoundedDecompressTests(unittest.TestCase): - # gh-151857: ZipExtFile._read1() bounds the output of each decompress() - # call for DEFLATE members, but historically did not for bzip2/lzma/zstd. - # A small, spec-conformant member that declares a large uncompressed size - # could therefore expand into one unbounded allocation even when the - # consumer deliberately reads in small chunks. Verify the bound now holds - # for every non-DEFLATE compression method. - @unittest.skipUnless(sys.platform.startswith("linux"), - "RLIMIT_AS is only reliably enforced on Linux") - def test_decompress_is_chunk_bounded(self): - for comp in ("ZIP_BZIP2", "ZIP_LZMA", "ZIP_ZSTANDARD"): - with self.subTest(compression=comp): - child = f"""if True: - import resource, io, zipfile - comp = getattr(zipfile, {comp!r}, None) - if comp is None: - print("SKIP"); raise SystemExit - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w", compression=comp) as z: - z.writestr("big", b"\\0" * (256 * 1024 * 1024)) - data = buf.getvalue() - # 512 MiB address-space cap: ample for a bounded streaming - # read, far below the 256 MiB single-shot expansion the bug - # would attempt on top of the interpreter's own footprint. - soft, hard = resource.getrlimit(resource.RLIMIT_AS) - resource.setrlimit(resource.RLIMIT_AS, (512*1024*1024, hard)) - with zipfile.ZipFile(io.BytesIO(data)) as z: - got = z.open("big").read(8192) - assert len(got) == 8192, len(got) - print("OK") - """ - r = subprocess.run([sys.executable, "-c", child], - capture_output=True, text=True) - out = (r.stdout + r.stderr).strip() - if "SKIP" in out: - self.skipTest(f"{comp} unavailable") - self.assertEqual(r.returncode, 0, out) - self.assertIn("OK", out) +class AbstractBoundedDecompressTests: + # ZipExtFile._read1() bounds the output of each decompress() call so that a + # small member declaring a large uncompressed size cannot expand into one + # unbounded read. DEFLATE was already bounded; check the other methods too. + def test_read1_output_is_bounded(self): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compression=self.compression) as zf: + zf.writestr("big", b"\0" * (4 * 1024 * 1024)) + with zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf: + with zf.open("big") as f: + self.assertLessEqual(len(f._read1(100)), f.MIN_READ_SIZE) + + +@requires_bz2() +class Bzip2BoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + + +@requires_lzma() +class LzmaBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_LZMA + + +@requires_zstd() +class ZstdBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_ZSTANDARD class AbstractBadCrcTests: From 6135ee606411bba408371c1e96b91133b847a145 Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Tue, 11 Aug 2026 22:36:29 +0800 Subject: [PATCH 4/7] GHSA-p384-rgv5-vhc2: Add ZIP_STORED/ZIP_DEFLATED bound tests; trim comment --- Lib/test/test_zipfile/test_core.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 56f554c317650ae..2fdb20f86bb4523 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -4835,7 +4835,7 @@ def tearDown(self): class AbstractBoundedDecompressTests: # ZipExtFile._read1() bounds the output of each decompress() call so that a # small member declaring a large uncompressed size cannot expand into one - # unbounded read. DEFLATE was already bounded; check the other methods too. + # unbounded read. def test_read1_output_is_bounded(self): buf = io.BytesIO() with zipfile.ZipFile(buf, "w", compression=self.compression) as zf: @@ -4845,6 +4845,17 @@ def test_read1_output_is_bounded(self): self.assertLessEqual(len(f._read1(100)), f.MIN_READ_SIZE) +class StoredBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_STORED + + +@requires_zlib() +class DeflateBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + + @requires_bz2() class Bzip2BoundedDecompressTests(AbstractBoundedDecompressTests, unittest.TestCase): From da69f5d14ba116bac915267de66a901598979392 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 18 Aug 2026 13:54:09 +0200 Subject: [PATCH 5/7] Add a blurb --- .../Library/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst diff --git a/Misc/NEWS.d/next/Library/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst b/Misc/NEWS.d/next/Library/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst new file mode 100644 index 000000000000000..a0f03f052af9d56 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst @@ -0,0 +1,3 @@ +In :mod:`zipfile`, bound the amount of data that's decompressed at once +using bzip2, LZMA and Zstandard to avoid allocating excessive memory if the +archive declares a large uncompressed size. From 5f474f25d89fe4ed9c59979d85d0194fda0228bd Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 18 Aug 2026 17:15:46 +0200 Subject: [PATCH 6/7] Use previous blurb --- .../Library/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst | 7 ++++--- .../2026-06-22-09-45-00.GHSA-p384-rgv5-vhc2.zKb3Lm.rst | 4 ---- 2 files changed, 4 insertions(+), 7 deletions(-) delete mode 100644 Misc/NEWS.d/next/Security/2026-06-22-09-45-00.GHSA-p384-rgv5-vhc2.zKb3Lm.rst diff --git a/Misc/NEWS.d/next/Library/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst b/Misc/NEWS.d/next/Library/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst index a0f03f052af9d56..4e49ad5ce8fa00a 100644 --- a/Misc/NEWS.d/next/Library/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst +++ b/Misc/NEWS.d/next/Library/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst @@ -1,3 +1,4 @@ -In :mod:`zipfile`, bound the amount of data that's decompressed at once -using bzip2, LZMA and Zstandard to avoid allocating excessive memory if the -archive declares a large uncompressed size. +Bound the amount of data :mod:`zipfile` decompresses per read for members +compressed with bzip2, LZMA, or Zstandard, matching the existing limit for +deflate. A small archive member could previously expand into an unbounded +allocation even when read in small chunks. diff --git a/Misc/NEWS.d/next/Security/2026-06-22-09-45-00.GHSA-p384-rgv5-vhc2.zKb3Lm.rst b/Misc/NEWS.d/next/Security/2026-06-22-09-45-00.GHSA-p384-rgv5-vhc2.zKb3Lm.rst deleted file mode 100644 index 4e49ad5ce8fa00a..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-06-22-09-45-00.GHSA-p384-rgv5-vhc2.zKb3Lm.rst +++ /dev/null @@ -1,4 +0,0 @@ -Bound the amount of data :mod:`zipfile` decompresses per read for members -compressed with bzip2, LZMA, or Zstandard, matching the existing limit for -deflate. A small archive member could previously expand into an unbounded -allocation even when read in small chunks. From dc8e61b473c0a4c3399f76e24e2422242f389cca Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 20 Aug 2026 14:54:41 +0200 Subject: [PATCH 7/7] Move blurb to Security --- .../2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/{Library => Security}/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst (100%) diff --git a/Misc/NEWS.d/next/Library/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst b/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst similarity index 100% rename from Misc/NEWS.d/next/Library/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst rename to Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst