From a6badd1b8180edcfba12ec0fe7ccbe885d9e8d7a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 24 Jun 2026 11:23:45 +0200 Subject: [PATCH 1/2] [3.10.22] gh-151497: Avoid huge pre-allocation for oversized tarfile extended headers (GH-151498) (GH-151978) tarfile reads a member's extended header (a GNU long name/link or a pax header) with a single read sized by the header's size field: buf = tarfile.fileobj.read(self._block(self.size)) The size is taken from the archive and is not validated, so a ~512-byte crafted file can claim several gigabytes (or, via base-256 encoding, far more) and make read() pre-allocate that much memory -- on open/iterate, before any extraction filter runs. Read the extended-header data in bounded chunks instead, so an oversized or truncated header can no longer force a huge allocation. The bytes returned for valid archives are unchanged. (cherry picked from commit da99711d37dba3413af05207ea8b12cb06041c0f) Co-authored-by: Shardul Deshpande (cherry picked from commit b83961ade8bbd4b6d6a14a86b8324a3e4141362c) 3.10.22.x: the test_tarfile conflict was context only (test_extractfile_attrs, gh-74468, isn't on this line); only the ReadSizeRecorder and ExtendedHeaderMemoryTest classes are added. --- Lib/tarfile.py | 30 +++++++++++- Lib/test/test_tarfile.py | 48 +++++++++++++++++++ ...-06-15-15-32-36.gh-issue-151497.1cfmSV.rst | 4 ++ 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 41e91c664441782..13e135c9f8b38e7 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -261,6 +261,32 @@ def copyfileobj(src, dst, length=None, exception=OSError, bufsize=None): dst.write(buf) return +# Maximum number of bytes read in a single call when reading a member's +# extended header (a GNU long name/link or a pax header). The size of such +# a header is taken from the archive and is not trustworthy, so it is read in +# bounded chunks to avoid a huge up-front allocation when a crafted or +# truncated archive claims far more data than the file actually contains +# (gh-151497). +_EXTHEADER_READ_CHUNK = 1024 * 1024 # 1 MiB + +def _safe_read(fileobj, size): + """Read up to *size* bytes from *fileobj* in bounded chunks. + + Returns the same bytes as ``fileobj.read(size)`` would (including a short + result at end of file), but limits pre-allocation, so an + oversized size field in a crafted header cannot force a huge allocation. + """ + if size <= _EXTHEADER_READ_CHUNK: + return fileobj.read(size) + chunks = [] + while size > 0: + chunk = fileobj.read(min(size, _EXTHEADER_READ_CHUNK)) + if not chunk: + break + chunks.append(chunk) + size -= len(chunk) + return b"".join(chunks) + def _safe_print(s): encoding = getattr(sys.stdout, 'encoding', None) if encoding is not None: @@ -1401,7 +1427,7 @@ def _proc_gnulong(self, tarfile): """Process the blocks that hold a GNU longname or longlink member. """ - buf = tarfile.fileobj.read(self._block(self.size)) + buf = _safe_read(tarfile.fileobj, self._block(self.size)) # Fetch the next header and process it. try: @@ -1457,7 +1483,7 @@ def _proc_pax(self, tarfile): POSIX.1-2008. """ # Read the header information. - buf = tarfile.fileobj.read(self._block(self.size)) + buf = _safe_read(tarfile.fileobj, self._block(self.size)) # A pax header stores supplemental information for either # the following file (extended) or all following files diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 4779f9526365a69..b02d26fa885e47b 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -475,6 +475,54 @@ def test_length_zero_header(self): with tarfile.open(support.findfile('recursion.tar')) as tar: pass + +class ReadSizeRecorder(io.BytesIO): + # Records the largest size ever passed to read(), so a test can check + # that tarfile does not request far more data than the archive holds + # (which on a real file would pre-allocate it). + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.max_read_size = 0 + + def read(self, size=-1): + if size is not None and size >= 0: + self.max_read_size = max(self.max_read_size, size) + return super().read(size) + + +@support.cpython_only +class ExtendedHeaderMemoryTest(unittest.TestCase): + # gh-151497: the size of a GNU long name/link or a pax extended header is + # read from the archive and is untrusted. A crafted header can claim a + # size far larger than the file actually contains; opening such an archive + # must not try to read (and so pre-allocate) the claimed size in one go. + + def crafted_archive(self, hdrtype): + tarinfo = tarfile.TarInfo("A") + tarinfo.type = hdrtype + tarinfo.size = 0xFFFFFFFF # ~4 GiB claimed in a 512-byte header + return tarinfo.tobuf(format=tarfile.GNU_FORMAT) + + def check(self, hdrtype): + fobj = ReadSizeRecorder(self.crafted_archive(hdrtype)) + try: + with tarfile.open(fileobj=fobj, mode="r:") as tar: + tar.getmembers() + except tarfile.ReadError: + pass # a truncated header is fine; we only check the allocation + # The bogus ~4 GiB size must never reach a single read() call. + self.assertLessEqual(fobj.max_read_size, tarfile._EXTHEADER_READ_CHUNK) + + def test_gnu_longname_oversized_size(self): + self.check(tarfile.GNUTYPE_LONGNAME) + + def test_gnu_longlink_oversized_size(self): + self.check(tarfile.GNUTYPE_LONGLINK) + + def test_pax_header_oversized_size(self): + self.check(tarfile.XHDTYPE) + + class MiscReadTestBase(CommonReadTest): def requires_name_attribute(self): pass diff --git a/Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst b/Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst new file mode 100644 index 000000000000000..a4c03c9d71d7618 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst @@ -0,0 +1,4 @@ +Opening a :mod:`tarfile` archive no longer attempts to pre-allocate a huge +buffer when a crafted or truncated member claims an oversized extended header +(a GNU long name/link or a pax header). The extended header is now read in +bounded chunks, so its size field can no longer trigger memory exhaustion. From 1f2382edd2fa58dbf7c5fa577283d95211c2f87d Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Thu, 29 Oct 2026 15:14:10 -0500 Subject: [PATCH 2/2] 3.10.22.x: rename XML_SetHashSalt16Bytes in pyexpatns.h pyexpatns.h prefixes the bundled Expat's exported symbols with PyExpat_ so they cannot clash with a system libexpat loaded in the same process. Expat 2.9.0 added XML_SetHashSalt16Bytes, which pyexpat and _elementtree call, but the header did not rename it (upstream main does not either). The AllocTracker functions are already renamed on this line. Same change as 3.7.17.x 69ab1c85f13 (ActiveState/cpython#117 review). Co-Authored-By: Claude Opus 5.5 --- Modules/expat/pyexpatns.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h index 4c550b7d0131819..8b149b9046e2ecb 100644 --- a/Modules/expat/pyexpatns.h +++ b/Modules/expat/pyexpatns.h @@ -113,6 +113,7 @@ #define XML_SetExternalEntityRefHandler PyExpat_XML_SetExternalEntityRefHandler #define XML_SetExternalEntityRefHandlerArg PyExpat_XML_SetExternalEntityRefHandlerArg #define XML_SetHashSalt PyExpat_XML_SetHashSalt +#define XML_SetHashSalt16Bytes PyExpat_XML_SetHashSalt16Bytes #define XML_SetNamespaceDeclHandler PyExpat_XML_SetNamespaceDeclHandler #define XML_SetNotationDeclHandler PyExpat_XML_SetNotationDeclHandler #define XML_SetNotStandaloneHandler PyExpat_XML_SetNotStandaloneHandler