From 8dc82da9e82d8b888b7f1da52a275d0f44e5dd98 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 17 Apr 2026 13:43:20 +0200 Subject: [PATCH 01/73] Fix reported memory errors in asan environment by using the debugging versions of python --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 256ee95e..c01b469a 100644 --- a/tox.ini +++ b/tox.ini @@ -23,10 +23,12 @@ commands = coverage xml -i [testenv:asan] +# Use debug python as default python gives address sanitization errors. +basepython=python3d setenv= PYTHONDEVMODE=1 PYTHONMALLOC=malloc - CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer + CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer -Og -g allowlist_externals=bash commands= bash -c 'export LD_PRELOAD=$(gcc -print-file-name=libasan.so) && printenv LD_PRELOAD && python -c "from isal import isal_zlib" && pytest tests' From 093cb819b746333f75542479fd1e6c63e4cb418a Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 17 Apr 2026 13:51:58 +0200 Subject: [PATCH 02/73] Update macos labels --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 955ca769..66a1b3d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,7 @@ jobs: - "pypy-3.11" os: ["ubuntu-latest"] include: - - os: "macos-13" + - os: "macos-15-intel" python-version: "3.10" - os: "macos-latest" python-version: "3.10" @@ -136,7 +136,7 @@ jobs: shell: bash -l {0} strategy: matrix: - os: ["ubuntu-latest", "macos-13", "windows-latest"] + os: ["ubuntu-latest", "macos-15-intel", "windows-latest"] python_version: [ "python" ] include: - os: "ubuntu-latest" @@ -172,8 +172,8 @@ jobs: matrix: os: - ubuntu-latest - - macos-13 - - macos-14 + - macos-15-intel + - macos-latest - windows-latest cibw_archs_linux: ["x86_64"] cibw_before_all_linux: From d99d49b1441aed10c3fddebb813e583f8a85b258 Mon Sep 17 00:00:00 2001 From: Matt Stone Date: Wed, 20 May 2026 15:54:34 -0400 Subject: [PATCH 03/73] Re-enable PyPy wheel builds --- .github/workflows/ci.yml | 1 + CHANGELOG.rst | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 955ca769..0425bb7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -216,6 +216,7 @@ jobs: - name: Build wheels run: cibuildwheel --output-dir dist env: + CIBW_ENABLE: "pypy" CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_*arm64 cp39-macosx_*arm64" # Skip 32 bit and problematic mac builds. CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }} diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 36e20a2d..b8471793 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,10 @@ Changelog .. that users understand how the changes affect the new version. +version 1.8.1-dev +----------------- ++ Restore PyPy wheel builds. + version 1.8.0 ----------------- + Python 3.14 is supported. From dff90d1e1986cc1f6b7deedc28915fa24de3d41a Mon Sep 17 00:00:00 2001 From: Vladislav Glinkin Date: Tue, 29 Sep 2026 17:27:32 +0300 Subject: [PATCH 04/73] Fix zlib headers for preset dictionary compression --- CHANGELOG.rst | 2 + src/isal/isal_zlibmodule.c | 47 +++++++++++++++++++++- tests/test_dictionary_header.py | 70 +++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 tests/test_dictionary_header.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b8471793..491073a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,8 @@ Changelog version 1.8.1-dev ----------------- + Restore PyPy wheel builds. ++ Fix zlib streams compressed with a preset dictionary so they can be decoded + by standard zlib, including builds linked against older ISA-L releases. version 1.8.0 ----------------- diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 3a280ec8..06e72988 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -570,6 +570,7 @@ typedef struct PyObject_HEAD uint8_t *level_buf; PyObject *zdict; + struct isal_zlib_header zlib_header; int is_initialised; PyThread_type_lock lock; // isal_zstream should be at the bottom as it contains buffers inside the struct. @@ -595,6 +596,7 @@ newcompobject() return NULL; self->is_initialised = 0; self->zdict = NULL; + self->zlib_header.dict_flag = 0; self->level_buf = NULL; self->lock = PyThread_allocate_lock(); if (self->lock == NULL) { @@ -755,8 +757,18 @@ isal_zlib_compressobj_impl(PyObject *module, int level, int method, int wbits, } else { err = isal_deflate_set_dict(&(self->zst), zdict->buf, (uint32_t)zdict->len); - if (err == COMP_OK) + if (err == COMP_OK) { + if (flag == IGZIP_ZLIB && zdict->len != 0) { + isal_zlib_header_init(&self->zlib_header); + self->zlib_header.info = hist_bits - 8; + self->zlib_header.level = level == 0 ? 0 : 1; + self->zlib_header.dict_flag = 1; + self->zlib_header.dict_id = isal_adler32( + 1, zdict->buf, (uint32_t)zdict->len); + self->zst.gzip_flag = IGZIP_ZLIB_NO_HDR; + } goto success; + } PyErr_SetString(PyExc_ValueError, "Invalid dictionary"); goto error; } @@ -817,6 +829,31 @@ isal_zlib_decompressobj_impl(PyObject *module, int wbits, PyObject *zdict) return (PyObject *)self; } +/* Called with the compressor locked and a newly arranged output buffer. + * The six-byte dictionary header is written once, including when flush() + * is the first operation. ISA-L still generates the Adler-32 trailer. */ +static int +write_pending_zlib_header(compobject *self) +{ + if (self->zlib_header.dict_flag) { + uint8_t *header = self->zst.next_out; + if (isal_write_zlib_header(&self->zst, &self->zlib_header) != 0) { + PyErr_SetString(PyExc_RuntimeError, + "Insufficient space for the zlib dictionary header"); + return -1; + } + /* Older ISA-L releases write DICTID in little-endian order. + * RFC 1950 requires network byte order, regardless of ISA-L version. */ + uint32_t dict_id = self->zlib_header.dict_id; + header[2] = (uint8_t)(dict_id >> 24); + header[3] = (uint8_t)(dict_id >> 16); + header[4] = (uint8_t)(dict_id >> 8); + header[5] = (uint8_t)dict_id; + self->zlib_header.dict_flag = 0; + } + return 0; +} + static PyObject * isal_zlib_Compress_compress_impl(compobject *self, Py_buffer *data) /*[clinic end generated code: output=5d5cd791cbc6a7f4 input=0d95908d6e64fab8]*/ @@ -838,6 +875,9 @@ isal_zlib_Compress_compress_impl(compobject *self, Py_buffer *data) if (obuflen < 0) goto error; + if (write_pending_zlib_header(self) < 0) + goto error; + Py_BEGIN_ALLOW_THREADS err = isal_deflate(&self->zst); Py_END_ALLOW_THREADS @@ -1049,6 +1089,11 @@ isal_zlib_Compress_flush_impl(compobject *self, int mode) goto error; } + if (write_pending_zlib_header(self) < 0) { + Py_CLEAR(RetVal); + goto error; + } + Py_BEGIN_ALLOW_THREADS err = isal_deflate(&self->zst); Py_END_ALLOW_THREADS diff --git a/tests/test_dictionary_header.py b/tests/test_dictionary_header.py new file mode 100644 index 00000000..028b88c7 --- /dev/null +++ b/tests/test_dictionary_header.py @@ -0,0 +1,70 @@ +"""Cross-library checks for dictionary headers and compressor lifecycle.""" +import random +import zlib + +from isal import isal_zlib + +import pytest + + +@pytest.mark.parametrize('level', range(4)) +@pytest.mark.parametrize('wbits', [9, 15, -15]) +@pytest.mark.parametrize('dictionary_size', [11, 32768, 40000]) +@pytest.mark.parametrize( + 'first_operation', ['compress', 'empty', 'sync', 'finish']) +def test_dictionary_cross_decode(level, wbits, dictionary_size, + first_operation): + dictionary = random.Random(21).randbytes(dictionary_size) + data = dictionary[-256:] * 5 + random.Random(22).randbytes(4096) + compressor = isal_zlib.compressobj(level, wbits=wbits, zdict=dictionary) + packed = b'' + if first_operation == 'empty': + packed += compressor.compress(b'') + elif first_operation == 'sync': + packed += compressor.flush(isal_zlib.Z_SYNC_FLUSH) + elif first_operation == 'finish': + data = b'' + for pos in range(0, len(data), 17): + packed += compressor.compress(data[pos:pos+17]) + packed += compressor.flush() + if wbits > 0: + assert packed[0] >> 4 == wbits - 8 + assert packed[1] & 32 + assert int.from_bytes(packed[2:6], 'big') == zlib.adler32(dictionary) + assert int.from_bytes(packed[:2], 'big') % 31 == 0 + decompressor = zlib.decompressobj(wbits, zdict=dictionary) + assert decompressor.decompress(packed) + decompressor.flush() == data + assert decompressor.eof + assert decompressor.unused_data == b'' + + +@pytest.mark.parametrize('level', range(4)) +@pytest.mark.parametrize( + 'flush_mode', [isal_zlib.Z_SYNC_FLUSH, isal_zlib.Z_FULL_FLUSH]) +def test_dictionary_multiple_flushes(level, flush_mode): + dictionary = b'hello world' + compressor = isal_zlib.compressobj(level, zdict=dictionary) + data = dictionary * 1000 + packed = compressor.compress(data[:5500]) + compressor.flush(flush_mode) + packed += compressor.compress(data[5500:]) + compressor.flush() + for module in (zlib, isal_zlib): + decompressor = module.decompressobj(zdict=dictionary) + assert decompressor.decompress(packed) + decompressor.flush() == data + + +@pytest.mark.parametrize('dictionary', [None, b'']) +def test_no_dictionary_header(dictionary): + kwargs = {} if dictionary is None else {'zdict': dictionary} + compressor = isal_zlib.compressobj(**kwargs) + data = b'hello world' * 100 + packed = compressor.compress(data) + compressor.flush() + assert not packed[1] & 32 + assert zlib.decompress(packed) == data + + +def test_wrong_dictionary_rejected_by_zlib(): + compressor = isal_zlib.compressobj(zdict=b'hello world') + packed = compressor.compress(b'hello world' * 10) + compressor.flush() + decompressor = zlib.decompressobj(zdict=b'wrong dictionary') + with pytest.raises(zlib.error): + decompressor.decompress(packed) From 1178365e6ffa5ddac3a50bfa2ab0e6d387b122d5 Mon Sep 17 00:00:00 2001 From: Vladislav Glinkin Date: Tue, 29 Sep 2026 17:27:32 +0300 Subject: [PATCH 05/73] Discard compressed input buffer on backward gzip seek --- CHANGELOG.rst | 2 ++ src/isal/isal_zlibmodule.c | 4 ++++ tests/test_seek_buffer_reset.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 tests/test_seek_buffer_reset.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b8471793..37edd493 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,8 @@ Changelog version 1.8.1-dev ----------------- + Restore PyPy wheel builds. ++ Fix backward seeking in igzip files by discarding buffered compressed data + when rewinding the underlying file. version 1.8.0 ----------------- diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 3a280ec8..fe779176 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1918,6 +1918,10 @@ GzipReader_seek(GzipReader *self, PyObject *args, PyObject *kwargs) if (seek_result == NULL) { return NULL; } + Py_DECREF(seek_result); + /* Discard compressed data buffered before rewinding the file. */ + self->current_pos = self->input_buffer; + self->buffer_end = self->input_buffer; self->stream_phase = GzipReader_HEADER; self->_pos = 0; self->all_bytes_read = 0; diff --git a/tests/test_seek_buffer_reset.py b/tests/test_seek_buffer_reset.py new file mode 100644 index 00000000..023ec86f --- /dev/null +++ b/tests/test_seek_buffer_reset.py @@ -0,0 +1,30 @@ +"""Force backward seeks past the outer buffer and through the raw reader.""" +import gzip +import io +import random + +from isal import igzip + +import pytest + + +@pytest.mark.parametrize('writer', [gzip.compress, igzip.compress]) +@pytest.mark.parametrize('random_data', [False, True]) +def test_backward_seek_discards_compressed_buffer(writer, random_data): + data = (random.Random(42).randbytes(1048576) if random_data + else b'hello world' * 100000) + with igzip.open(io.BytesIO(writer(data)), 'rb') as reader: + for offset in (12345, 0, 750000, 10, 32768, 0): + assert reader.seek(offset) == offset + assert reader.read(400) == data[offset:offset+400] + reader.seek(0) + assert reader.read() == data + + +@pytest.mark.parametrize('size', [110, 8192, 32768]) +def test_raw_reader_backward_seek(size): + data = random.Random(42).randbytes(size) + reader = igzip._GzipReader(io.BytesIO(gzip.compress(data))) + assert reader.read(1) == data[:1] + assert reader.seek(0) == 0 + assert reader.readall() == data From f19767ab886d802559f11497c9c8b20d6d8c3150 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 13 Oct 2026 09:54:11 +0200 Subject: [PATCH 06/73] Update python versions (#259) * update python versions * Update version of setup python * Check is resize size == 0 to prevent resizing the empty string object * Fix workflow file error * Revert " Check is resize size == 0 to prevent resizing the empty string object" This reverts commit 246a3075bc88f6a7a2129821a309b3cb121672a4. * Prevent empty string object from being resized * Use newer miniconda version * Pin python version for dynamic builds * Stop testing pypy versions for dynamic builds. Conda-forge does not support PyPy anymore: https://conda-forge.org/news/2024/08/14/sunsetting-pypy/ --- .github/workflows/ci.yml | 34 ++++++++++++++++------------------ CHANGELOG.rst | 2 ++ pyproject.toml | 2 +- src/isal/igzip_libmodule.c | 6 ++++-- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4320c656..e82cebfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,10 +16,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.10 - uses: actions/setup-python@v5 + - name: Set up Python 3.11 + uses: actions/setup-python@v7 with: - python-version: "3.10" + python-version: "3.11" - name: Install tox run: pip install tox - name: Lint @@ -37,9 +37,9 @@ jobs: with: submodules: recursive - name: Set up Python 3.10 - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: - python-version: "3.10" + python-version: "3.11" - name: Install isal run: sudo apt-get install libisal-dev - name: Install tox and upgrade setuptools and pip @@ -54,28 +54,29 @@ jobs: strategy: matrix: python-version: - - "3.10" - "3.11" - "3.12" - "3.13" - "3.14" - "3.14t" - - "pypy-3.10" + - "3.15" + - "3.15t" - "pypy-3.11" + - "pypy-3.12" os: ["ubuntu-latest"] include: - os: "macos-15-intel" - python-version: "3.10" + python-version: "3.11" - os: "macos-latest" - python-version: "3.10" + python-version: "3.11" - os: "windows-latest" - python-version: "3.10" + python-version: "3.11" steps: - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: true @@ -106,7 +107,7 @@ jobs: strategy: matrix: python_version: - - "3.10" + - "3.11" steps: - uses: actions/checkout@v4 with: @@ -137,16 +138,13 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "macos-15-intel", "windows-latest"] - python_version: [ "python" ] - include: - - os: "ubuntu-latest" - python_version: "pypy" + python_version: [ "python==3.11" ] steps: - uses: actions/checkout@v4 with: submodules: recursive - name: Install miniconda. - uses: conda-incubator/setup-miniconda@v3 # https://github.com/conda-incubator/setup-miniconda. + uses: conda-incubator/setup-miniconda@v4 # https://github.com/conda-incubator/setup-miniconda. with: channels: conda-forge,defaults - name: Install requirements (universal) @@ -195,7 +193,7 @@ jobs: with: submodules: recursive fetch-depth: 0 # Fetch everything to get accurately versioned tag. - - uses: actions/setup-python@v2 # Some issues where caused by higher versions. + - uses: actions/setup-python@v7 name: Install Python - name: Install cibuildwheel twine wheel run: python -m pip install cibuildwheel twine wheel diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2f4d3bb0..bcd8c197 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,8 @@ Changelog version 1.8.1-dev ----------------- ++ Python 3.10 and PyPy 3.10 are no longer supported. Added support for PyPy + 3.12 and Python 3.15. + Restore PyPy wheel builds. + Fix zlib streams compressed with a preset dictionary so they can be decoded by standard zlib, including builds linked against older ISA-L releases. diff --git a/pyproject.toml b/pyproject.toml index c1cbce68..d1bbfb46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,11 +17,11 @@ requires-python = ">=3.9" # Because of setuptools version classifiers = [ "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index d498145f..352de9f5 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -161,8 +161,10 @@ decompress_buf(IgzipDecompressor *self, Py_ssize_t max_length) self->avail_in_real += self->state.avail_in; - if (_PyBytes_Resize(&RetVal, self->state.next_out - - (uint8_t *)PyBytes_AS_STRING(RetVal)) != 0) + /* Prevent empty string object from being resized. */ + if (PyBytes_GET_SIZE(RetVal) != 0 && + _PyBytes_Resize(&RetVal, self->state.next_out - + (uint8_t *)PyBytes_AS_STRING(RetVal)) != 0) goto error; return RetVal; From 262b296c3b56cc4ea90765d38a49b1ab5d34379a Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 13 Oct 2026 09:58:39 +0200 Subject: [PATCH 07/73] Update ISA-L to 2.32.1 --- CHANGELOG.rst | 1 + src/isal/isa-l | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bcd8c197..63e109c6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,7 @@ Changelog version 1.8.1-dev ----------------- ++ Updated ISA-L to 2.32.1. + Python 3.10 and PyPy 3.10 are no longer supported. Added support for PyPy 3.12 and Python 3.15. + Restore PyPy wheel builds. diff --git a/src/isal/isa-l b/src/isal/isa-l index c387163f..7c3479e0 160000 --- a/src/isal/isa-l +++ b/src/isal/isa-l @@ -1 +1 @@ -Subproject commit c387163fcbca62701d646149564c550c78a4f985 +Subproject commit 7c3479e0a9dac17f448603ec1ad64c7c625f530c From 31798c5f9da6f536fe8c9234bfe9ecc21868ec19 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 13 Oct 2026 10:31:04 +0200 Subject: [PATCH 08/73] Create test for adverserial input --- tests/test_igzip.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_igzip.py b/tests/test_igzip.py index 7b32b363..9ce57bbc 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -411,7 +411,7 @@ def test_very_long_header_in_data(): def test_very_long_header_in_file(): # header with a very long filename. header = (b"\x1f\x8b\x08\x08\x00\x00\x00\x00\x00\xff" + - igzip.READ_BUFFER_SIZE * 2 * b"A" + + (igzip.READ_BUFFER_SIZE - 11) * b"A" + b"\x00") compressed = header + isal_zlib.compress(b"", 3, -15) + 8 * b"\00" f = io.BytesIO(compressed) @@ -473,3 +473,26 @@ def test_bgzip(): with igzip.open(gzip_file, "rb") as gz: gz_data = gz.read() assert bgz_data == gz_data + + +def test_adverserial_header(): + magic = b"\x1f\x8b" + method = 8 + mtime = 0 + flags = FCOMMENT + os = 255 + xfl = 0 + header = struct.pack("<2sBBIBB", magic, method, flags, mtime, xfl, os) + comment = b"A" * 10 * 1024 * 1024 + b"\00" + compressed = isal_zlib.compress(b"", wbits = -15) + crc = isal_zlib.crc32(b"") + length = 0 + trailer = struct.pack(" Date: Tue, 13 Oct 2026 10:44:43 +0200 Subject: [PATCH 09/73] Throw an overflow on big headers --- src/isal/isal_zlibmodule.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 25224161..d2cef656 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1629,14 +1629,12 @@ GzipReader_read_from_file(GzipReader *self) /* Buffer is full but a new read request was issued. This will be due to the header being bigger than the header. Enlarge the buffer to accommodate the hzip header. */ - size_t new_buffer_size = self->buffer_size * 2; - uint8_t *tmp_buffer = PyMem_Realloc(self->input_buffer, new_buffer_size); - if (tmp_buffer == NULL) { - PyErr_NoMemory(); - return -1; - } - self->input_buffer = tmp_buffer; - self->buffer_size = new_buffer_size; + PyErr_Format( + PyExc_OverflowError, + "Gzip header exceeds input buffer size of %lu", + self->buffer_size + ); + return -1; } else if (remaining > 0) { memmove(self->input_buffer, current_pos, remaining); } From 93a14abffa5daa258a716f74f1119eaaf0ee6409 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 13 Oct 2026 14:06:09 +0200 Subject: [PATCH 10/73] Fix permission error in test --- tests/test_igzip.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_igzip.py b/tests/test_igzip.py index 9ce57bbc..c0661197 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -480,19 +480,20 @@ def test_adverserial_header(): method = 8 mtime = 0 flags = FCOMMENT - os = 255 + _os = 255 xfl = 0 - header = struct.pack("<2sBBIBB", magic, method, flags, mtime, xfl, os) + header = struct.pack("<2sBBIBB", magic, method, flags, mtime, xfl, _os) comment = b"A" * 10 * 1024 * 1024 + b"\00" - compressed = isal_zlib.compress(b"", wbits = -15) + compressed = isal_zlib.compress(b"", wbits=-15) crc = isal_zlib.crc32(b"") length = 0 trailer = struct.pack(" Date: Tue, 13 Oct 2026 14:15:06 +0200 Subject: [PATCH 11/73] Update comment --- src/isal/isal_zlibmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index d2cef656..ea9457fd 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1626,9 +1626,9 @@ GzipReader_read_from_file(GzipReader *self) uint8_t *buffer_end = self->buffer_end; size_t remaining = buffer_end - current_pos; if (remaining == self->buffer_size) { - /* Buffer is full but a new read request was issued. This will be due - to the header being bigger than the header. Enlarge the buffer - to accommodate the hzip header. */ + /* Buffer is full but a new read request was issued. This will be due + to the header being bigger than the buffer. To prevent adverserial + inputs with infinite header sizes an error is thrown here. */ PyErr_Format( PyExc_OverflowError, "Gzip header exceeds input buffer size of %lu", From 3d561a5d4bbbeddc230228839d849d8026c26203 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 13 Oct 2026 14:41:31 +0200 Subject: [PATCH 12/73] Update changelog --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 63e109c6..4b200d3c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,10 @@ version 1.8.1-dev by standard zlib, including builds linked against older ISA-L releases. + Fix backward seeking in igzip files by discarding buffered compressed data when rewinding the underlying file. ++ Extraordinarily large gzip headers with either a very long comment or + filename will now cause an ``OverflowError`` rather than use more and more + memory. The size is limited to the internal buffer size which is currently + set at 512 KiB. version 1.8.0 ----------------- From 8c35b7d90136fd742fbfabd6d7ea1a010a83fcf0 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Thu, 15 Oct 2026 13:18:14 -0400 Subject: [PATCH 13/73] Fix a crash when _GzipReader gets an object that is not bytes-like GzipReader_dealloc used members that were not initialized yet when the buffer protocol failed. igzip.decompress("x") crashed with a segmentation fault. --- src/isal/isal_zlibmodule.c | 22 +++++++++++++++++----- tests/test_igzip.py | 12 ++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index ea9457fd..54a1b288 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1533,7 +1533,9 @@ static void GzipReader_dealloc(GzipReader *self) PyMem_Free(self->memview); } Py_XDECREF(self->fp); - PyThread_free_lock(self->lock); + if (self->lock != NULL) { + PyThread_free_lock(self->lock); + } Py_TYPE(self)->tp_free(self); } @@ -1574,8 +1576,15 @@ GzipReader__new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) return NULL; } GzipReader *self = PyObject_New(GzipReader, type); + if (self == NULL) { + return NULL; + } + // GzipReader_dealloc uses these members when initialization fails + self->input_buffer = NULL; + self->memview = NULL; + self->fp = NULL; + self->lock = NULL; if (PyObject_HasAttrString(fp, "read")) { - self->memview = NULL; self->buffer_size = buffer_size; self->input_buffer = PyMem_Malloc(self->buffer_size); if (self->input_buffer == NULL) { @@ -1585,14 +1594,17 @@ GzipReader__new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) self->buffer_end = self->input_buffer; self->all_bytes_read = 0; } else { - self->memview = PyMem_Malloc(sizeof(Py_buffer)); - if (self->memview == NULL) { + Py_buffer *memview = PyMem_Malloc(sizeof(Py_buffer)); + if (memview == NULL) { + Py_DECREF(self); return PyErr_NoMemory(); } - if (PyObject_GetBuffer(fp, self->memview, PyBUF_SIMPLE) < 0) { + if (PyObject_GetBuffer(fp, memview, PyBUF_SIMPLE) < 0) { + PyMem_Free(memview); Py_DECREF(self); return NULL; } + self->memview = memview; self->buffer_size = self->memview->len; self->input_buffer = self->memview->buf; self->buffer_end = self->input_buffer + self->buffer_size; diff --git a/tests/test_igzip.py b/tests/test_igzip.py index c0661197..650cd172 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -328,6 +328,18 @@ def test_decompress_empty(): assert igzip.decompress(b"") == b"" +@pytest.mark.parametrize("data", ["x", 1, None]) +def test_decompress_not_bytes_like(data): + with pytest.raises(TypeError) as error: + igzip.decompress(data) + error.match("a bytes-like object is required") + + +def test_decompress_not_contiguous(): + with pytest.raises(BufferError): + igzip.decompress(memoryview(b"abcd")[::2]) + + def headers(): magic = b"\x1f\x8b" method = b"\x08" From c79b10bcd4935136c4adaa03be063ab8d2d4d451 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Thu, 15 Oct 2026 13:18:50 -0400 Subject: [PATCH 14/73] Keep a reference to the zdict of IgzipDecompressor IgzipDecompressor stored zdict without increasing its reference count, but released it on deallocation. Creating a few decompressors with a dictionary crashed the interpreter. --- src/isal/igzip_libmodule.c | 1 + tests/test_igzip_lib.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index 352de9f5..0e300855 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -463,6 +463,7 @@ igzip_lib_IgzipDecompressor__new__(PyTypeObject *type, self->avail_in_real = 0; self->input_buffer = NULL; self->input_buffer_size = 0; + Py_XINCREF(zdict); self->zdict = zdict; self->unused_data = PyBytes_FromStringAndSize(NULL, 0); if (self->unused_data == NULL) { diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index edfe8bb5..95c451fb 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -257,6 +257,17 @@ def test_igzip_decompressor_raw_deflate_unused_data_gzip(test_offset): assert igzd.unused_data == true_unused_data +@pytest.mark.skipif(not hasattr(sys, "getrefcount"), + reason="Needs reference counting") +def test_igzip_decompressor_keeps_zdict_reference(): + zdict = bytes(range(256)) + refcount = sys.getrefcount(zdict) + decompressor = IgzipDecompressor(flag=DECOMP_ZLIB, zdict=zdict) + assert sys.getrefcount(zdict) == refcount + 1 + del decompressor + assert sys.getrefcount(zdict) == refcount + + @pytest.mark.parametrize(["unused_size", "flag_pair", "data_size"], itertools.product([26], FLAGS, [128 * 1024 - 3, 874, 81923, 9111])) From aeb5daea6183fd352acda8deba77d20a2730d3bf Mon Sep 17 00:00:00 2001 From: ptweezy Date: Thu, 15 Oct 2026 13:19:35 -0400 Subject: [PATCH 15/73] Release the lock when Compress.flush gets an unsupported mode The error path returned without releasing the lock, so the next call on the same object hung forever. --- src/isal/isal_zlibmodule.c | 1 + tests/test_compat.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 54a1b288..155bef69 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1076,6 +1076,7 @@ isal_zlib_Compress_flush_impl(compobject *self, int mode) } else { PyErr_Format(IsalError, "Unsupported flush mode: %d", mode); + LEAVE_ZLIB(self); return NULL; } diff --git a/tests/test_compat.py b/tests/test_compat.py index 0ea54f70..82536f2b 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -11,6 +11,7 @@ import gzip import itertools +import threading import zlib from pathlib import Path @@ -100,6 +101,21 @@ def test_compress_compressobj(data_size, level, wbits, memLevel): assert data == decompressed +def test_compressobj_flush_unsupported_mode(): + compressor = isal_zlib.compressobj() + with pytest.raises(isal_zlib.error) as error: + compressor.flush(isal_zlib.Z_PARTIAL_FLUSH) + error.match("Unsupported flush mode") + # The next call hangs if the failed flush kept the lock + result = [] + thread = threading.Thread(target=lambda: result.append( + compressor.compress(b"data") + compressor.flush()), daemon=True) + thread.start() + thread.join(timeout=10) + assert result + assert zlib.decompress(result[0]) == b"data" + + @pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel"], itertools.product([128 * 1024], range(4), WBITS_RANGE, range(1, 10))) From f57d39bea946540e283570d1f3b8f68c2e3b2445 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Thu, 15 Oct 2026 13:19:52 -0400 Subject: [PATCH 16/73] Fix a double free when a level 0 Compress is finished twice flush(Z_FINISH) freed the level buffer but kept the pointer. Level 0 accepts a second Z_FINISH flush, which freed the buffer again. --- src/isal/isal_zlibmodule.c | 1 + tests/test_compat.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 155bef69..3679ceb2 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1112,6 +1112,7 @@ isal_zlib_Compress_flush_impl(compobject *self, int mode) mode is Z_FINISH, but checking both for safety*/ if (self->zst.internal_state.state == ZSTATE_END && mode == Z_FINISH) { PyMem_FREE(self->level_buf); + self->level_buf = NULL; self->zst.level_buf_size = 0; self->zst.level_buf = NULL; self->is_initialised = 0; diff --git a/tests/test_compat.py b/tests/test_compat.py index 82536f2b..2034c7ed 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -116,6 +116,13 @@ def test_compressobj_flush_unsupported_mode(): assert zlib.decompress(result[0]) == b"data" +def test_compressobj_level_0_flush_after_finish(): + compressor = isal_zlib.compressobj(0) + compressed = compressor.compress(b"data") + compressor.flush() + assert compressor.flush() == b"" + assert zlib.decompress(compressed) == b"data" + + @pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel"], itertools.product([128 * 1024], range(4), WBITS_RANGE, range(1, 10))) From e88a3fe7abd2db4fd1dfc98abdddd89d8c22224c Mon Sep 17 00:00:00 2001 From: ptweezy Date: Thu, 15 Oct 2026 13:23:22 -0400 Subject: [PATCH 17/73] Fix unused_data when max_length splits the end of a stream When a stream ends, ISA-L moves next_in back over the unused bytes in its bit buffer. If an earlier call read those bytes, next_in points before the current input, and unused_data got bytes from outside the buffer. Save the bit buffer after each call and take those bytes from it. Also clear unconsumed_tail at the end of the stream, as CPython does since gh-158169. --- src/isal/igzip_libmodule.c | 18 ++++++++++++++++-- src/isal/isal_shared.h | 34 ++++++++++++++++++++++++++++++++++ src/isal/isal_zlibmodule.c | 21 ++++++++++++++++++--- tests/test_compat.py | 15 +++++++++++++++ tests/test_igzip_lib.py | 25 +++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 5 deletions(-) diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index 0e300855..8702aa0a 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -34,6 +34,8 @@ typedef struct { Py_ssize_t avail_in_real; char eof; /* T_BOOL expects a char */ char needs_input; + uint8_t saved_bitbuffer[8]; + Py_ssize_t saved_bitbuffer_size; /* Struct inflate state contains a massive buffer at the end. Put it at the end of the IgzipDecompressor so members can be accessed easily. */ struct inflate_state state; @@ -226,6 +228,7 @@ decompress(IgzipDecompressor *self, uint8_t *data, size_t len, Py_ssize_t max_le input_buffer_in_use = 0; } + uint8_t *input_start = self->state.next_in; result = decompress_buf(self, max_length); if(result == NULL) { self->state.next_in = NULL; @@ -242,8 +245,14 @@ decompress(IgzipDecompressor *self, uint8_t *data, size_t len, Py_ssize_t max_le goto error; char *new_data_ptr = PyBytes_AS_STRING(new_data); bitbuffer_copy(&(self->state), new_data_ptr, bytes_in_bitbuffer); - memcpy(new_data_ptr + bytes_in_bitbuffer, self->state.next_in, - self->avail_in_real); + if (copy_from_next_in(&(self->state), input_start, + self->saved_bitbuffer, + self->saved_bitbuffer_size, + new_data_ptr + bytes_in_bitbuffer, + self->avail_in_real) < 0) { + Py_DECREF(new_data); + goto error; + } Py_XSETREF(self->unused_data, new_data); } } @@ -282,6 +291,10 @@ decompress(IgzipDecompressor *self, uint8_t *data, size_t len, Py_ssize_t max_le self->state.next_in = self->input_buffer; } } + if (!self->eof) { + save_bitbuffer(&(self->state), self->saved_bitbuffer, + &self->saved_bitbuffer_size); + } return result; error: @@ -460,6 +473,7 @@ igzip_lib_IgzipDecompressor__new__(PyTypeObject *type, IgzipDecompressor *self = PyObject_New(IgzipDecompressor, type); self->eof = 0; self->needs_input = 1; + self->saved_bitbuffer_size = 0; self->avail_in_real = 0; self->input_buffer = NULL; self->input_buffer_size = 0; diff --git a/src/isal/isal_shared.h b/src/isal/isal_shared.h index 61610276..568e9e3e 100644 --- a/src/isal/isal_shared.h +++ b/src/isal/isal_shared.h @@ -219,6 +219,40 @@ static int bitbuffer_copy(struct inflate_state *state, char *to, size_t n){ return 0; } +/* When a stream ends, isal_inflate moves next_in back over the unused bytes + in the bit buffer. If a previous call read those bytes, next_in points + before the current input. This keeps a copy of them after each call. */ +static inline void +save_bitbuffer(struct inflate_state *state, uint8_t *saved, + Py_ssize_t *saved_size) +{ + *saved_size = bitbuffer_size(state); + bitbuffer_copy(state, (char *)saved, *saved_size); +} + +/* Copy size bytes starting at state->next_in. Bytes before input_start come + from the saved bit buffer. */ +static int +copy_from_next_in(struct inflate_state *state, uint8_t *input_start, + uint8_t *saved, Py_ssize_t saved_size, + char *to, Py_ssize_t size) +{ + uint8_t *from = state->next_in; + Py_ssize_t before_input = 0; + if ((uintptr_t)from < (uintptr_t)input_start) { + before_input = (Py_ssize_t)((uintptr_t)input_start - (uintptr_t)from); + if (before_input > saved_size || before_input > size) { + PyErr_SetString(PyExc_RuntimeError, + "Leftover input before the buffer is unavailable"); + return -1; + } + memcpy(to, saved + saved_size - before_input, before_input); + from = input_start; + } + memcpy(to + before_input, from, size - before_input); + return 0; +} + static inline void arrange_input_buffer(uint32_t *avail_in, Py_ssize_t *remains) { diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 3679ceb2..c166c9da 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -617,6 +617,8 @@ typedef struct int is_initialised; int method_set; char eof; + uint8_t saved_bitbuffer[8]; + Py_ssize_t saved_bitbuffer_size; PyThread_type_lock lock; // inflate_state should be at the bottom as it contains buffers inside the struct. struct inflate_state zst; @@ -665,6 +667,7 @@ newdecompobject() if (self == NULL) return NULL; self->eof = 0; + self->saved_bitbuffer_size = 0; self->is_initialised = 0; self->method_set = 0; self->zdict = NULL; @@ -915,7 +918,8 @@ save_unconsumed_input(decompobject *self, Py_buffer *data, int err) Py_ssize_t new_size, left_size; PyObject *new_data; Py_ssize_t bytes_in_bitbuffer = bitbuffer_size(&(self->zst)); - left_size = (uint8_t *)data->buf + data->len - self->zst.next_in; + left_size = (Py_ssize_t)((uintptr_t)data->buf + data->len - + (uintptr_t)self->zst.next_in); if (left_size + bytes_in_bitbuffer > (PY_SSIZE_T_MAX - old_size)) { PyErr_NoMemory(); return -1; @@ -929,9 +933,16 @@ save_unconsumed_input(decompobject *self, Py_buffer *data, int err) memcpy(new_data_ptr, PyBytes_AS_STRING(self->unused_data), old_size); bitbuffer_copy(&(self->zst), new_data_ptr + old_size, bytes_in_bitbuffer); - memcpy(new_data_ptr + old_size + bytes_in_bitbuffer, - self->zst.next_in, left_size); + if (copy_from_next_in(&(self->zst), data->buf, + self->saved_bitbuffer, + self->saved_bitbuffer_size, + new_data_ptr + old_size + bytes_in_bitbuffer, + left_size) < 0) { + Py_DECREF(new_data); + return -1; + } Py_SETREF(self->unused_data, new_data); + self->zst.next_in = (uint8_t *)data->buf + data->len; self->zst.avail_in = 0; } } @@ -948,6 +959,10 @@ save_unconsumed_input(decompobject *self, Py_buffer *data, int err) Py_SETREF(self->unconsumed_tail, new_data); } + if (self->zst.block_state != ISAL_BLOCK_FINISH) { + save_bitbuffer(&(self->zst), self->saved_bitbuffer, + &self->saved_bitbuffer_size); + } return 0; } diff --git a/tests/test_compat.py b/tests/test_compat.py index 2034c7ed..89f19313 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -185,6 +185,21 @@ def test_unused_data(unused_size, wbits): assert decompressor.unused_data == unused_data +@pytest.mark.parametrize("wbits", [-15, 15, 31]) +def test_decompressobj_unconsumed_tail_after_eof(wbits): + source = b"abcdefghijklmnopqrstuvwxyz" + remainder = b"0123456789" + compressor = zlib.compressobj(wbits=wbits) + compressed = compressor.compress(source) + compressor.flush() + decompressor = isal_zlib.decompressobj(wbits=wbits) + data = decompressor.decompress(compressed + remainder, 1) + data += decompressor.decompress(decompressor.unconsumed_tail) + assert decompressor.eof + assert data == source + assert decompressor.unconsumed_tail == b"" + assert decompressor.unused_data == remainder + + def test_zlib_dictionary_decompress(): dictionary = b"bla" data = b"bladiebla" diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index 95c451fb..e53f4af3 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -257,6 +257,31 @@ def test_igzip_decompressor_raw_deflate_unused_data_gzip(test_offset): assert igzd.unused_data == true_unused_data +@pytest.mark.parametrize(["flag", "wbits"], [(DECOMP_DEFLATE, -15), + (DECOMP_ZLIB, 15), + (DECOMP_GZIP, 31)]) +def test_igzip_decompressor_unused_data_after_max_length(flag, wbits): + data = b"hello world " * 300 + unused_data = b"0123456789" + compressor = zlib.compressobj(wbits=wbits) + compressed = compressor.compress(data) + compressor.flush() + unused_data + stream_end = len(compressed) - len(unused_data) + # The stream can end in bytes that an earlier call read + for chunk_size in range(1, 25): + decompressor = IgzipDecompressor(flag=flag) + result = b"" + position = 0 + while not decompressor.eof: + chunk = b"" + if decompressor.needs_input: + assert position < len(compressed) + chunk = compressed[position:position + chunk_size] + position += chunk_size + result += decompressor.decompress(chunk, 7) + assert result == data + assert decompressor.unused_data == compressed[stream_end:position] + + @pytest.mark.skipif(not hasattr(sys, "getrefcount"), reason="Needs reference counting") def test_igzip_decompressor_keeps_zdict_reference(): From cd5e2664aa55078201352c30e8bb23bf7cfba4a1 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Thu, 15 Oct 2026 13:23:54 -0400 Subject: [PATCH 18/73] Raise argument errors from _ParallelCompress instead of SystemError The buffersize check set ValueError without returning, and failed argument parsing went unnoticed because PyArg_ParseTupleAndKeywords returns 0, not a negative value. Both raised SystemError, for example for a large block_size or a non-integer compresslevel in igzip_threaded.open. --- src/isal/isal_zlibmodule.c | 5 +++-- tests/test_igzip_threaded.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index c166c9da..50e6f740 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -310,8 +310,8 @@ ParallelCompress__new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) int level = ISAL_DEFAULT_COMPRESSION; static char *format = "n|i:ParallelCompress__new__"; static char *kwarg_names[] = {"buffersize", "level", NULL}; - if (PyArg_ParseTupleAndKeywords(args, kwargs, format, kwarg_names, - &buffer_size, &level) < 0) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, kwarg_names, + &buffer_size, &level)) { return NULL; } uint32_t level_buf_size; @@ -323,6 +323,7 @@ ParallelCompress__new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyErr_Format(PyExc_ValueError, "buffersize must be at most %zd, got %zd", (Py_ssize_t)UINT32_MAX, buffer_size); + return NULL; } ParallelCompress *self = PyObject_New(ParallelCompress, type); if (self == NULL) { diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index d2aee567..72b64480 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -156,6 +156,17 @@ def test_writer_too_low_threads(): error.match("at least 1") +def test_writer_block_size_too_large(): + with pytest.raises(ValueError) as error: + igzip_threaded.open(io.BytesIO(), "wb", block_size=2**32) + error.match("buffersize must be at most") + + +def test_writer_level_not_an_integer(): + with pytest.raises(TypeError): + igzip_threaded.open(io.BytesIO(), "wb", compresslevel="x") + + def test_reader_read_after_close(): with open(TEST_FILE, "rb") as test_f: f = igzip_threaded._ThreadedGzipReader(test_f) From b238c9ff9b8044187fc8ab29860be1aa68567eba Mon Sep 17 00:00:00 2001 From: ptweezy Date: Thu, 15 Oct 2026 13:24:49 -0400 Subject: [PATCH 19/73] Update changelog --- CHANGELOG.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4b200d3c..761e3584 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,22 @@ version 1.8.1-dev by standard zlib, including builds linked against older ISA-L releases. + Fix backward seeking in igzip files by discarding buffered compressed data when rewinding the underlying file. ++ Fix a crash when ``igzip.decompress`` gets an object that is not + bytes-like, such as a ``str``. It raises a ``TypeError`` instead. ++ Fix a crash after creating ``igzip_lib.IgzipDecompressor`` objects with a + ``zdict``. ++ Fix a hang in ``isal_zlib.Compress``: after ``flush()`` raised an error for + an unsupported mode, the next call on the same object never returned. ++ Fix a crash when you call ``flush(Z_FINISH)`` twice on a level 0 + ``isal_zlib.Compress`` object. ++ Fix ``unused_data`` of ``isal_zlib.Decompress`` and + ``igzip_lib.IgzipDecompressor``, which could contain bytes from outside the + input buffer when ``max_length`` split the end of a stream. At the end of a + stream, ``isal_zlib.Decompress.unconsumed_tail`` is empty, matching the + CPython fix for gh-158169. ++ ``igzip_threaded.open`` raises ``ValueError`` for a ``block_size`` of about + 3.6 GiB or more and ``TypeError`` for a ``compresslevel`` that is not an + integer. Both raised ``SystemError`` before. + Extraordinarily large gzip headers with either a very long comment or filename will now cause an ``OverflowError`` rather than use more and more memory. The size is limited to the internal buffer size which is currently From 007ca8b6ef1ba3932817c5d94e70d6db34a0edd4 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Thu, 15 Oct 2026 15:04:01 -0400 Subject: [PATCH 20/73] Import fewer modules in igzip and igzip_threaded argparse and shutil are only needed by the command line interface, and multiprocessing was only used for cpu_count, which os.cpu_count replaces. Importing igzip_threaded takes 4.8 ms instead of 11.1 ms on Python 3.14. --- CHANGELOG.rst | 3 +++ src/isal/igzip.py | 4 ++-- src/isal/igzip_threaded.py | 6 +----- tests/test_igzip.py | 13 +++++++++++++ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4b200d3c..eda5929b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,9 @@ Changelog version 1.8.1-dev ----------------- ++ Speed up ``import isal.igzip`` and ``import isal.igzip_threaded`` by + importing ``argparse`` and ``shutil`` only when the command line interface + runs, and by dropping the ``multiprocessing`` import. + Updated ISA-L to 2.32.1. + Python 3.10 and PyPy 3.10 are no longer supported. Added support for PyPy 3.12 and Python 3.15. diff --git a/src/isal/igzip.py b/src/isal/igzip.py index af1b4712..d1b0e1a9 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -24,12 +24,10 @@ """Similar to the stdlib gzip module. But using the Intel Storage Accelaration Library to speed up its methods.""" -import argparse import builtins import gzip import io import os -import shutil import struct import sys import time @@ -255,6 +253,7 @@ def decompress(data): def _argument_parser(): + import argparse # Only the command line interface needs it parser = argparse.ArgumentParser() parser.description = ( "A simple command line interface for the igzip module. " @@ -302,6 +301,7 @@ def _argument_parser(): def main(): + import shutil # Only the command line interface needs it args = _argument_parser().parse_args() compresslevel = args.compresslevel or _COMPRESS_LEVEL_TRADEOFF diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index 355b7f72..1e1912cb 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -7,7 +7,6 @@ import builtins import io -import multiprocessing import os import queue import struct @@ -52,10 +51,7 @@ def open(filename, mode="rb", compresslevel=igzip._COMPRESS_LEVEL_TRADEOFF, try: threads = len(os.sched_getaffinity(0)) except: # noqa: E722 - try: - threads = multiprocessing.cpu_count() - except: # noqa: E722 - threads = 1 + threads = os.cpu_count() or 1 if "r" in mode: gzip_file = io.BufferedReader( _ThreadedGzipReader(filename, block_size=block_size)) diff --git a/tests/test_igzip.py b/tests/test_igzip.py index c0661197..2774c1c7 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -15,6 +15,7 @@ import re import shutil import struct +import subprocess import sys import tempfile import zlib @@ -261,6 +262,18 @@ def test_compress_infile_out_file_no_name(tmp_path, capsysbinary): assert output[4:8] == b"\x00\x00\x00\x00" # No timestamp set. +def test_import_does_not_load_cli_modules(): + # Each of these modules takes milliseconds to import + code = ("import sys; before = set(sys.modules); " + "import isal.igzip, isal.igzip_threaded; " + "print(*(set(sys.modules) - before))") + result = subprocess.run([sys.executable, "-c", code], check=True, + capture_output=True, text=True) + new_modules = set(result.stdout.split()) + assert "isal.igzip_threaded" in new_modules + assert not new_modules & {"argparse", "shutil", "multiprocessing"} + + def test_decompress(): assert igzip.decompress(COMPRESSED_DATA) == DATA From 15ea79991fcc612831d4138c2fd3707f40d1fea6 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Thu, 15 Oct 2026 15:09:50 -0400 Subject: [PATCH 21/73] Check for a closed file faster in IGzipFile.readline GzipFile.readline checks for a closed file through two Python calls for each line. Checking fileobj directly makes line iteration, readlines(), and readline() 1.13 times faster for FASTQ data and 1.5 times faster for lines of a few bytes. --- CHANGELOG.rst | 3 ++ src/isal/igzip.py | 8 ++++ tests/test_igzip.py | 89 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4b200d3c..7789b9a3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,9 @@ version 1.8.1-dev + Python 3.10 and PyPy 3.10 are no longer supported. Added support for PyPy 3.12 and Python 3.15. + Restore PyPy wheel builds. ++ Speed up reading lines from binary ``igzip`` files through iteration, + ``readlines()``, and ``readline()``. On an Apple M4, reading FASTQ data is + 1.13 times faster, and reading lines of a few bytes is 1.5 times faster. + Fix zlib streams compressed with a preset dictionary so they can be decoded by standard zlib, including builds linked against older ISA-L releases. + Fix backward seeking in igzip files by discarding buffered compressed data diff --git a/src/isal/igzip.py b/src/isal/igzip.py index af1b4712..5c95d665 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -166,6 +166,14 @@ def __init__(self, filename=None, mode=None, raw = _GzipReader(self.fileobj, READ_BUFFER_SIZE) self._buffer = io.BufferedReader(raw) + def readline(self, size=-1): + """Read a line of decompressed data, or at most size bytes of it.""" + # GzipFile.closed tests whether fileobj is None, so testing fileobj + # here skips the two Python calls GzipFile.readline makes per line. + if self.fileobj is None: + self._check_not_closed() + return self._buffer.readline(size) + def __repr__(self): s = repr(self.fileobj) return '' diff --git a/tests/test_igzip.py b/tests/test_igzip.py index c0661197..c5585fae 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -11,6 +11,7 @@ import gzip import io +import itertools import os import re import shutil @@ -72,6 +73,94 @@ def test_igzipfile_read_truncated(): "reached") +LINES_DATA = [ + b"", + b"no newline", + b"a\nb\n\nc", + b"".join(b"line %d\n" % i for i in range(10_000)), + b"x" * 200_000 + b"\n" + b"y" * 10, +] + + +@pytest.mark.parametrize("data", LINES_DATA, + ids=["empty", "no_newline", "blank_line", + "many_lines", "long_line"]) +def test_igzipfile_iteration_like_gzipfile(data): + compressed = gzip.compress(data) + with gzip.GzipFile(fileobj=io.BytesIO(compressed)) as gzip_file: + expected = list(gzip_file) + with igzip.IGzipFile(fileobj=io.BytesIO(compressed)) as igzip_file: + assert list(igzip_file) == expected + igzip_file.seek(0) + assert igzip_file.readlines() == expected + igzip_file.seek(0) + assert list(itertools.islice(igzip_file, 2)) == expected[:2] + assert igzip_file.read() == b"".join(expected[2:]) + + +def test_igzipfile_is_its_own_iterator(): + with igzip.IGzipFile(fileobj=io.BytesIO(COMPRESSED_DATA)) as igzip_file: + assert iter(igzip_file) is igzip_file + + +def test_igzipfile_readline_skips_check_not_closed_when_open(monkeypatch): + calls = [] + check_not_closed = igzip.IGzipFile._check_not_closed + monkeypatch.setattr(igzip.IGzipFile, "_check_not_closed", + lambda self: calls.append(1) or check_not_closed(self)) + data = gzip.compress(b"a\nb\n" * 100) + igzip_file = igzip.IGzipFile(fileobj=io.BytesIO(data)) + assert len(list(igzip_file)) == 200 + # Open files skip the check, which costs two Python calls per line + assert calls == [] + igzip_file.close() + with pytest.raises(ValueError, match="I/O operation on closed file"): + igzip_file.readline() + assert calls == [1] + + +def test_igzipfile_iteration_uses_replaced_methods(monkeypatch): + class ReadlineSubclass(igzip.IGzipFile): + def readline(self, size=-1): + return super().readline(size).upper() + + class NextSubclass(igzip.IGzipFile): + def __next__(self): + return super().__next__().upper() + + compressed = gzip.compress(b"a\nb\n") + for cls in (ReadlineSubclass, NextSubclass): + with cls(fileobj=io.BytesIO(compressed)) as igzip_file: + assert list(igzip_file) == [b"A\n", b"B\n"] + with igzip.IGzipFile(fileobj=io.BytesIO(compressed)) as igzip_file: + igzip_file.readline = lambda size=-1: b"" + assert list(igzip_file) == [] + monkeypatch.setattr(igzip.IGzipFile, "readline", lambda self, size=-1: b"") + with igzip.IGzipFile(fileobj=io.BytesIO(compressed)) as igzip_file: + assert list(igzip_file) == [] + + +def test_igzipfile_iteration_after_close(): + igzip_file = igzip.IGzipFile(fileobj=io.BytesIO(gzip.compress(b"a\nb\n"))) + lines = iter(igzip_file) + assert next(lines) == b"a\n" + igzip_file.close() + with pytest.raises(ValueError, match="I/O operation on closed file"): + next(lines) + with pytest.raises(ValueError, match="I/O operation on closed file"): + iter(igzip_file) + + +def test_igzipfile_iteration_in_write_mode_like_gzipfile(): + errors = [] + for cls in (gzip.GzipFile, igzip.IGzipFile): + with cls(fileobj=io.BytesIO(), mode="wb") as gzip_file: + with pytest.raises(Exception) as error: + next(iter(gzip_file)) + errors.append(type(error.value)) + assert errors[0] == errors[1] + + @pytest.mark.parametrize("level", range(1, 10)) def test_decompress_stdin_stdout(capsysbinary, level): """Test if the command line can decompress data that has been compressed From eb4270c0ecf7a3bcb5c801045795f2b7b24aeda3 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 16 Oct 2026 09:48:49 +0200 Subject: [PATCH 22/73] Update release checklist --- .github/release_checklist.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/release_checklist.md b/.github/release_checklist.md index 8ae899dc..4f98307c 100644 --- a/.github/release_checklist.md +++ b/.github/release_checklist.md @@ -1,5 +1,5 @@ Release checklist -- [ ] Check outstanding issues on JIRA and Github. +- [ ] Check outstanding issues on GitHub. - [ ] Check [latest documentation](https://python-isal.readthedocs.io/en/latest/) looks fine. - [ ] Create a release branch. - [ ] Change current development version in `CHANGELOG.rst` to stable version. @@ -7,9 +7,9 @@ Release checklist - [ ] Merge the release branch into `main`. - [ ] Created an annotated tag with the stable version number. Include changes from CHANGELOG.rst. -- [ ] Push tag to remote. This triggers the wheel/sdist build on github CI. +- [ ] Push tag to remote. This triggers the wheel/sdist build on GitHub CI. - [ ] merge `main` branch back into `develop`. - [ ] Build the new tag on readthedocs. Only build the last patch version of each minor version. So `1.1.1` and `1.2.0` but not `1.1.0`, `1.1.1` and `1.2.0`. -- [ ] Create a new release on github. +- [ ] Create a new release on GitHub. - [ ] Update the package on conda-forge. From c299a201033bec4e69809c973a4a30a6dcda488d Mon Sep 17 00:00:00 2001 From: ptweezy Date: Wed, 14 Oct 2026 19:20:03 -0400 Subject: [PATCH 23/73] Build ISA-L's aarch64 assembly on macOS arm64 Makefile.unx does not recognize the name arm64 and falls back to the portable C code. Fixes #254. --- CHANGELOG.rst | 4 ++++ setup.py | 8 +++++++- tests/test_isal_build.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tests/test_isal_build.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 96f6f245..97ade68a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -44,6 +44,10 @@ version 1.8.1-dev filename will now cause an ``OverflowError`` rather than use more and more memory. The size is limited to the internal buffer size which is currently set at 512 KiB. ++ Fix builds on macOS with Apple Silicon, which used the portable C code of + ISA-L rather than its optimized aarch64 assembly. On an Apple M4, + decompression is 1.6 to 3 times faster, CRC-32 is 50 times faster, and + Adler-32 is 7 times faster. version 1.8.0 ----------------- diff --git a/setup.py b/setup.py index 1ed64c2d..75c52987 100644 --- a/setup.py +++ b/setup.py @@ -113,15 +113,21 @@ def build_isa_l(): cpu_count = os.cpu_count() or 1 # os.cpu_count() can return None run_args = dict(cwd=build_dir, env=build_env) if SYSTEM_IS_UNIX: + make_args = [] if platform.machine() == "aarch64": cflags_param = "CFLAGS_aarch64" + elif sys.platform == "darwin" and platform.machine() == "arm64": + # Makefile.unx does not recognize "arm64" and builds the slow + # portable C code. ISA-L still checks CPU features at runtime. + cflags_param = "CFLAGS_aarch64" + make_args = ["host_cpu=aarch64", "arch=aarch64"] else: cflags_param = "CFLAGS_" make_cmd = "make" if SYSTEM_IS_BSD: make_cmd = "gmake" subprocess.run([make_cmd, "-j", str(cpu_count), "-f", "Makefile.unx", - "isa-l.h", "bin/isa-l.a", + "isa-l.h", "bin/isa-l.a", *make_args, f"{cflags_param}={build_env.get('CFLAGS', '')}"], **run_args) elif SYSTEM_IS_WINDOWS: diff --git a/tests/test_isal_build.py b/tests/test_isal_build.py new file mode 100644 index 00000000..ccd63c56 --- /dev/null +++ b/tests/test_isal_build.py @@ -0,0 +1,29 @@ +"""Check that the statically linked ISA-L uses its optimized code paths.""" +import platform +from pathlib import Path + +from isal import isal_zlib + +import pytest + +EXTENSION_BYTES = Path(isal_zlib.__file__).read_bytes() + +# Function names from ISA-L's portable C code. The extension binary contains +# them only when ISA-L is linked statically and the symbols are not stripped. +STATIC_ISAL = b"decode_huffman_code_block_stateless_base" in EXTENSION_BYTES + + +@pytest.mark.skipif(platform.machine().lower() not in ("aarch64", "arm64"), + reason="Only applies to aarch64 builds") +@pytest.mark.skipif(not STATIC_ISAL, + reason="ISA-L is linked dynamically or stripped") +@pytest.mark.parametrize("symbol", [ + b"decode_huffman_code_block_stateless_aarch64", # Inflate + b"isal_deflate_body_aarch64", # Deflate + b"crc32_gzip_refl_pmull", # CRC-32 + b"adler32_neon", # Adler-32 +]) +def test_aarch64_build_includes_optimized_functions(symbol): + # Without these functions, ISA-L falls back to its portable C code, + # which is several times slower. + assert symbol in EXTENSION_BYTES From 73db6bf239855e40f753cdcb3ca82974d13b7927 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Wed, 14 Oct 2026 19:20:03 -0400 Subject: [PATCH 24/73] List arm64 in the dependency markers Closes #252. --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 734b2f74..d90f791c 100644 --- a/README.rst +++ b/README.rst @@ -113,12 +113,12 @@ your project please list a python-isal dependency as follows. ``setup.cfg``:: install_requires = - isal; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64" + isal; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64" or platform.machine == "arm64" ``setup.py``:: extras_require={ - ":platform.machine == 'x86_64' or platform.machine == 'AMD64' or platform.machine == 'aarch64'": ['isal'] + ":platform.machine == 'x86_64' or platform.machine == 'AMD64' or platform.machine == 'aarch64' or platform.machine == 'arm64'": ['isal'] }, .. dependency end From f7e3c0a455684db7c12633578b372f8e87b8b750 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Wed, 14 Oct 2026 19:20:03 -0400 Subject: [PATCH 25/73] Use the GzipFile write buffer in IGzipFile Since Python 3.12 GzipFile buffers small writes. IGzipFile.write bypassed that buffer and compressed each write separately. --- CHANGELOG.rst | 4 ++++ src/isal/igzip.py | 20 +++++++++++++++++- tests/test_igzip.py | 49 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 96f6f245..d4e7bb54 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -44,6 +44,10 @@ version 1.8.1-dev filename will now cause an ``OverflowError`` rather than use more and more memory. The size is limited to the internal buffer size which is currently set at 512 KiB. ++ Speed up many small writes to ``igzip.IGzipFile`` on Python 3.12 and later. + ``IGzipFile`` collects these writes in the write buffer of + ``gzip.GzipFile`` and compresses them in large chunks, which makes writing + lines 1.4 times faster. version 1.8.0 ----------------- diff --git a/src/isal/igzip.py b/src/isal/igzip.py index 1938181b..52717d67 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -55,6 +55,10 @@ BadGzipFile = gzip.BadGzipFile # type: ignore +# Since Python 3.12 GzipFile.write buffers small writes and passes them to +# _write_raw in large chunks, which compress much faster. +_GZIP_BUFFERS_WRITES = hasattr(gzip.GzipFile, "_write_raw") + # The open method was copied from the CPython source with minor adjustments. def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_TRADEOFF, @@ -208,7 +212,12 @@ def write(self, data): if self.fileobj is None: raise ValueError("write() on closed IGzipFile object") - if isinstance(data, bytes): + if _GZIP_BUFFERS_WRITES: + return self._buffer.write(data) + return self._write_raw(data) + + def _write_raw(self, data): + if isinstance(data, (bytes, bytearray)): length = len(data) else: # accept any data that supports the buffer protocol @@ -222,6 +231,15 @@ def write(self, data): self.offset += length return length + def seek(self, offset, whence=io.SEEK_SET): + position = super().seek(offset, whence) + if self.mode == WRITE and _GZIP_BUFFERS_WRITES: + # GzipFile.seek returns the offset before the zeros it writes + # leave the write buffer. + self._buffer.flush() + return self.offset + return position + # Aliases for improved compatibility with CPython gzip module. GzipFile = IGzipFile diff --git a/tests/test_igzip.py b/tests/test_igzip.py index 9c6efa52..4ebc477e 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -611,3 +611,52 @@ def test_adverserial_header(): gzip_file.read() finally: os.remove(tmpfile.name) + + +@pytest.mark.skipif(not igzip._GZIP_BUFFERS_WRITES, + reason="GzipFile buffers writes from Python 3.12 onwards") +def test_small_writes_are_compressed_in_large_chunks(monkeypatch): + chunk_sizes = [] + write_raw = igzip.IGzipFile._write_raw + + def recording_write_raw(self, data): + chunk_sizes.append(memoryview(data).nbytes) + return write_raw(self, data) + + monkeypatch.setattr(igzip.IGzipFile, "_write_raw", recording_write_raw) + lines = [b"line %d\n" % i for i in range(10_000)] + output = io.BytesIO() + with igzip.IGzipFile(fileobj=output, mode="wb") as gzip_file: + for line in lines: + assert gzip_file.write(line) == len(line) + data = b"".join(lines) + assert sum(chunk_sizes) == len(data) + # Each compressor call has overhead, so small writes must be combined + assert len(chunk_sizes) < len(lines) // 100 + assert gzip.decompress(output.getvalue()) == data + + +def test_seek_in_write_mode_returns_new_position(): + output = io.BytesIO() + with igzip.IGzipFile(fileobj=output, mode="wb") as gzip_file: + gzip_file.write(b"abc") + assert gzip_file.seek(10) == 10 + assert gzip_file.seek(5, io.SEEK_CUR) == 15 + assert gzip_file.tell() == 15 + assert gzip.decompress(output.getvalue()) == b"abc" + bytes(12) + + +def test_small_writes_tell_and_flush(): + output = io.BytesIO() + with igzip.IGzipFile(fileobj=output, mode="wb") as gzip_file: + assert gzip_file.write(b"ab") == 2 + assert gzip_file.tell() == 2 + assert gzip_file.write(bytearray(b"cd")) == 2 + assert gzip_file.write(memoryview(b"ef")) == 2 + assert gzip_file.tell() == 6 + gzip_file.flush() + # A flush emits all data written so far + decompressor = zlib.decompressobj(wbits=31) + assert decompressor.decompress(output.getvalue()) == b"abcdef" + gzip_file.write(b"gh") + assert gzip.decompress(output.getvalue()) == b"abcdefgh" From 2019fd46564f5efe3027be311673cae53ab0d190 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Wed, 14 Oct 2026 19:20:03 -0400 Subject: [PATCH 26/73] Wake igzip_threaded threads instead of waiting for poll timeouts Closing a threaded file or reaching EOF waited up to 50 ms for a queue timeout. --- CHANGELOG.rst | 3 ++ src/isal/igzip_threaded.py | 54 +++++++++++++++++++---- tests/test_igzip_threaded.py | 84 ++++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 96f6f245..6e30ebc3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -44,6 +44,9 @@ version 1.8.1-dev filename will now cause an ``OverflowError`` rather than use more and more memory. The size is limited to the internal buffer size which is currently set at 512 KiB. ++ Fix ``igzip_threaded`` readers and writers that waited for a polling timeout + of up to 50 ms at the end of the stream or when closed. Writing and closing + a small file takes less than a millisecond instead of 60 ms. version 1.8.0 ----------------- diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index 1e1912cb..778ca2a5 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -112,20 +112,25 @@ def _decompress(self): data = self.fileobj.read(block_size) except Exception as e: self.exception = e - return - if not data: - return + data = b"" + # An empty block tells the reader about EOF or an error while self.running and self._calling_thread.is_alive(): try: block_queue.put(data, timeout=0.05) break except queue.Full: pass + if not data: + return def readinto(self, b): self._check_closed() result = self.buffer.readinto(b) if result == 0: + if self.eof: + if self.exception: + raise self.exception + return 0 while True: try: data_from_queue = self.queue.get(timeout=0.01) @@ -136,6 +141,11 @@ def readinto(self, b): raise self.exception # EOF reached return 0 + if not data_from_queue: + self.eof = True + if self.exception: + raise self.exception + return 0 self.buffer = io.BytesIO(data_from_queue) result = self.buffer.readinto(b) self.pos += result @@ -152,6 +162,12 @@ def close(self) -> None: if self._closed: return self.running = False + # Empty the queue so a worker blocked on put can stop + while True: + try: + self.queue.get_nowait() + except queue.Empty: + break self.worker.join() self.fileobj.close() if self.closefd: @@ -231,9 +247,12 @@ def __init__(self, level=level) for _ in range(threads) ] if threads > 1: - self.input_queues: List[queue.Queue[Tuple[bytes, memoryview]]] = [ + # None is the sentinel from stop() + self.input_queues: List[ + queue.Queue[Optional[Tuple[bytes, memoryview]]]] = [ queue.Queue(queue_size) for _ in range(threads)] - self.output_queues: List[queue.Queue[Tuple[bytes, int, int]]] = [ + self.output_queues: List[ + queue.Queue[Optional[Tuple[bytes, int, int]]]] = [ queue.Queue(queue_size) for _ in range(threads)] self.output_worker = threading.Thread(target=self._write) self.compression_workers = [ @@ -283,6 +302,13 @@ def start(self): def stop(self): """Stop, but do not care for remaining work""" self.running = False + # Wake up threads that wait on a queue. A thread with a full queue + # is busy and checks self.running when it is done. + for q in self.input_queues + self.output_queues: + try: + q.put_nowait(None) + except queue.Full: + pass for worker in self.compression_workers: worker.join() self.output_worker.join() @@ -351,11 +377,15 @@ def _compress(self, index: int): compressor: isal_zlib._ParallelCompress = self.compressors[index] while True: try: - data, zdict = in_queue.get(timeout=0.05) + item = in_queue.get(timeout=0.05) except queue.Empty: if not (self.running and self._calling_thread.is_alive()): return continue + if item is None: # Sentinel from stop() + in_queue.task_done() + return + data, zdict = item try: compressed, crc = compressor.compress_and_crc(data, zdict) except Exception as e: @@ -373,11 +403,15 @@ def _write(self): out_index = index % self.threads output_queue = output_queues[out_index] try: - compressed, crc, data_length = output_queue.get(timeout=0.05) + item = output_queue.get(timeout=0.05) except queue.Empty: if not (self.running and self._calling_thread.is_alive()): return continue + if item is None: # Sentinel from stop() + output_queue.task_done() + return + compressed, crc, data_length = item self._crc = isal_zlib.crc32_combine(self._crc, crc, data_length) self._size += data_length self.raw.write(compressed) @@ -391,11 +425,15 @@ def _compress_and_write(self): compressor = self.compressors[0] while True: try: - data, zdict = in_queue.get(timeout=0.05) + item = in_queue.get(timeout=0.05) except queue.Empty: if not (self.running and self._calling_thread.is_alive()): return continue + if item is None: # Sentinel from stop() + in_queue.task_done() + return + data, zdict = item try: compressed, crc = compressor.compress_and_crc(data, zdict) except Exception as e: diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 72b64480..cec924f8 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -9,9 +9,12 @@ import io import itertools import os +import queue import subprocess import sys import tempfile +import threading +import types import zlib from pathlib import Path @@ -281,3 +284,84 @@ def test_flush(tmp_path, threads): deflate_block = data[10:] + deflate_last_block assert zlib.decompress(deflate_block, wbits=-15) == b"123" assert gzip.decompress(test_file.read_bytes()) == b"123" + + +# Threads in igzip_threaded poll their queues with a timeout, so they can exit +# with the calling thread. Normal use should never wait for that timeout. This +# fixture stretches the timeouts to 2 seconds and records the ones that expire. +@pytest.fixture +def queue_monitor(monkeypatch): + monitor = types.SimpleNamespace(expired=[], put_blocked=threading.Event()) + + class PatientQueue(queue.Queue): + def get(self, block=True, timeout=None): + if not block or timeout is None: + return super().get(block, timeout) + try: + return super().get(block, 2.0) + except queue.Empty: + monitor.expired.append("get") + raise + + def put(self, item, block=True, timeout=None): + if not block or timeout is None: + return super().put(item, block, timeout) + if self.full(): + monitor.put_blocked.set() + try: + return super().put(item, block, 2.0) + except queue.Full: + monitor.expired.append("put") + raise + + monkeypatch.setattr(igzip_threaded, "queue", types.SimpleNamespace( + Queue=PatientQueue, Empty=queue.Empty, Full=queue.Full)) + return monitor + + +@pytest.mark.timeout(10) +@pytest.mark.parametrize("threads", [1, 3]) +def test_writer_does_not_wait_for_queue_timeouts(queue_monitor, threads): + output = io.BytesIO() + with igzip_threaded.open(output, "wb", threads=threads) as f: + f.write(b"first") + f.flush() + f.write(b"second") + assert gzip.decompress(output.getvalue()) == b"firstsecond" + assert queue_monitor.expired == [] + + +@pytest.mark.timeout(10) +def test_reader_does_not_wait_for_queue_timeouts(queue_monitor): + data = os.urandom(100_000) + with igzip_threaded.open(io.BytesIO(gzip.compress(data)), "rb", + block_size=8 * 1024) as f: + assert f.read() == data + # Reading at the end of the stream returns immediately + assert f.read() == b"" + assert f.read(10) == b"" + assert queue_monitor.expired == [] + + +@pytest.mark.timeout(10) +def test_reader_closes_early_without_waiting_for_queue_timeouts( + queue_monitor): + data = os.urandom(1024 * 1024) + with igzip_threaded.open(io.BytesIO(gzip.compress(data)), "rb", + block_size=8 * 1024) as f: + assert f.read(10) == data[:10] + # Close while the reader thread waits for room in its full queue + assert queue_monitor.put_blocked.wait(timeout=5) + assert queue_monitor.expired == [] + + +@pytest.mark.timeout(10) +def test_reader_error_does_not_wait_for_queue_timeouts(queue_monitor): + truncated = gzip.compress(os.urandom(100_000))[:-8] + with igzip_threaded.open(io.BytesIO(truncated), "rb") as f: + with pytest.raises(EOFError): + f.read() + # The error is raised again on subsequent reads + with pytest.raises(EOFError): + f.read() + assert queue_monitor.expired == [] From 6c08ca171c7c661c97b284ca4de52c2b67a93dfa Mon Sep 17 00:00:00 2001 From: ptweezy Date: Fri, 16 Oct 2026 15:45:09 -0400 Subject: [PATCH 27/73] Run the tests with AddressSanitizer and ThreadSanitizer on CI The test-sanitizers job builds python-isal and the C code of ISA-L with a sanitizer, and runs the tests on CPython builds with the same sanitizer from nascheme/cpython_sanity: cpython-asan:3.14 and the free-threaded cpython-tsan:3.14t. The existing tests never share an object between threads, so ThreadSanitizer could not see a missing lock. The new tests share one Compress, Decompress, or IgzipDecompressor between threads. test_threaded_program_can_exit_on_error checks the exit code of its child process, so a sanitizer report in that process fails the test. --- .github/workflows/ci.yml | 44 +++++++++++++++++++++++++ tests/test_freethreading.py | 63 ++++++++++++++++++++++++++++++++++++ tests/test_igzip_threaded.py | 5 ++- 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e82cebfd..0b0b09b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,6 +157,49 @@ jobs: env: PYTHON_ISAL_LINK_DYNAMIC: True + # Run the tests on CPython builds with AddressSanitizer and ThreadSanitizer + # from https://github.com/nascheme/cpython_sanity. + test-sanitizers: + runs-on: ubuntu-latest + # A crash under ThreadSanitizer can hang instead of exiting. + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - sanitizer: address + image: ghcr.io/nascheme/cpython-asan:3.14 + # The free-threaded build runs Python threads in parallel. + - sanitizer: thread + image: ghcr.io/nascheme/cpython-tsan:3.14t + container: + image: ${{ matrix.image }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install build dependencies + run: apt-get update && apt-get install -y nasm + - name: Install python-isal with ${{ matrix.sanitizer }} sanitizer + # setuptools-scm needs git, and the container user does not own the + # repository. + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + python -m pip install . pytest pytest-timeout + env: + # The extension modules get -fsanitize from CPython. CFLAGS adds it + # to the C code of ISA-L. + CFLAGS: -fsanitize=${{ matrix.sanitizer }} -fno-omit-frame-pointer -g + - name: Run tests + # Do not set PYTHONDEVMODE. Its debug allocator pads each allocation, + # so AddressSanitizer misses small overflows. --capture=sys prints + # sanitizer reports from C code even when they end the process, and + # -v prints the name of the test before its report. + run: python -m pytest -v tests --capture=sys + env: + # Exit at the first data race. A crash after a race can hang. + TSAN_OPTIONS: halt_on_error=1 + deploy: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') runs-on: ${{ matrix.os }} @@ -166,6 +209,7 @@ jobs: - test-static - test-dynamic - test-arch + - test-sanitizers strategy: matrix: os: diff --git a/tests/test_freethreading.py b/tests/test_freethreading.py index 10d1d53b..728c3274 100644 --- a/tests/test_freethreading.py +++ b/tests/test_freethreading.py @@ -108,3 +108,66 @@ def test_isal_compress_decompress_threaded(compress, decompress): ] for future in concurrent.futures.as_completed(futures): future.result() # To fire assertion error if there is one + + +def run_in_threads(function): + """Call function(index) in NUM_THREADS threads at the same time.""" + start = threading.Barrier(parties=NUM_THREADS) + + def run(index): + start.wait() + return function(index) + + with concurrent.futures.ThreadPoolExecutor(NUM_THREADS) as executor: + return list(executor.map(run, range(NUM_THREADS))) + + +def test_compressobj_shared_between_threads(): + compressobj = isal_zlib.compressobj() + + def compress(index): + for _ in range(NUM_ITERATIONS): + compressobj.compress(HAMLET_SCENE) + if index == 0: + compressobj.flush(isal_zlib.Z_SYNC_FLUSH) + + # The order of the calls is unknown, so the output cannot be checked. + # This test is for ThreadSanitizer. + run_in_threads(compress) + compressobj.flush() + + +def test_decompressobj_shared_between_threads(): + decompressobj = isal_zlib.decompressobj() + decompressobj.decompress(isal_zlib.compress(HAMLET_SCENE)) + assert decompressobj.eof + + def decompress(index): + # Data after the end of the stream is added to unused_data. + for _ in range(NUM_ITERATIONS): + assert decompressobj.decompress(bytes([index]) * 10) == b"" + + run_in_threads(decompress) + unused_data = decompressobj.unused_data + blocks = [unused_data[i:i + 10] for i in range(0, len(unused_data), 10)] + assert sorted(blocks) == sorted( + bytes([index]) * 10 + for index in range(NUM_THREADS) + for _ in range(NUM_ITERATIONS) + ) + + +def test_igzip_decompressor_shared_between_threads(): + data = HAMLET_SCENE * 100 + decompressor = igzip_lib.IgzipDecompressor() + first = decompressor.decompress(igzip_lib.compress(data), 100) + + def decompress(index): + size = 0 + while True: + try: + size += len(decompressor.decompress(b"", 100)) + except EOFError: + return size + + assert len(first) + sum(run_in_threads(decompress)) == len(data) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 72b64480..e4bef1b0 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -250,7 +250,10 @@ def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): f"mode='{mode}', threads={threads})\n" ) f.write("raise Exception('Error')\n") - subprocess.run([sys.executable, str(program)]) + result = subprocess.run([sys.executable, str(program)], + capture_output=True) + # A sanitizer report changes the exit code. + assert result.returncode == 1, result.stderr.decode() @pytest.mark.parametrize("threads", [1, 2]) From 31ba1f3dd1db38cef018c176982c3a10a3d8675c Mon Sep 17 00:00:00 2001 From: ptweezy Date: Fri, 16 Oct 2026 15:45:27 -0400 Subject: [PATCH 28/73] Fix a crash when reading decompressor attributes from another thread decompress() replaces unused_data and unconsumed_tail while it holds the object's lock, but CPython reads PyMemberDef members without that lock. On the free-threaded build, a thread that read unused_data while another thread decompressed could get an object that decompress() had just freed. Turn the attributes of isal_zlib.Decompress and igzip_lib.IgzipDecompressor into getters that take the lock, as CPython 3.15 does for zlib, and detect gzip or zlib in Decompress.decompress after taking the lock. --- CHANGELOG.rst | 6 +++ src/isal/igzip_libmodule.c | 64 ++++++++++++++++++++++------ src/isal/isal_shared.h | 1 - src/isal/isal_zlibmodule.c | 61 ++++++++++++++++++++------- tests/test_freethreading.py | 84 +++++++++++++++++++++++++++++++++++++ 5 files changed, 187 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 96f6f245..3ef945e6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -44,6 +44,12 @@ version 1.8.1-dev filename will now cause an ``OverflowError`` rather than use more and more memory. The size is limited to the internal buffer size which is currently set at 512 KiB. ++ Fix a crash on the free-threaded build when one thread reads + ``unused_data`` or ``unconsumed_tail`` of an ``isal_zlib`` decompressor + while another thread decompresses with it. Reading the attributes of + ``isal_zlib`` decompressors and ``igzip_lib.IgzipDecompressor`` from other + threads is now safe, and so is calling ``decompress()`` from several + threads at the same time when ``wbits`` detects the format. version 1.8.0 ----------------- diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index 8702aa0a..8d1aa045 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -32,7 +32,7 @@ typedef struct { separately. Conversion and looping is encapsulated in decompress_buf() */ Py_ssize_t avail_in_real; - char eof; /* T_BOOL expects a char */ + char eof; char needs_input; uint8_t saved_bitbuffer[8]; Py_ssize_t saved_bitbuffer_size; @@ -519,16 +519,56 @@ PyDoc_STRVAR(IgzipDecompressor_needs_input_doc, PyDoc_STRVAR(IgzipDecompressor_crc_doc, "The checksum that is saved if DECOMP_ZLIB* or DECOMP_GZIP* flags are used."); -static PyMemberDef IgzipDecompressor_members[] = { - {"eof", T_BOOL, offsetof(IgzipDecompressor, eof), - READONLY, IgzipDecompressor_eof__doc__}, - {"unused_data", T_OBJECT_EX, offsetof(IgzipDecompressor, unused_data), - READONLY, IgzipDecompressor_unused_data__doc__}, - {"needs_input", T_BOOL, offsetof(IgzipDecompressor, needs_input), READONLY, - IgzipDecompressor_needs_input_doc}, - {"crc", T_UINT, - offsetof(IgzipDecompressor, state) + offsetof(struct inflate_state, crc), - READONLY, IgzipDecompressor_crc_doc}, +/* The getters take the lock, because without the GIL another thread can + change these attributes in decompress(). */ +static PyObject * +IgzipDecompressor_get_eof(IgzipDecompressor *self, void *Py_UNUSED(closure)) +{ + ENTER_ZLIB(self); + char eof = self->eof; + LEAVE_ZLIB(self); + return PyBool_FromLong(eof); +} + +static PyObject * +IgzipDecompressor_get_unused_data(IgzipDecompressor *self, + void *Py_UNUSED(closure)) +{ + ENTER_ZLIB(self); + PyObject *unused_data = self->unused_data; + Py_INCREF(unused_data); + LEAVE_ZLIB(self); + return unused_data; +} + +static PyObject * +IgzipDecompressor_get_needs_input(IgzipDecompressor *self, + void *Py_UNUSED(closure)) +{ + ENTER_ZLIB(self); + char needs_input = self->needs_input; + LEAVE_ZLIB(self); + return PyBool_FromLong(needs_input); +} + +static PyObject * +IgzipDecompressor_get_crc(IgzipDecompressor *self, void *Py_UNUSED(closure)) +{ + ENTER_ZLIB(self); + uint32_t crc = self->state.crc; + LEAVE_ZLIB(self); + return PyLong_FromUnsignedLong(crc); +} + +static PyGetSetDef IgzipDecompressor_properties[] = { + {"eof", (getter)IgzipDecompressor_get_eof, NULL, + IgzipDecompressor_eof__doc__, NULL}, + {"unused_data", (getter)IgzipDecompressor_get_unused_data, NULL, + IgzipDecompressor_unused_data__doc__, NULL}, + {"needs_input", (getter)IgzipDecompressor_get_needs_input, NULL, + IgzipDecompressor_needs_input_doc, NULL}, + {"crc", (getter)IgzipDecompressor_get_crc, NULL, + IgzipDecompressor_crc_doc, NULL}, {NULL} }; @@ -540,7 +580,7 @@ static PyTypeObject IgzipDecompressor_Type = { .tp_flags = Py_TPFLAGS_DEFAULT, .tp_doc = igzip_lib_IgzipDecompressor___init____doc__, .tp_methods = IgzipDecompressor_methods, - .tp_members = IgzipDecompressor_members, + .tp_getset = IgzipDecompressor_properties, .tp_new = igzip_lib_IgzipDecompressor__new__, }; diff --git a/src/isal/isal_shared.h b/src/isal/isal_shared.h index 568e9e3e..f4f4519b 100644 --- a/src/isal/isal_shared.h +++ b/src/isal/isal_shared.h @@ -21,7 +21,6 @@ Changes compared to CPython: */ #define PY_SSIZE_T_CLEAN #include -#include "structmember.h" // PyMemberDef #include #include diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 50e6f740..f6bbd79b 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -983,6 +983,12 @@ isal_zlib_Decompress_decompress_impl(decompobject *self, Py_buffer *data, else hard_limit = max_length; + /* limit amount of data allocated to max_length */ + if (max_length && obuflen > max_length) + obuflen = max_length; + + ENTER_ZLIB(self); + if (!self->method_set) { if (data_is_gzip(data)){ self->zst.crc_flag = ISAL_GZIP; @@ -993,12 +999,6 @@ isal_zlib_Decompress_decompress_impl(decompobject *self, Py_buffer *data, self->method_set = 1; } - /* limit amount of data allocated to max_length */ - if (max_length && obuflen > max_length) - obuflen = max_length; - - ENTER_ZLIB(self); - self->zst.next_in = data->buf; ibuflen = data->len; @@ -1490,8 +1490,6 @@ static PyTypeObject IsalZlibCompType = { .tp_methods = comp_methods, }; -#define COMP_OFF(x) offsetof(decompobject, x) - PyDoc_STRVAR(Decomp_unconsumed_tail, "A bytes object that contains any data that was not consumed by the last\n" "decompress() call because it exceeded the limit for the uncompressed data\n" @@ -1499,13 +1497,44 @@ PyDoc_STRVAR(Decomp_unconsumed_tail, "feed it (possibly with further data concatenated to it) back to a \n" "subsequent decompress() method call in order to get correct output."); -static PyMemberDef Decomp_members[] = { - {"unused_data", T_OBJECT, COMP_OFF(unused_data), READONLY, - "Data found after the end of the compressed stream."}, - {"unconsumed_tail", T_OBJECT, COMP_OFF(unconsumed_tail), READONLY, - Decomp_unconsumed_tail}, - {"eof", T_BOOL, COMP_OFF(eof), READONLY, - "True if the end-of-stream marker has been reached."}, +/* The getters take the lock, because without the GIL another thread can + change these attributes in decompress() or flush(). */ +static PyObject * +Decomp_get_unused_data(decompobject *self, void *Py_UNUSED(closure)) +{ + ENTER_ZLIB(self); + PyObject *unused_data = self->unused_data; + Py_INCREF(unused_data); + LEAVE_ZLIB(self); + return unused_data; +} + +static PyObject * +Decomp_get_unconsumed_tail(decompobject *self, void *Py_UNUSED(closure)) +{ + ENTER_ZLIB(self); + PyObject *unconsumed_tail = self->unconsumed_tail; + Py_INCREF(unconsumed_tail); + LEAVE_ZLIB(self); + return unconsumed_tail; +} + +static PyObject * +Decomp_get_eof(decompobject *self, void *Py_UNUSED(closure)) +{ + ENTER_ZLIB(self); + char eof = self->eof; + LEAVE_ZLIB(self); + return PyBool_FromLong(eof); +} + +static PyGetSetDef Decomp_properties[] = { + {"unused_data", (getter)Decomp_get_unused_data, NULL, + "Data found after the end of the compressed stream.", NULL}, + {"unconsumed_tail", (getter)Decomp_get_unconsumed_tail, NULL, + Decomp_unconsumed_tail, NULL}, + {"eof", (getter)Decomp_get_eof, NULL, + "True if the end-of-stream marker has been reached.", NULL}, {NULL}, }; @@ -1516,7 +1545,7 @@ static PyTypeObject IsalZlibDecompType = { .tp_flags = Py_TPFLAGS_DEFAULT, .tp_dealloc = (destructor)Decomp_dealloc, .tp_methods = Decomp_methods, - .tp_members = Decomp_members, + .tp_getset = Decomp_properties, }; #define GzipReader_HEADER 1 diff --git a/tests/test_freethreading.py b/tests/test_freethreading.py index 10d1d53b..fbafd390 100644 --- a/tests/test_freethreading.py +++ b/tests/test_freethreading.py @@ -108,3 +108,87 @@ def test_isal_compress_decompress_threaded(compress, decompress): ] for future in concurrent.futures.as_completed(futures): future.result() # To fire assertion error if there is one + + +def run_in_threads(function): + """Call function(index) in NUM_THREADS threads at the same time.""" + start = threading.Barrier(parties=NUM_THREADS) + + def run(index): + start.wait() + return function(index) + + with concurrent.futures.ThreadPoolExecutor(NUM_THREADS) as executor: + return list(executor.map(run, range(NUM_THREADS))) + + +def test_decompressobj_shared_between_threads_detects_format(): + compressed = isal_zlib.compress(HAMLET_SCENE, wbits=31) + for _ in range(NUM_ITERATIONS): + # wbits=47 detects gzip or zlib in the first decompress() call. + decompressobj = isal_zlib.decompressobj(wbits=47) + results = run_in_threads( + lambda index: decompressobj.decompress(compressed)) + assert sorted(results) == [b""] * (NUM_THREADS - 1) + [HAMLET_SCENE] + assert decompressobj.unused_data == compressed * (NUM_THREADS - 1) + + +def test_decompressobj_attributes_while_decompressing(): + data = HAMLET_SCENE * 100 + compressed = isal_zlib.compress(data) + b"unused" + decompressobj = isal_zlib.decompressobj() + done = threading.Event() + + def decompress(): + output = [] + unconsumed = compressed + while not decompressobj.eof: + output.append(decompressobj.decompress(unconsumed, 100)) + unconsumed = decompressobj.unconsumed_tail + # Each call replaces unused_data with a longer bytes object. + for _ in range(NUM_ITERATIONS): + decompressobj.decompress(b"!") + return b"".join(output) + + def run(index): + if index == 0: + try: + return decompress() + finally: + done.set() + while not done.is_set(): + decompressobj.eof + decompressobj.unused_data + decompressobj.unconsumed_tail + + assert run_in_threads(run)[0] == data + assert decompressobj.unused_data == b"unused" + b"!" * NUM_ITERATIONS + + +def test_igzip_decompressor_attributes_while_decompressing(): + data = HAMLET_SCENE * 100 + compressed = igzip_lib.compress(data, flag=igzip_lib.COMP_GZIP) + decompressor = igzip_lib.IgzipDecompressor(flag=igzip_lib.DECOMP_GZIP) + done = threading.Event() + + def decompress(): + output = [decompressor.decompress(compressed + b"unused", 100)] + while not decompressor.eof: + output.append(decompressor.decompress(b"", 100)) + return b"".join(output) + + def run(index): + if index == 0: + try: + return decompress() + finally: + done.set() + while not done.is_set(): + decompressor.eof + decompressor.unused_data + decompressor.needs_input + decompressor.crc + + assert run_in_threads(run)[0] == data + assert decompressor.unused_data == b"unused" + assert decompressor.crc == isal_zlib.crc32(data) From 192bbc114ff5a52eb982c737f7f4893358665857 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Sun, 18 Oct 2026 12:21:00 -0400 Subject: [PATCH 29/73] Keep isal_zlib importable with ISA-L 2.30 isal_zlib_header_init was added in ISA-L 2.31. Read the Docs builds on Ubuntu 22.04, which ships libisal 2.30, so isal_zlib failed to import there and the API documentation for isal_zlib, igzip and igzip_threaded was empty. The function only zeroes the four fields of the struct, and all four are set right after the call, so it is not needed. Run the docs check on the same Ubuntu release as Read the Docs so CI catches this kind of breakage. --- .github/workflows/ci.yml | 4 +++- src/isal/isal_zlibmodule.c | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e82cebfd..6749c165 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,9 @@ jobs: tox_env: - docs - twine_check - runs-on: ubuntu-latest + # Use the same Ubuntu release as build.os in .readthedocs.yml, so the + # documentation is checked against the libisal that Read the Docs uses. + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 50e6f740..c62f7280 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -763,7 +763,8 @@ isal_zlib_compressobj_impl(PyObject *module, int level, int method, int wbits, zdict->buf, (uint32_t)zdict->len); if (err == COMP_OK) { if (flag == IGZIP_ZLIB && zdict->len != 0) { - isal_zlib_header_init(&self->zlib_header); + // Set every field instead of calling isal_zlib_header_init, + // which ISA-L 2.30 does not have. self->zlib_header.info = hist_bits - 8; self->zlib_header.level = level == 0 ? 0 : 1; self->zlib_header.dict_flag = 1; From f2e057d93573c103929b6a4a8a87f672c870541d Mon Sep 17 00:00:00 2001 From: ptweezy Date: Sun, 18 Oct 2026 12:48:25 -0400 Subject: [PATCH 30/73] Raise write errors of igzip_threaded writers instead of waiting forever When raw.write() raised an error in the thread that writes the compressed blocks, that thread exited without calling task_done(). As a result, flush() and close() waited forever, and write() blocked when an input queue was full. The thread now stores the error and drops later blocks until stop() sends the None sentinel, so write(), flush(), and close() can raise the error in the calling thread. After an error, close() closes raw only if closefd is true, so it no longer closes a file object passed to igzip_threaded.open(). --- CHANGELOG.rst | 5 +++ src/isal/igzip_threaded.py | 49 +++++++++++++++------- tests/test_igzip_threaded.py | 79 ++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0600575f..79ea58dc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -55,6 +55,11 @@ version 1.8.1-dev ISA-L rather than its optimized aarch64 assembly. On an Apple M4, decompression is 1.6 to 3 times faster, CRC-32 is 50 times faster, and Adler-32 is 7 times faster. ++ Fix ``igzip_threaded`` writers that waited forever when writing to the + underlying file failed, for example, because the disk was full. A later call + to ``write()``, ``flush()``, or ``close()`` raises the error. After an error, + ``close()`` no longer closes a file object that you passed to + ``igzip_threaded.open``. version 1.8.0 ----------------- diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index 778ca2a5..784fefcf 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -346,26 +346,32 @@ def flush(self): # Wait for all data to be written for out_q in self.output_queues: out_q.join() + if self.exception: + raise self.exception self.raw.flush() def close(self) -> None: if self._closed: return - self.flush() - self.raw.write(isal_zlib.compress(b"", wbits=-15)) - trailer = struct.pack(" bool: @@ -414,7 +420,7 @@ def _write(self): compressed, crc, data_length = item self._crc = isal_zlib.crc32_combine(self._crc, crc, data_length) self._size += data_length - self.raw.write(compressed) + self._write_compressed(compressed) output_queue.task_done() index += 1 @@ -443,9 +449,22 @@ def _compress_and_write(self): data_length = len(data) self._crc = isal_zlib.crc32_combine(self._crc, crc, data_length) self._size += data_length - self.raw.write(compressed) + self._write_compressed(compressed) in_queue.task_done() + def _write_compressed(self, compressed: bytes): + # After a write error, this thread keeps taking blocks from the queues + # and drops them until stop() sends the None sentinel. That way, no + # thread waits forever on a full queue, and write(), flush(), and + # close() can raise the error in the calling thread. + if self.exception: + return + try: + self.raw.write(compressed) + except Exception as e: + with self.lock: + self.exception = e + def _set_error_and_empty_queue(self, error, q): with self.lock: self.exception = error diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index cec924f8..9b9649a0 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -5,6 +5,7 @@ # This file is part of python-isal which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. +import errno import gzip import io import itertools @@ -14,6 +15,7 @@ import sys import tempfile import threading +import time import types import zlib from pathlib import Path @@ -115,6 +117,83 @@ def test_threaded_write_error(threads): error.match("Compressed output exceeds buffer size") +class FullDiskStream(io.BytesIO): + """Accepts the 10-byte gzip header, then fails like a full disk.""" + before_error = None + + def write(self, b): + if self.tell() == 0: + return super().write(b) + if self.before_error: + self.before_error() + raise OSError(errno.ENOSPC, os.strerror(errno.ENOSPC)) + + +def assert_writer_closed(f): + assert f.closed + workers = [f.output_worker] + f.compression_workers + assert not any(worker.is_alive() for worker in workers) + # Make sure double closing does not raise errors + f.close() + + +@pytest.mark.timeout(5) +@pytest.mark.parametrize("threads", [1, 3]) +def test_threaded_write_error_with_full_queues(threads): + raw = FullDiskStream() + f = igzip_threaded._ThreadedGzipWriter(raw, threads=threads) + queues = f.input_queues + f.output_queues + + def wait_until_queues_are_full(): + while not all(q.full() for q in queues): + time.sleep(0.001) + + # Fail when all queues are full, so write() usually waits for room in a + # full input queue. + raw.before_error = wait_until_queues_are_full + with pytest.raises(OSError) as error: + while True: + f.write(b"x") + assert error.value.errno == errno.ENOSPC + with pytest.raises(OSError): + f.close() + assert_writer_closed(f) + + +@pytest.mark.timeout(5) +@pytest.mark.parametrize("threads", [1, 3]) +def test_threaded_flush_error(threads): + raw = FullDiskStream() + f = igzip_threaded.open(raw, "wb", threads=threads) + f.write(b"x") + with pytest.raises(OSError) as error: + f.flush() + assert error.value.errno == errno.ENOSPC + # Later calls raise the error again. + with pytest.raises(OSError): + f.flush() + with pytest.raises(OSError): + f.close() + assert_writer_closed(f.raw) + # The test passed the stream to open(), so close() leaves it open. + assert not raw.closed + + +@pytest.mark.timeout(5) +@pytest.mark.parametrize("threads", [1, 3]) +def test_threaded_close_error(monkeypatch, threads): + raw = FullDiskStream() + # Make open() treat raw as a file that it opened by name. + monkeypatch.setattr(igzip_threaded, "builtins", + types.SimpleNamespace(open=lambda name, mode: raw)) + with pytest.raises(OSError) as error: + with igzip_threaded.open("full.gz", "wb", threads=threads) as f: + f.write(b"x") + assert error.value.errno == errno.ENOSPC + assert_writer_closed(f.raw) + assert raw.closed + + def test_close_reader(): tmp = io.BytesIO(Path(TEST_FILE).read_bytes()) f = igzip_threaded._ThreadedGzipReader(tmp, "rb") From 7fe752546b1688cfc0e6c1127996509ecc4b832d Mon Sep 17 00:00:00 2001 From: ptweezy Date: Mon, 19 Oct 2026 10:15:58 -0400 Subject: [PATCH 31/73] Fix igzip_threaded writers at level 0 for data that does not compress At level 0, isal_deflate() codes all data with one precomputed Huffman table and never falls back to stored blocks, so random data grows by about 23%. Each compressor had an output buffer only about 10% larger than the block size, so compress_and_crc() raised OverflowError. At level 0, make the buffer twice the block size plus 500 bytes, because deflate uses at most 15 bits per input byte. --- CHANGELOG.rst | 4 ++++ src/isal/igzip_threaded.py | 10 +++++++++- tests/test_igzip_threaded.py | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 79ea58dc..0a7ab759 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -40,6 +40,10 @@ version 1.8.1-dev + ``igzip_threaded.open`` raises ``ValueError`` for a ``block_size`` of about 3.6 GiB or more and ``TypeError`` for a ``compresslevel`` that is not an integer. Both raised ``SystemError`` before. ++ Fix ``igzip_threaded`` writers at compression level 0, which raised + ``OverflowError`` for data that does not compress, such as random bytes or + already compressed files. At compression level 0, ``block_size`` must now be + less than about 2 GiB. + Extraordinarily large gzip headers with either a very long comment or filename will now cause an ``OverflowError`` rather than use more and more memory. The size is limited to the internal buffer size which is currently diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index 784fefcf..cba3cdcf 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -240,7 +240,15 @@ def __init__(self, self.previous_block = b"" # Deflating random data results in an output a little larger than the # input. Making the output buffer 10% larger is sufficient overkill. - compress_buffer_size = block_size + max(block_size // 10, 500) + # Level 0 is the exception: it codes all data with one precomputed + # Huffman table and never falls back to stored blocks, so random data + # grows by more than 20%. Deflate uses at most 15 bits per input byte, + # so twice the block size plus 500 bytes for the block header, the + # end-of-block code, and the sync flush marker always fits. + if level == 0: + compress_buffer_size = 2 * block_size + 500 + else: + compress_buffer_size = block_size + max(block_size // 10, 500) self.block_size = block_size self.compressors: List[isal_zlib._ParallelCompress] = [ isal_zlib._ParallelCompress(buffersize=compress_buffer_size, diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index a5e7ea0f..572ece53 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -103,6 +103,21 @@ def test_threaded_write_oversized_block_no_error(threads): assert data == decompressed +@pytest.mark.timeout(5) +@pytest.mark.parametrize(["threads", "block_size"], + itertools.product([1, 3], [8 * 1024, 1024 * 1024])) +def test_threaded_write_incompressible_level_0(tmp_path, threads, block_size): + # Level 0 codes all data with one precomputed Huffman table and never + # falls back to stored blocks, so random bytes grow by more than 20%. + data = os.urandom(3 * block_size) + test_file = tmp_path / "output.gz" + with igzip_threaded.open(test_file, "wb", compresslevel=0, + threads=threads, block_size=block_size) as f: + f.write(data) + with gzip.open(test_file, "rb") as f: + assert f.read() == data + + @pytest.mark.timeout(5) @pytest.mark.parametrize("threads", [1, 3]) def test_threaded_write_error(threads): From 80b86bad75e85619def56c64da59506957c73900 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Mon, 19 Oct 2026 13:18:19 -0400 Subject: [PATCH 32/73] Check for a closed file faster in IGzipFile.write IGzipFile.write called _check_not_closed() for each write, which costs two Python calls. It now calls it only when fileobj is None, as readline() does. On CPython 3.14.8, writing FASTQ data or Python source code one line at a time is 1.1 times faster. --- CHANGELOG.rst | 3 +++ src/isal/igzip.py | 5 ++++- tests/test_igzip.py | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 79ea58dc..a79f1eb4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,9 @@ version 1.8.1-dev + Speed up reading lines from binary ``igzip`` files through iteration, ``readlines()``, and ``readline()``. On an Apple M4, reading FASTQ data is 1.13 times faster, and reading lines of a few bytes is 1.5 times faster. ++ Speed up each ``write()`` call to binary ``igzip`` files on CPython. On an + AMD Ryzen 7 9800X3D with Python 3.14.8, writing FASTQ data or Python source + code one line at a time is 1.1 times faster. + Fix zlib streams compressed with a preset dictionary so they can be decoded by standard zlib, including builds linked against older ISA-L releases. + Fix backward seeking in igzip files by discarding buffered compressed data diff --git a/src/isal/igzip.py b/src/isal/igzip.py index 52717d67..6496a465 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -204,7 +204,10 @@ def _write_gzip_header(self, compresslevel=_COMPRESS_LEVEL_TRADEOFF): super()._write_gzip_header() def write(self, data): - self._check_not_closed() + # GzipFile.closed tests whether fileobj is None, so testing fileobj + # here skips two Python calls for each write. + if self.fileobj is None: + self._check_not_closed() if self.mode != WRITE: import errno raise OSError(errno.EBADF, "write() on read-only IGzipFile object") diff --git a/tests/test_igzip.py b/tests/test_igzip.py index 4ebc477e..67e6f25f 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -52,6 +52,24 @@ def test_write_readonly_file(): error.match(r"write\(\) on read-only IGzipFile object") +def test_igzipfile_write_skips_check_not_closed_when_open(monkeypatch): + calls = [] + check_not_closed = igzip.IGzipFile._check_not_closed + monkeypatch.setattr(igzip.IGzipFile, "_check_not_closed", + lambda self: calls.append(1) or check_not_closed(self)) + output = io.BytesIO() + igzip_file = igzip.IGzipFile(fileobj=output, mode="wb") + for _ in range(100): + assert igzip_file.write(b"a\n") == 2 + # Open files skip the check, which costs two Python calls per write + assert calls == [] + igzip_file.close() + assert gzip.decompress(output.getvalue()) == b"a\n" * 100 + with pytest.raises(ValueError, match="I/O operation on closed file"): + igzip_file.write(b"a\n") + assert calls == [1] + + def test_igzip_reader_readall(): data = io.BytesIO(COMPRESSED_DATA) test = igzip._IGzipReader(data) From 20bd255b60c0ed6bc415372cb4591487638aafa0 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 20 Oct 2026 09:49:20 -0400 Subject: [PATCH 33/73] Use fixed Huffman codes for Z_FIXED at isal_zlib level 0 isal_zlib.compressobj() gave a warning for every strategy other than Z_DEFAULT_STRATEGY and then ignored it. At level 0, ISA-L compresses with the Huffman table that is set on the stream, so calling isal_deflate_set_hufftables() with IGZIP_HUFFTABLE_STATIC gives fixed Huffman blocks. This avoids the table of about 110 bytes that level 0 writes at the start of each block, which matters when small messages are flushed one at a time, as in websocket permessage-deflate. Blocks of a few KiB or more can come out larger with the fixed codes. Levels 1 to 3 choose the codes for each block and ignore that table, so Z_FIXED keeps its warning there. The warning text, the docstring, the README, and the CHANGELOG describe the new behavior. --- CHANGELOG.rst | 8 +++++ README.rst | 5 +-- src/isal/isal_zlibmodule.c | 23 +++++++++++--- tests/test_compat.py | 64 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b2845082..30e57e29 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,14 @@ Changelog version 1.8.1-dev ----------------- ++ ``isal_zlib.compressobj`` supports ``strategy=Z_FIXED`` at compression + level 0. Level 0 then uses fixed Huffman codes instead of starting each + block with a Huffman table of about 110 bytes. This shrinks small flushed + writes, such as websocket messages: JSON messages of about 440 bytes, each + followed by ``flush(Z_SYNC_FLUSH)``, compress to an average of 75 instead + of 188 bytes. Blocks of a few KiB or more can be larger: about 5% for JSON + and 10% for Python source code at 64 KiB. Other strategies, and ``Z_FIXED`` + at levels 1 to 3, still give a warning and use the default strategy. + Speed up ``import isal.igzip`` and ``import isal.igzip_threaded`` by importing ``argparse`` and ``shutil`` only when the command line interface runs, and by dropping the ``multiprocessing`` import. diff --git a/README.rst b/README.rst index d90f791c..0ac85ac4 100644 --- a/README.rst +++ b/README.rst @@ -139,8 +139,9 @@ Differences with zlib and gzip modules ``FINISH_FLUSH``. Other flush modes are not supported and will raise errors. + ``zlib.Z_DEFAULT_STRATEGY``, ``zlib.Z_RLE`` etc. are exposed as ``isal_zlib.Z_DEFAULT_STRATEGY``, ``isal_zlib.Z_RLE`` etc. for compatibility - reasons. However, ``isal_zlib`` only supports a default strategy and will - give warnings when other strategies are used. + reasons. However, ``isal_zlib`` supports only the default strategy, and + ``Z_FIXED`` at compression level 0. It gives a warning and uses the default + strategy for other strategies, and for ``Z_FIXED`` at levels 1 to 3. + ``zlib`` supports different memory levels from 1 to 9 (with 8 default). ``isal_zlib`` supports memory levels smallest, small, medium, large and largest. These have been mapped to levels 1, 2-3, 4-6, 7-8 and 9. So diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index c30f9d5d..87e1313d 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -707,11 +707,11 @@ isal_zlib_compressobj_impl(PyObject *module, int level, int method, int wbits, method); goto error; } - if (strategy != Z_DEFAULT_STRATEGY){ + if (strategy != Z_DEFAULT_STRATEGY && !(strategy == Z_FIXED && level == 0)){ err = PyErr_WarnEx( PyExc_UserWarning, - "Only one strategy is supported when using isal_zlib. Using the " - "default strategy.", + "isal_zlib supports only Z_DEFAULT_STRATEGY, and Z_FIXED at " + "compression level 0. Using the default strategy.", 1); if (err == -1) // Warning was turned into an exception. @@ -754,6 +754,17 @@ isal_zlib_compressobj_impl(PyObject *module, int level, int method, int wbits, self->zst.level = level; self->zst.hist_bits = (uint16_t)hist_bits; self->zst.gzip_flag = (uint16_t)flag; + if (strategy == Z_FIXED && level == 0) { + /* Level 0 encodes every block with the stream's Huffman table, so the + static table gives fixed Huffman blocks only. Levels 1 to 3 choose + the codes for each block themselves and ignore this setting. */ + err = isal_deflate_set_hufftables(&(self->zst), NULL, + IGZIP_HUFFTABLE_STATIC); + if (err != COMP_OK) { + isal_deflate_error(err); + goto error; + } + } self->is_initialised = 1; if (zdict->buf == NULL) { @@ -1239,8 +1250,10 @@ PyDoc_STRVAR(isal_zlib_compressobj__doc__, " Valid values range from 1 to 9. Higher values result in higher memory\n" " usage, faster compression, and smaller output.\n" " strategy\n" -" Used to tune the compression algorithm. Not supported by ISA-L.\n" -" Only a default strategy is used.\n" +" Used to tune the compression algorithm. At level 0, Z_FIXED uses fixed\n" +" Huffman codes, which gives smaller output when small amounts of data are\n" +" flushed. Other strategies, and Z_FIXED at levels 1 to 3, give a warning\n" +" and use the default strategy.\n" " zdict\n" " The predefined compression dictionary - a sequence of bytes\n" " containing subsequences that are likely to occur in the input data."); diff --git a/tests/test_compat.py b/tests/test_compat.py index 89f19313..cf9251f0 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -11,7 +11,9 @@ import gzip import itertools +import random import threading +import warnings import zlib from pathlib import Path @@ -123,6 +125,68 @@ def test_compressobj_level_0_flush_after_finish(): assert zlib.decompress(compressed) == b"data" +@pytest.mark.parametrize("flush_mode", [isal_zlib.Z_SYNC_FLUSH, + isal_zlib.Z_FULL_FLUSH]) +@pytest.mark.parametrize(["strategy", "btype"], + [(isal_zlib.Z_DEFAULT_STRATEGY, 2), + (isal_zlib.Z_FIXED, 1)]) +def test_compressobj_level_0_block_type(strategy, btype, flush_mode): + with warnings.catch_warnings(): + warnings.simplefilter("error") + compressor = isal_zlib.compressobj(0, wbits=-15, strategy=strategy) + decompressor = zlib.decompressobj(wbits=-15) + for i in range(0, 64 * 1024, 4096): + data = DATA[i:i + 4096] + compressed = compressor.compress(data) + compressor.flush(flush_mode) + # Sync and full flushes end on a byte boundary, so each output starts + # with a new block header. Bits 1 and 2 of the first byte hold BTYPE: + # 1 for fixed Huffman codes, 2 for dynamic Huffman codes. + assert compressed[0] >> 1 & 3 == btype + assert decompressor.decompress(compressed) == data + + +# Gzip has no preset dictionary, so zdict is only tested with zlib and raw. +@pytest.mark.parametrize(["wbits", "use_zdict"], + [(wbits, False) for wbits in WBITS_RANGE] + + [(wbits, True) for wbits in WBITS_RANGE + if not 25 <= wbits <= 31]) +def test_compressobj_level_0_z_fixed_round_trip(wbits, use_zdict): + kwargs = {"zdict": DATA[:1024]} if use_zdict else {} + with warnings.catch_warnings(): + warnings.simplefilter("error") + compressor = isal_zlib.compressobj( + 0, wbits=wbits, strategy=isal_zlib.Z_FIXED, **kwargs) + # Random bytes use all literal codes, including the 9-bit ones. + data = DATA[:64 * 1024] + random.Random(0).randbytes(64 * 1024) + flush_modes = itertools.cycle([isal_zlib.Z_SYNC_FLUSH, + isal_zlib.Z_FULL_FLUSH, + isal_zlib.Z_NO_FLUSH]) + compressed = b"" + for i, mode in zip(range(0, len(data), 1000), flush_modes): + compressed += (compressor.compress(data[i:i + 1000]) + + compressor.flush(mode)) + compressed += compressor.flush() + decompressor = zlib.decompressobj(wbits=wbits, **kwargs) + assert decompressor.decompress(compressed) == data + assert decompressor.eof + + +@pytest.mark.parametrize(["level", "strategy"], + list(itertools.product(range(4), + [isal_zlib.Z_FILTERED, + isal_zlib.Z_HUFFMAN_ONLY, + isal_zlib.Z_RLE])) + + [(level, isal_zlib.Z_FIXED) for level in range(1, 4)]) +def test_compressobj_unsupported_strategy(level, strategy): + data = DATA[:64 * 1024] + with pytest.warns(UserWarning, match="Using the default strategy"): + compressor = isal_zlib.compressobj(level, strategy=strategy) + default_compressor = isal_zlib.compressobj(level) + compressed = compressor.compress(data) + compressor.flush() + assert compressed == (default_compressor.compress(data) + + default_compressor.flush()) + + @pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel"], itertools.product([128 * 1024], range(4), WBITS_RANGE, range(1, 10))) From 61f920c16f2a05ebb0c9be3bea0602f76e0b2ce9 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 20 Oct 2026 09:32:45 -0400 Subject: [PATCH 34/73] Raise TypeError for bad arguments to crc32_combine and _GzipReader PyArg_ParseTuple() and PyArg_ParseTupleAndKeywords() return 0 on failure, but the code checked for a negative value. crc32_combine() raised SystemError, and _GzipReader.read() and seek() kept running with the exception set, which ended in SystemError. --- CHANGELOG.rst | 4 ++++ src/isal/isal_zlibmodule.c | 6 +++--- tests/test_compat.py | 7 +++++++ tests/test_igzip.py | 12 ++++++++++++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b2845082..449a39d0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -73,6 +73,10 @@ version 1.8.1-dev ``isal_zlib`` decompressors and ``igzip_lib.IgzipDecompressor`` from other threads is now safe, and so is calling ``decompress()`` from several threads at the same time when ``wbits`` detects the format. ++ ``isal_zlib.crc32_combine`` and the ``read()`` and ``seek()`` methods of the + private ``isal_zlib._GzipReader`` class raise ``TypeError`` instead of + ``SystemError`` for arguments of the wrong type or number. + ``igzip.IGzipFile`` was not affected. version 1.8.0 ----------------- diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index c30f9d5d..c1a0f5db 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -280,7 +280,7 @@ isal_zlib_crc32_combine(PyObject *module, PyObject *args) { uint32_t crc2 = 0; Py_ssize_t crc2_length = 0; static char *format = "IIn:crc32combine"; - if (PyArg_ParseTuple(args, format, &crc1, &crc2, &crc2_length) < 0) { + if (!PyArg_ParseTuple(args, format, &crc1, &crc2, &crc2_length)) { return NULL; } return PyLong_FromUnsignedLong( @@ -1974,7 +1974,7 @@ GzipReader_seek(GzipReader *self, PyObject *args, PyObject *kwargs) Py_ssize_t whence = SEEK_SET; static char *keywords[] = {"offset", "whence", NULL}; static char format[] = {"n|n:GzipReader.seek"}; - if (PyArg_ParseTupleAndKeywords(args, kwargs, format, keywords, &offset, &whence) < 0) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, keywords, &offset, &whence)) { return NULL; } // Recalculate offset as an absolute file position. @@ -2133,7 +2133,7 @@ static PyObject * GzipReader_read(GzipReader *self, PyObject *args) { Py_ssize_t size = -1; - if (PyArg_ParseTuple(args, "|n:GzipReader.read", &size) < 0) { + if (!PyArg_ParseTuple(args, "|n:GzipReader.read", &size)) { return NULL; } if (size < 0) { diff --git a/tests/test_compat.py b/tests/test_compat.py index 89f19313..3078dfd8 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -50,6 +50,13 @@ def test_adler32(data_size, value): assert zlib.adler32(data, value) == isal_zlib.adler32(data, value) +@pytest.mark.parametrize("args", [("a", 1, 2), (1, "a", 2), (1, 2, "a"), + (1, 2)]) +def test_crc32_combine_wrong_arguments(args): + with pytest.raises(TypeError): + isal_zlib.crc32_combine(*args) + + @pytest.mark.parametrize(["data_size", "level"], itertools.product(DATA_SIZES, range(4))) def test_compress(data_size, level): diff --git a/tests/test_igzip.py b/tests/test_igzip.py index 67e6f25f..35184df1 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -82,6 +82,18 @@ def test_igzip_reader_read_zero(): assert test.read(0) == b"" +@pytest.mark.parametrize(["method", "args"], [ + ("read", ("x",)), + ("read", (1, 2)), + ("seek", ("x",)), + ("seek", (0, "x")), +]) +def test_igzip_reader_wrong_arguments(method, args): + test = igzip._IGzipReader(io.BytesIO(COMPRESSED_DATA)) + with pytest.raises(TypeError): + getattr(test, method)(*args) + + def test_igzipfile_read_truncated(): # Chop of trailer (8 bytes) and part of DEFLATE stream data = io.BytesIO(COMPRESSED_DATA[:-10]) From e03bb0a9d0433f47de517c4004b620a118fd172f Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 20 Oct 2026 09:35:37 -0400 Subject: [PATCH 35/73] Prevent creating Compress and Decompress objects directly on PyPy CPython does not let you call a static type without tp_new, but PyPy does. The object it creates is not initialized, so type(compressobj())() followed by compress() crashed PyPy. Py_TPFLAGS_DISALLOW_INSTANTIATION makes PyPy raise TypeError like CPython. Python 3.9 does not have the flag, so it is defined as 0 there. --- CHANGELOG.rst | 5 +++++ src/isal/isal_zlibmodule.c | 12 ++++++++++-- tests/test_compat.py | 7 +++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 449a39d0..9b5bc556 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -77,6 +77,11 @@ version 1.8.1-dev private ``isal_zlib._GzipReader`` class raise ``TypeError`` instead of ``SystemError`` for arguments of the wrong type or number. ``igzip.IGzipFile`` was not affected. ++ Fix a crash on PyPy when you create an ``isal_zlib.Compress`` or + ``isal_zlib.Decompress`` object by calling the type, for example, + ``isal_zlib.Compress()``, and then use it. Calling the type raises + ``TypeError`` now, as it does on CPython. Use ``compressobj()`` and + ``decompressobj()`` instead. version 1.8.0 ----------------- diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index c1a0f5db..981fd251 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -60,6 +60,14 @@ Changes compared to CPython: #define MEMORYVIEW_READINTO_FLAGS PyBUF_WRITE #endif +/* Only compressobj() and decompressobj() create Compress and Decompress + objects. CPython does not let you call a static type without tp_new, but + PyPy does, and the uninitialized object crashes when you use it. This flag + makes PyPy raise TypeError too. Python 3.9 does not have the flag. */ +#ifndef Py_TPFLAGS_DISALLOW_INSTANTIATION +#define Py_TPFLAGS_DISALLOW_INSTANTIATION 0 +#endif + static PyTypeObject IsalZlibCompType; static PyTypeObject IsalZlibDecompType; @@ -1486,7 +1494,7 @@ static PyTypeObject IsalZlibCompType = { .tp_name = "isal_zlib.Compress", .tp_doc = "Object returned by isal_zlib.compressobj", .tp_basicsize = sizeof(compobject), - .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_dealloc = (destructor)Comp_dealloc, .tp_methods = comp_methods, }; @@ -1543,7 +1551,7 @@ static PyTypeObject IsalZlibDecompType = { .tp_name = "isal_zlib.Decompress", .tp_doc = "Object returned by isal_zlib.compressobj.", .tp_basicsize = sizeof(decompobject), - .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_dealloc = (destructor)Decomp_dealloc, .tp_methods = Decomp_methods, .tp_getset = Decomp_properties, diff --git a/tests/test_compat.py b/tests/test_compat.py index 3078dfd8..45179f20 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -57,6 +57,13 @@ def test_crc32_combine_wrong_arguments(args): isal_zlib.crc32_combine(*args) +@pytest.mark.parametrize("factory", + [isal_zlib.compressobj, isal_zlib.decompressobj]) +def test_compress_and_decompress_types_cannot_be_called(factory): + with pytest.raises(TypeError): + type(factory())() + + @pytest.mark.parametrize(["data_size", "level"], itertools.product(DATA_SIZES, range(4))) def test_compress(data_size, level): From d6799d56515e814d40680ef743160b339fa10a5b Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 20 Oct 2026 10:30:49 -0400 Subject: [PATCH 36/73] Refuse read-only buffers in _GzipReader.readinto readinto() requested a buffer without PyBUF_WRITABLE, so it decompressed into bytes objects and other read-only buffers. On CPython, this changed immutable objects, including cached single-byte bytes objects. It now raises TypeError, like the readinto() methods of io. --- CHANGELOG.rst | 3 +++ src/isal/isal_zlibmodule.c | 6 +++++- tests/test_igzip.py | 8 ++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9b5bc556..94d0f148 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -82,6 +82,9 @@ version 1.8.1-dev ``isal_zlib.Compress()``, and then use it. Calling the type raises ``TypeError`` now, as it does on CPython. Use ``compressobj()`` and ``decompressobj()`` instead. ++ Fix ``readinto()`` of the private ``isal_zlib._GzipReader`` class, which + wrote into read-only objects such as ``bytes``. It raises ``TypeError`` now. + ``igzip.IGzipFile`` was not affected. version 1.8.0 ----------------- diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 981fd251..86658563 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1960,7 +1960,11 @@ static PyObject * GzipReader_readinto(GzipReader *self, PyObject *buffer_obj) { Py_buffer view; - if (PyObject_GetBuffer(buffer_obj, &view, PyBUF_SIMPLE) < 0) { + if (PyObject_GetBuffer(buffer_obj, &view, PyBUF_WRITABLE) < 0) { + /* Raise TypeError, like the readinto() methods of io. */ + PyErr_Format(PyExc_TypeError, + "readinto() argument must be read-write bytes-like " + "object, not %.200s", Py_TYPE(buffer_obj)->tp_name); return NULL; } uint8_t *buffer = view.buf; diff --git a/tests/test_igzip.py b/tests/test_igzip.py index 35184df1..b8e0a2eb 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -94,6 +94,14 @@ def test_igzip_reader_wrong_arguments(method, args): getattr(test, method)(*args) +def test_igzip_reader_readinto_read_only(): + test = igzip._IGzipReader(io.BytesIO(COMPRESSED_DATA)) + buffer = bytes(10) + with pytest.raises(TypeError): + test.readinto(buffer) + assert buffer == bytes(10) + + def test_igzipfile_read_truncated(): # Chop of trailer (8 bytes) and part of DEFLATE stream data = io.BytesIO(COMPRESSED_DATA[:-10]) From 6442f229c962cba1993a9b8c22fd8f686e11370a Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 20 Oct 2026 10:28:08 -0400 Subject: [PATCH 37/73] Detect the format of wbits=40..47 streams from their first byte decompressobj() with a wbits of 40 to 47 chose zlib when the first decompress() call got fewer than 2 bytes, and then failed on gzip data. A zlib stream cannot start with 31, so the first byte is enough to tell gzip from zlib. An empty call now returns b"" and waits for that byte. The one-shot decompress() uses the same check. Invalid data whose first byte is 31 and whose second byte is not 139 is now read as gzip. As with wbits=31, a Decompress object then raises an error when 10 bytes have arrived instead of 2. If the data ends before that, decompress() and flush() return b"", and eof stays False. The exception is still isal_zlib.error, but some messages change. --- CHANGELOG.rst | 4 ++++ src/isal/isal_zlibmodule.c | 10 +++++++++- tests/test_compat.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b2845082..c2f04b7e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -40,6 +40,10 @@ version 1.8.1-dev input buffer when ``max_length`` split the end of a stream. At the end of a stream, ``isal_zlib.Decompress.unconsumed_tail`` is empty, matching the CPython fix for gh-158169. ++ Fix ``isal_zlib.decompressobj`` with a ``wbits`` of 40 to 47, which detects + zlib or gzip data. It failed on gzip data when the first ``decompress()`` + call got fewer than 2 bytes, for example, when the data arrived byte by + byte. + ``igzip_threaded.open`` raises ``ValueError`` for a ``block_size`` of about 3.6 GiB or more and ``TypeError`` for a ``compresslevel`` that is not an integer. Both raised ``SystemError`` before. diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index c30f9d5d..77b34e1c 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -141,10 +141,13 @@ static inline int zlib_mem_level_to_isal(int mem_level) { return ZLIB_MEM_LEVEL_TO_ISAL[mem_level]; } +/* A gzip stream starts with 31, 139. A zlib stream cannot start with 31, + because the low 4 bits of its first byte are the compression method 8. + So the first byte is enough, even when the data arrives byte by byte. */ static inline int data_is_gzip(Py_buffer *data){ uint8_t *buf = (uint8_t *)data->buf; - return (data->len > 1) && (buf[0] == 31) && (buf[1] == 139); + return (data->len > 0) && (buf[0] == 31); } @@ -991,6 +994,11 @@ isal_zlib_Decompress_decompress_impl(decompobject *self, Py_buffer *data, ENTER_ZLIB(self); if (!self->method_set) { + if (data->len == 0) { + /* Detect the format when the first byte arrives. */ + LEAVE_ZLIB(self); + return PyBytes_FromStringAndSize(NULL, 0); + } if (data_is_gzip(data)){ self->zst.crc_flag = ISAL_GZIP; } diff --git a/tests/test_compat.py b/tests/test_compat.py index 89f19313..36a042c6 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -185,6 +185,40 @@ def test_unused_data(unused_size, wbits): assert decompressor.unused_data == unused_data +@pytest.mark.parametrize("wbits", [15, 31]) +def test_decompressobj_detects_format_byte_by_byte(wbits): + data = DATA[:1000] + compressor = zlib.compressobj(wbits=wbits) + compressed = compressor.compress(data) + compressor.flush() + decompressor = isal_zlib.decompressobj(wbits=47) + result = b"".join(decompressor.decompress(compressed[i:i + 1]) + for i in range(len(compressed))) + assert result == data + assert decompressor.eof + + +@pytest.mark.parametrize("wbits", [15, 31]) +def test_decompressobj_detects_format_after_empty_input(wbits): + data = DATA[:1000] + compressor = zlib.compressobj(wbits=wbits) + compressed = compressor.compress(data) + compressor.flush() + for module in (zlib, isal_zlib): + decompressor = module.decompressobj(wbits=47) + assert decompressor.decompress(b"") == b"" + assert decompressor.decompress(compressed[:1]) == b"" + assert decompressor.decompress(compressed[1:] + b"unused") == data + assert decompressor.eof + assert decompressor.unused_data == b"unused" + + +@pytest.mark.parametrize("data", [b"", b"\x1f", b"\x78"]) +def test_decompress_detect_format_incomplete(data): + with pytest.raises(zlib.error): + zlib.decompress(data, wbits=47) + with pytest.raises(isal_zlib.error): + isal_zlib.decompress(data, wbits=47) + + @pytest.mark.parametrize("wbits", [-15, 15, 31]) def test_decompressobj_unconsumed_tail_after_eof(wbits): source = b"abcdefghijklmnopqrstuvwxyz" From 2f157d633f130ee5c065f44e87c66245e5ea8f3d Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 20 Oct 2026 10:28:26 -0400 Subject: [PATCH 38/73] Fix flush() before any data in wbits=40..47 decompressors flush() called isal_inflate() before the first data set the format, so ISA-L started a raw deflate block. It then decoded the zlib or gzip header that came next as deflate data and failed with "Invalid deflate block found". flush() now returns b"" until the format is known. --- CHANGELOG.rst | 3 +++ src/isal/isal_zlibmodule.c | 8 ++++++++ tests/test_compat.py | 12 ++++++++++++ 3 files changed, 23 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c2f04b7e..48f8e28f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -44,6 +44,9 @@ version 1.8.1-dev zlib or gzip data. It failed on gzip data when the first ``decompress()`` call got fewer than 2 bytes, for example, when the data arrived byte by byte. ++ Fix ``isal_zlib.decompressobj`` with a ``wbits`` of 40 to 47 after a + ``flush()`` call that came before any data. It then failed on both zlib and + gzip data. + ``igzip_threaded.open`` raises ``ValueError`` for a ``block_size`` of about 3.6 GiB or more and ``TypeError`` for a ``compresslevel`` that is not an integer. Both raised ``SystemError`` before. diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 77b34e1c..796bff8d 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1170,6 +1170,14 @@ isal_zlib_Decompress_flush_impl(decompobject *self, Py_ssize_t length) ENTER_ZLIB(self); + if (!self->method_set) { + /* No data yet, so the format is unknown. isal_inflate would start a + raw deflate block, and then decode the zlib or gzip header that + comes next as deflate data. */ + LEAVE_ZLIB(self); + return PyBytes_FromStringAndSize(NULL, 0); + } + if (PyObject_GetBuffer(self->unconsumed_tail, &data, PyBUF_SIMPLE) == -1) { LEAVE_ZLIB(self); return NULL; diff --git a/tests/test_compat.py b/tests/test_compat.py index 36a042c6..ffddddaf 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -211,6 +211,18 @@ def test_decompressobj_detects_format_after_empty_input(wbits): assert decompressor.unused_data == b"unused" +@pytest.mark.parametrize("wbits", [15, 31]) +def test_decompressobj_detects_format_after_flush(wbits): + data = DATA[:1000] + compressor = zlib.compressobj(wbits=wbits) + compressed = compressor.compress(data) + compressor.flush() + for module in (zlib, isal_zlib): + decompressor = module.decompressobj(wbits=47) + assert decompressor.flush() == b"" + assert decompressor.decompress(compressed) == data + assert decompressor.eof + + @pytest.mark.parametrize("data", [b"", b"\x1f", b"\x78"]) def test_decompress_detect_format_incomplete(data): with pytest.raises(zlib.error): From e3d720a9d0818a89c2acd5d95eb34bdb0e2fa91e Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 20 Oct 2026 09:37:20 -0400 Subject: [PATCH 39/73] Raise the right error classes when several interpreters import isal With m_size 0, igzip_lib and isal_zlib ran their init functions again in each interpreter that imported them, for example, a legacy subinterpreter. Each run replaced the C globals IsalError and BadGzipFile, so only the interpreter that imported the modules last caught the errors with isal_zlib.error, igzip_lib.error, or gzip.BadGzipFile. With m_size -1, other interpreters get a copy of the module dict instead, so all interpreters share IsalError. Each interpreter has its own gzip.BadGzipFile, so _GzipReader now looks it up when it raises the error, and importing isal_zlib no longer imports gzip. --- CHANGELOG.rst | 7 +++++ src/isal/igzip_libmodule.c | 5 +++- src/isal/isal_zlibmodule.c | 52 +++++++++++++++++++++-------------- tests/test_subinterpreters.py | 37 +++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 22 deletions(-) create mode 100644 tests/test_subinterpreters.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b2845082..a08b6f40 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -31,6 +31,13 @@ version 1.8.1-dev bytes-like, such as a ``str``. It raises a ``TypeError`` instead. + Fix a crash after creating ``igzip_lib.IgzipDecompressor`` objects with a ``zdict``. ++ Fix error handling when more than one interpreter imports + ``isal.isal_zlib`` or ``isal.igzip_lib``, for example, the main interpreter + and a legacy subinterpreter that ``Py_NewInterpreter()`` creates. Each + import replaced the error classes that the C code raises, so + ``except isal_zlib.error``, ``except igzip_lib.error``, and + ``except gzip.BadGzipFile`` caught errors only in the interpreter that + imported the modules last. + Fix a hang in ``isal_zlib.Compress``: after ``flush()`` raised an error for an unsupported mode, the next call on the same object never returned. + Fix a crash when you call ``flush(Z_FINISH)`` twice on a level 0 diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index 8d1aa045..b21a4581 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -662,7 +662,10 @@ static struct PyModuleDef igzip_lib_module = { PyModuleDef_HEAD_INIT, "igzip_lib", /* name of module */ igizp_lib_module_documentation, /* module documentation, may be NULL */ - 0, + /* -1 because IsalError is a C global. A subinterpreter then gets a copy + of this module instead of running PyInit_igzip_lib again, which would + replace IsalError with a new class. */ + -1, IgzipLibMethods }; diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index c30f9d5d..3cedcfad 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1732,7 +1732,28 @@ GzipReader_read_from_file(GzipReader *self) #define FNAME 8 #define FCOMMENT 16 -static PyObject *BadGzipFile; // Import BadGzipFile error for consistency +/* Raise gzip.BadGzipFile. Look up the class every time, because each + interpreter has its own gzip module, but all interpreters share the C + globals of this module. */ +static void +raise_bad_gzip_file(const char *format, ...) +{ + PyObject *gzip_module = PyImport_ImportModule("gzip"); + if (gzip_module == NULL) { + return; + } + PyObject *bad_gzip_file = PyObject_GetAttrString(gzip_module, + "BadGzipFile"); + Py_DECREF(gzip_module); + if (bad_gzip_file == NULL) { + return; + } + va_list vargs; + va_start(vargs, format); + PyErr_FormatV(bad_gzip_file, format, vargs); + va_end(vargs); + Py_DECREF(bad_gzip_file); +} static inline uint32_t load_u32_le(void *address) { #if PY_BIG_ENDIAN @@ -1787,7 +1808,7 @@ GzipReader_read_into_buffer(GzipReader *self, uint8_t *out_buffer, size_t out_bu if (!(magic1 == 0x1f && magic2 == 0x8b)) { Py_BLOCK_THREADS; PyObject *magic_obj = PyBytes_FromStringAndSize((char *)current_pos, 2); - PyErr_Format(BadGzipFile, + raise_bad_gzip_file( "Not a gzipped file (%R)", magic_obj); Py_DECREF(magic_obj); @@ -1796,7 +1817,7 @@ GzipReader_read_into_buffer(GzipReader *self, uint8_t *out_buffer, size_t out_bu uint8_t method = current_pos[2]; if (method != 8) { Py_BLOCK_THREADS; - PyErr_SetString(BadGzipFile, "Unknown compression method"); + raise_bad_gzip_file("Unknown compression method"); return -1; } uint8_t flags = current_pos[3]; @@ -1840,8 +1861,7 @@ GzipReader_read_into_buffer(GzipReader *self, uint8_t *out_buffer, size_t out_bu 0, current_pos, header_cursor - current_pos) & 0xFFFF; if (header_crc != crc) { Py_BLOCK_THREADS; - PyErr_Format( - BadGzipFile, + raise_bad_gzip_file( "Corrupted gzip header. Checksums do not " "match: %04x != %04x", crc, header_crc @@ -1895,8 +1915,7 @@ GzipReader_read_into_buffer(GzipReader *self, uint8_t *out_buffer, size_t out_bu current_pos += 4; if (crc != self->state.crc) { Py_BLOCK_THREADS; - PyErr_Format( - BadGzipFile, + raise_bad_gzip_file( "CRC check failed %u != %u", crc, self->state.crc ); @@ -1906,7 +1925,7 @@ GzipReader_read_into_buffer(GzipReader *self, uint8_t *out_buffer, size_t out_bu current_pos += 4; if (length != self->state.total_out) { Py_BLOCK_THREADS; - PyErr_SetString(BadGzipFile, "Incorrect length of data produced"); + raise_bad_gzip_file("Incorrect length of data produced"); return -1; } self->stream_phase = GzipReader_NULL_BYTES; @@ -2278,7 +2297,10 @@ static struct PyModuleDef isal_zlib_module = { PyModuleDef_HEAD_INIT, "isal_zlib", /* name of module */ isal_zlib_module_documentation, /* module documentation, may be NULL */ - 0, + /* -1, as in igzip_lib, because IsalError is a C global. A subinterpreter + then gets a copy of this module instead of running PyInit_isal_zlib + again. */ + -1, IsalZlibMethods, }; @@ -2304,18 +2326,6 @@ PyInit_isal_zlib(void) return NULL; } - PyObject *gzip_module = PyImport_ImportModule("gzip"); - if (gzip_module == NULL) { - return NULL; - } - - BadGzipFile = PyObject_GetAttrString(gzip_module, "BadGzipFile"); - if (BadGzipFile == NULL) { - return NULL; - } - Py_INCREF(BadGzipFile); - - Py_INCREF(IsalError); if (PyModule_AddObject(m, "IsalError", IsalError) < 0) { return NULL; diff --git a/tests/test_subinterpreters.py b/tests/test_subinterpreters.py new file mode 100644 index 00000000..d869da2a --- /dev/null +++ b/tests/test_subinterpreters.py @@ -0,0 +1,37 @@ +"""Use isal from more than one interpreter.""" +import gzip +import sys +import sysconfig +import textwrap + +from isal import igzip, igzip_lib, isal_zlib + +import pytest + + +@pytest.mark.skipif(sys.implementation.name != "cpython", + reason="Needs _testcapi.run_in_subinterp") +@pytest.mark.skipif(bool(sysconfig.get_config_var("Py_GIL_DISABLED")), + reason="Free-threaded subinterpreters cannot import isal") +def test_errors_after_import_in_subinterpreter(): + _testcapi = pytest.importorskip("_testcapi") + # Each interpreter has its own gzip.BadGzipFile class. An error that the + # except clause does not catch makes run_in_subinterp return -1. + code = textwrap.dedent(""" + import gzip + from isal import igzip, igzip_lib, isal_zlib + for decompress, error in [(isal_zlib.decompress, isal_zlib.error), + (igzip_lib.decompress, igzip_lib.error), + (igzip.decompress, gzip.BadGzipFile)]: + try: + decompress(b"not compressed") + except error: + pass + """) + assert _testcapi.run_in_subinterp(code) == 0 + with pytest.raises(isal_zlib.error): + isal_zlib.decompress(b"not compressed") + with pytest.raises(igzip_lib.error): + igzip_lib.decompress(b"not compressed") + with pytest.raises(gzip.BadGzipFile): + igzip.decompress(b"not compressed") From ed377d9fee6c5be55abdd832c3ec83cdefe3421a Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 20 Oct 2026 09:40:36 -0400 Subject: [PATCH 40/73] Check for a failed allocation in IgzipDecompressor The constructor wrote to the new object without checking whether PyObject_New() returned NULL, so running out of memory crashed instead of raising MemoryError. --- CHANGELOG.rst | 2 ++ src/isal/igzip_libmodule.c | 3 +++ tests/test_igzip_lib.py | 22 ++++++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 94d0f148..d0876cc7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -85,6 +85,8 @@ version 1.8.1-dev + Fix ``readinto()`` of the private ``isal_zlib._GzipReader`` class, which wrote into read-only objects such as ``bytes``. It raises ``TypeError`` now. ``igzip.IGzipFile`` was not affected. ++ Fix a crash when there is not enough memory to create an + ``igzip_lib.IgzipDecompressor``. It raises ``MemoryError`` now. version 1.8.0 ----------------- diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index 8d1aa045..b2575d93 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -471,6 +471,9 @@ igzip_lib_IgzipDecompressor__new__(PyTypeObject *type, return NULL; } IgzipDecompressor *self = PyObject_New(IgzipDecompressor, type); + if (self == NULL) { + return NULL; + } self->eof = 0; self->needs_input = 1; self->saved_bitbuffer_size = 0; diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index e53f4af3..32d246a6 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -12,7 +12,9 @@ import itertools import os import pickle +import subprocess import sys +import textwrap import zlib from typing import NamedTuple @@ -293,6 +295,26 @@ def test_igzip_decompressor_keeps_zdict_reference(): assert sys.getrefcount(zdict) == refcount +@pytest.mark.skipif(sys.implementation.name != "cpython", + reason="Needs _testcapi.set_nomemory") +def test_igzip_decompressor_no_memory(): + pytest.importorskip("_testcapi") + # Fail the first memory allocation, which is the object itself. A + # subprocess keeps the failing allocation away from the test runner. + code = textwrap.dedent(""" + import _testcapi + from isal.igzip_lib import IgzipDecompressor + _testcapi.set_nomemory(0, 1) + try: + IgzipDecompressor() + except MemoryError: + pass + finally: + _testcapi.remove_mem_hooks() + """) + subprocess.run([sys.executable, "-c", code], check=True) + + @pytest.mark.parametrize(["unused_size", "flag_pair", "data_size"], itertools.product([26], FLAGS, [128 * 1024 - 3, 874, 81923, 9111])) From a8547f8808331a50bb45e5ea29be9ea64b66b4dc Mon Sep 17 00:00:00 2001 From: ptweezy Date: Wed, 21 Oct 2026 22:04:30 -0400 Subject: [PATCH 41/73] Check for a failed allocation in the "Not a gzipped file" error GzipReader_read_into_buffer() passed the result of PyBytes_FromStringAndSize() to raise_bad_gzip_file() and Py_DECREF() without checking it, so running out of memory there crashed instead of raising MemoryError. --- CHANGELOG.rst | 3 +++ src/isal/isal_zlibmodule.c | 3 +++ tests/test_igzip.py | 23 +++++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bc1e5b55..52cec9f3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -109,6 +109,9 @@ version 1.8.1-dev ``igzip.IGzipFile`` was not affected. + Fix a crash when there is not enough memory to create an ``igzip_lib.IgzipDecompressor``. It raises ``MemoryError`` now. ++ Fix a crash when there is not enough memory to raise ``BadGzipFile`` for + data that is not in the gzip format, for example, in ``igzip.decompress``. + It raises ``MemoryError`` now. version 1.8.0 ----------------- diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 09715557..43f79a0e 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1845,6 +1845,9 @@ GzipReader_read_into_buffer(GzipReader *self, uint8_t *out_buffer, size_t out_bu if (!(magic1 == 0x1f && magic2 == 0x8b)) { Py_BLOCK_THREADS; PyObject *magic_obj = PyBytes_FromStringAndSize((char *)current_pos, 2); + if (magic_obj == NULL) { + return -1; + } raise_bad_gzip_file( "Not a gzipped file (%R)", magic_obj); diff --git a/tests/test_igzip.py b/tests/test_igzip.py index b8e0a2eb..4f0e44f3 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -19,6 +19,7 @@ import subprocess import sys import tempfile +import textwrap import zlib from gzip import FCOMMENT, FEXTRA, FHCRC, FNAME, FTEXT # type: ignore from pathlib import Path @@ -457,6 +458,28 @@ def test_decompress_not_a_gzip(): assert error.match(re.escape("Not a gzipped file (b'Th')")) +@pytest.mark.skipif(sys.implementation.name != "cpython", + reason="Needs _testcapi.set_nomemory") +def test_decompress_not_a_gzip_no_memory(): + pytest.importorskip("_testcapi") + # Fail each memory allocation in turn, such as the one for the bytes in + # the error message. A subprocess keeps the failing allocations away from + # the test runner. Each call raises MemoryError or BadGzipFile. + code = textwrap.dedent(""" + import _testcapi + from isal import igzip + for start in range(50): + _testcapi.set_nomemory(start, start + 1) + try: + igzip.decompress(b"This is not a gzip data stream.") + except (MemoryError, igzip.BadGzipFile): + pass + finally: + _testcapi.remove_mem_hooks() + """) + subprocess.run([sys.executable, "-c", code], check=True) + + def test_decompress_unknown_compression_method(): corrupted_data = COMPRESSED_DATA[:2] + b'\x09' + COMPRESSED_DATA[3:] with pytest.raises(igzip.BadGzipFile) as error: From f4a4999a25503a163374e856021b0c7738b98fdd Mon Sep 17 00:00:00 2001 From: ptweezy Date: Wed, 21 Oct 2026 22:04:31 -0400 Subject: [PATCH 42/73] Reset next_in after an error in IgzipDecompressor.decompress() When decompress() failed after decompress_buf() succeeded, for example, because the copy of the unconsumed input could not be allocated, state.next_in still pointed into the buffer of the caller. The next call then read from that buffer after the caller had released it, and wrote past its end when it got more data. CPython had the same bug in bz2.BZ2Decompressor, lzma.LZMADecompressor, and the internal zlib._ZlibDecompressor, and fixed it the same way in gh-148395 (CVE-2026-6100). --- CHANGELOG.rst | 4 ++++ src/isal/igzip_libmodule.c | 4 ++++ tests/test_igzip_lib.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 52cec9f3..6046e32e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -112,6 +112,10 @@ version 1.8.1-dev + Fix a crash when there is not enough memory to raise ``BadGzipFile`` for data that is not in the gzip format, for example, in ``igzip.decompress``. It raises ``MemoryError`` now. ++ Fix a crash in ``igzip_lib.IgzipDecompressor.decompress()`` after a call on + the same object raised ``MemoryError``. The next call used the input of the + failed call after the caller had released it. This matches the CPython fix + for gh-148395. version 1.8.0 ----------------- diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index 0189eee3..c0fdbe35 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -298,6 +298,10 @@ decompress(IgzipDecompressor *self, uint8_t *data, size_t len, Py_ssize_t max_le return result; error: + /* next_in can point to the buffer of the caller, which is released after + this call. Reset it so the next call does not use it, as CPython does + since gh-148395. */ + self->state.next_in = NULL; Py_XDECREF(result); return NULL; } diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index 32d246a6..d63aedc0 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -315,6 +315,37 @@ def test_igzip_decompressor_no_memory(): subprocess.run([sys.executable, "-c", code], check=True) +@pytest.mark.skipif(sys.implementation.name != "cpython", + reason="Needs _testcapi.set_nomemory") +def test_igzip_decompressor_decompress_no_memory(): + pytest.importorskip("_testcapi") + # After a MemoryError, the next call must not use the input of the failed + # call. That input is in an mmap, so using it after close() crashes. + code = textwrap.dedent(""" + import mmap + import os + import _testcapi + from isal import igzip_lib + # Random data does not compress, so most of the input is left over + # after 10 bytes of output. decompress() copies it to a new buffer. + compressed = igzip_lib.compress(os.urandom(256 * 1024)) + for start in range(20): + decompressor = igzip_lib.IgzipDecompressor() + buffer = mmap.mmap(-1, len(compressed)) + buffer.write(compressed) + _testcapi.set_nomemory(start, start + 1) + try: + decompressor.decompress(buffer, 10) + except MemoryError: + pass + finally: + _testcapi.remove_mem_hooks() + buffer.close() + decompressor.decompress(b"") + """) + subprocess.run([sys.executable, "-c", code], check=True) + + @pytest.mark.parametrize(["unused_size", "flag_pair", "data_size"], itertools.product([26], FLAGS, [128 * 1024 - 3, 874, 81923, 9111])) From 20b71d9f6f60b24d301fd7dbf365c4523ff0f516 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Wed, 21 Oct 2026 22:04:31 -0400 Subject: [PATCH 43/73] Do not free a NULL lock when a constructor runs out of memory When PyThread_allocate_lock() failed, compressobj(), decompressobj(), and IgzipDecompressor() released the new object, and its dealloc function passed the NULL lock to PyThread_free_lock(). That crashes on PyPy, and on builds of CPython 3.11 to 3.14 that make their locks from a mutex and a condition variable, such as the macOS builds. The dealloc functions now skip a NULL lock, like GzipReader_dealloc() does. PyObject_New() does not zero the object, so newdecompobject() and IgzipDecompressor() now set the lock, and newdecompobject() also sets unconsumed_tail, before the first failure path. On CPython, the PyBytes_FromStringAndSize() calls before the lock cannot fail, because with a size of zero they return the shared empty bytes object. --- CHANGELOG.rst | 3 +++ src/isal/igzip_libmodule.c | 5 ++++- src/isal/isal_zlibmodule.c | 13 ++++++++++--- tests/test_compat.py | 25 +++++++++++++++++++++++++ tests/test_igzip_lib.py | 20 +++++++++++--------- 5 files changed, 53 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6046e32e..1fb79cb4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -116,6 +116,9 @@ version 1.8.1-dev the same object raised ``MemoryError``. The next call used the input of the failed call after the caller had released it. This matches the CPython fix for gh-148395. ++ Fix a crash on macOS and PyPy when there is not enough memory to create the + internal lock of an ``isal_zlib.Compress``, ``isal_zlib.Decompress``, or + ``igzip_lib.IgzipDecompressor`` object. It raises ``MemoryError`` now. version 1.8.0 ----------------- diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index c0fdbe35..0f5779f0 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -44,7 +44,9 @@ typedef struct { static void IgzipDecompressor_dealloc(IgzipDecompressor *self) { - PyThread_free_lock(self->lock); + if (self->lock != NULL) { + PyThread_free_lock(self->lock); + } PyMem_Free(self->input_buffer); Py_CLEAR(self->unused_data); Py_CLEAR(self->zdict); @@ -484,6 +486,7 @@ igzip_lib_IgzipDecompressor__new__(PyTypeObject *type, self->avail_in_real = 0; self->input_buffer = NULL; self->input_buffer_size = 0; + self->lock = NULL; Py_XINCREF(zdict); self->zdict = zdict; self->unused_data = PyBytes_FromStringAndSize(NULL, 0); diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 43f79a0e..9d230f5e 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -594,7 +594,9 @@ Comp_dealloc(compobject *self) { if (self->is_initialised && self->level_buf != NULL) PyMem_Free(self->level_buf); - PyThread_free_lock(self->lock); + if (self->lock != NULL) { + PyThread_free_lock(self->lock); + } Py_XDECREF(self->zdict); Py_TYPE(self)->tp_free((PyObject *)self); } @@ -639,7 +641,9 @@ typedef struct static void Decomp_dealloc(decompobject *self) { - PyThread_free_lock(self->lock); + if (self->lock != NULL) { + PyThread_free_lock(self->lock); + } Py_XDECREF(self->unused_data); Py_XDECREF(self->unconsumed_tail); Py_XDECREF(self->zdict); @@ -683,6 +687,9 @@ newdecompobject() self->is_initialised = 0; self->method_set = 0; self->zdict = NULL; + // Decomp_dealloc uses these members when initialization fails + self->unconsumed_tail = NULL; + self->lock = NULL; self->unused_data = PyBytes_FromStringAndSize("", 0); if (self->unused_data == NULL) { Py_DECREF(self); @@ -693,7 +700,7 @@ newdecompobject() Py_DECREF(self); return NULL; } - self->lock = PyThread_allocate_lock(); + self->lock = PyThread_allocate_lock(); if (self->lock == NULL) { Py_DECREF(self); PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); diff --git a/tests/test_compat.py b/tests/test_compat.py index 020ed043..23a79dfb 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -12,6 +12,9 @@ import gzip import itertools import random +import subprocess +import sys +import textwrap import threading import warnings import zlib @@ -66,6 +69,28 @@ def test_compress_and_decompress_types_cannot_be_called(factory): type(factory())() +@pytest.mark.skipif(sys.implementation.name != "cpython", + reason="Needs _testcapi.set_nomemory") +@pytest.mark.parametrize("factory", ["compressobj", "decompressobj"]) +def test_compressobj_decompressobj_no_memory(factory): + pytest.importorskip("_testcapi") + # Fail each memory allocation in turn, such as the one for the lock. A + # subprocess keeps the failing allocations away from the test runner. + code = textwrap.dedent(f""" + import _testcapi + from isal import isal_zlib + for start in range(20): + _testcapi.set_nomemory(start, start + 1) + try: + isal_zlib.{factory}() + except MemoryError: + pass + finally: + _testcapi.remove_mem_hooks() + """) + subprocess.run([sys.executable, "-c", code], check=True) + + @pytest.mark.parametrize(["data_size", "level"], itertools.product(DATA_SIZES, range(4))) def test_compress(data_size, level): diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index d63aedc0..f2ccce1b 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -299,18 +299,20 @@ def test_igzip_decompressor_keeps_zdict_reference(): reason="Needs _testcapi.set_nomemory") def test_igzip_decompressor_no_memory(): pytest.importorskip("_testcapi") - # Fail the first memory allocation, which is the object itself. A - # subprocess keeps the failing allocation away from the test runner. + # Fail each memory allocation in turn, such as the one for the object + # itself or for its lock. A subprocess keeps the failing allocations away + # from the test runner. code = textwrap.dedent(""" import _testcapi from isal.igzip_lib import IgzipDecompressor - _testcapi.set_nomemory(0, 1) - try: - IgzipDecompressor() - except MemoryError: - pass - finally: - _testcapi.remove_mem_hooks() + for start in range(20): + _testcapi.set_nomemory(start, start + 1) + try: + IgzipDecompressor() + except MemoryError: + pass + finally: + _testcapi.remove_mem_hooks() """) subprocess.run([sys.executable, "-c", code], check=True) From 93fe0e294e6be68e8a52d4e61ff9e54c415f1427 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Wed, 21 Oct 2026 22:04:31 -0400 Subject: [PATCH 44/73] Fix the docstring of isal_zlib.Decompress It named compressobj instead of decompressobj. --- src/isal/isal_zlibmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 9d230f5e..7686d0a7 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -1585,7 +1585,7 @@ static PyGetSetDef Decomp_properties[] = { static PyTypeObject IsalZlibDecompType = { .tp_name = "isal_zlib.Decompress", - .tp_doc = "Object returned by isal_zlib.compressobj.", + .tp_doc = "Object returned by isal_zlib.decompressobj.", .tp_basicsize = sizeof(decompobject), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_dealloc = (destructor)Decomp_dealloc, From c917a90290c45bcb9a8b4ab9aa32d01793a20fff Mon Sep 17 00:00:00 2001 From: ptweezy Date: Wed, 21 Oct 2026 20:56:19 -0400 Subject: [PATCH 45/73] Raise igzip_threaded compression errors instead of waiting forever When compressing a block failed in a writer with more than one thread, that compression thread set running to False and stopped. The writer thread stopped at its next queue timeout, so compressed blocks of the other threads stayed in the output queues. flush() and close() waited forever, and the other compression threads waited forever to put their next block in a full output queue. Compression errors now work like write errors. The compression thread stores the error before it calls task_done(), passes an empty block on, and keeps taking blocks until stop() sends the None sentinel. The writer thread drops all blocks after an error. flush() now always sees the error, so close() no longer checks for it again after stop(). --- CHANGELOG.rst | 4 +++ src/isal/igzip_threaded.py | 43 ++++++++++++-------------------- tests/test_igzip_threaded.py | 48 ++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bc1e5b55..d509fea2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -89,6 +89,10 @@ version 1.8.1-dev to ``write()``, ``flush()``, or ``close()`` raises the error. After an error, ``close()`` no longer closes a file object that you passed to ``igzip_threaded.open``. ++ Fix ``igzip_threaded`` writers with more than one thread that waited + forever when compressing a block failed, for example, because there was not + enough memory. A later call to ``write()``, ``flush()``, or ``close()`` + raises the error. + Fix a crash on the free-threaded build when one thread reads ``unused_data`` or ``unconsumed_tail`` of an ``isal_zlib`` decompressor while another thread decompresses with it. Reading the attributes of diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index cba3cdcf..8968e5eb 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -354,6 +354,8 @@ def flush(self): # Wait for all data to be written for out_q in self.output_queues: out_q.join() + # The threads store an error before they call task_done(), so after + # the joins, this check sees an error in any block written so far. if self.exception: raise self.exception self.raw.flush() @@ -375,11 +377,6 @@ def close(self) -> None: self._closed = True if self.closefd: self.raw.close() - # flush() can miss a compression error because the worker threads - # call task_done() before they set the error. Check again now that - # the threads have stopped. - if self.exception: - raise self.exception @property def closed(self) -> bool: @@ -403,9 +400,11 @@ def _compress(self, index: int): try: compressed, crc = compressor.compress_and_crc(data, zdict) except Exception as e: - in_queue.task_done() - self._set_error_and_empty_queue(e, in_queue) - return + with self.lock: + self.exception = e + # Pass on an empty block, so the writer thread keeps taking + # blocks from the output queues in order. It drops the block. + compressed, crc = b"", 0 data_length = len(data) out_queue.put((compressed, crc, data_length)) in_queue.task_done() @@ -451,9 +450,10 @@ def _compress_and_write(self): try: compressed, crc = compressor.compress_and_crc(data, zdict) except Exception as e: - in_queue.task_done() - self._set_error_and_empty_queue(e, in_queue) - return + with self.lock: + self.exception = e + # _write_compressed() drops the block. + compressed, crc = b"", 0 data_length = len(data) self._crc = isal_zlib.crc32_combine(self._crc, crc, data_length) self._size += data_length @@ -461,10 +461,11 @@ def _compress_and_write(self): in_queue.task_done() def _write_compressed(self, compressed: bytes): - # After a write error, this thread keeps taking blocks from the queues - # and drops them until stop() sends the None sentinel. That way, no - # thread waits forever on a full queue, and write(), flush(), and - # close() can raise the error in the calling thread. + # After a compression or write error, the threads keep taking blocks + # from the queues, and this method drops them until stop() sends the + # None sentinel. That way, no thread waits forever on a full queue, + # and write(), flush(), and close() can raise the error in the + # calling thread. if self.exception: return try: @@ -473,17 +474,5 @@ def _write_compressed(self, compressed: bytes): with self.lock: self.exception = e - def _set_error_and_empty_queue(self, error, q): - with self.lock: - self.exception = error - # Abort everything and empty the queue - self.running = False - while True: - try: - _ = q.get(timeout=0.05) - q.task_done() - except queue.Empty: - return - def writable(self) -> bool: return True diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 572ece53..7406cc43 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -462,3 +462,51 @@ def test_reader_error_does_not_wait_for_queue_timeouts(queue_monitor): with pytest.raises(EOFError): f.read() assert queue_monitor.expired == [] + + +@pytest.mark.timeout(5) +@pytest.mark.parametrize("threads", [1, 2, 3]) +def test_threaded_compression_error(monkeypatch, threads): + isal_zlib = igzip_threaded.isal_zlib + ParallelCompress = isal_zlib._ParallelCompress + all_written = threading.Event() + + class FailingCompressor: + """Fails on the block b"bad" after the test wrote all blocks.""" + def __init__(self, **kwargs): + self.compressor = ParallelCompress(**kwargs) + + def compress_and_crc(self, data, zdict): + if data == b"bad": + all_written.wait() + raise MemoryError() + return self.compressor.compress_and_crc(data, zdict) + + monkeypatch.setattr(isal_zlib, "_ParallelCompress", FailingCompressor) + raw = io.BytesIO() + f = igzip_threaded._ThreadedGzipWriter(raw, threads=threads) + # Block 0 fails, so the writer thread must take the blocks of the other + # threads from their output queues and drop them. Write as many blocks + # as fit without waiting: two for each thread. The first thread works on + # block 0, and its next block waits in its input queue, so one more block + # would make write() wait for the first thread. + try: + f.write(b"bad") + for _ in range(2 * threads - 1): + f.write(b"good") + finally: + # Set the event even if a write fails, so the first thread does not + # wait forever. + all_written.set() + with pytest.raises(MemoryError): + f.flush() + # Later calls raise the error again. + with pytest.raises(MemoryError): + f.write(b"good") + with pytest.raises(MemoryError): + f.close() + assert_writer_closed(f) + assert not raw.closed + # raw holds only the gzip header. The writer thread dropped all blocks, + # and close() did not end the stream. + assert len(raw.getvalue()) == 10 From 4cf20d1f409f7626ad3bcd3858898581a432620e Mon Sep 17 00:00:00 2001 From: ptweezy Date: Wed, 21 Oct 2026 22:04:32 -0400 Subject: [PATCH 46/73] Take the lock in _GzipReader.seek() read(), readinto(), and readall() hold the lock while they decompress, and decompression releases the GIL. seek() did not take the lock, so a seek() in one thread could reset or advance the reader while a read() in another thread used it. On GIL and free-threaded builds, this caused crashes, wrong data, and errors for valid data. The code after the argument parsing moved unchanged from GzipReader_seek() into GzipReader_seek_impl(). GzipReader_seek() calls it with the lock held, so one LEAVE_ZLIB() covers every return path. igzip.IGzipFile was not affected, because io.BufferedReader serializes its calls to the readinto(), readall(), and seek() methods of the reader. --- CHANGELOG.rst | 3 +++ src/isal/isal_zlibmodule.c | 28 ++++++++++++++++++++-------- tests/test_freethreading.py | 20 ++++++++++++++++++++ 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bc1e5b55..4daed65e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -99,6 +99,9 @@ version 1.8.1-dev private ``isal_zlib._GzipReader`` class raise ``TypeError`` instead of ``SystemError`` for arguments of the wrong type or number. ``igzip.IGzipFile`` was not affected. ++ Fix crashes and wrong data when one thread calls ``seek()`` of the + private ``isal_zlib._GzipReader`` class while another thread reads from the + same object. ``igzip.IGzipFile`` was not affected. + Fix a crash on PyPy when you create an ``isal_zlib.Compress`` or ``isal_zlib.Decompress`` object by calling the type, for example, ``isal_zlib.Compress()``, and then use it. Calling the type raises diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 09715557..2001b3b6 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -2027,16 +2027,10 @@ GzipReader_readinto(GzipReader *self, PyObject *buffer_obj) return PyLong_FromSsize_t((Py_ssize_t)written_size); } +/* Called with self->lock held */ static PyObject * -GzipReader_seek(GzipReader *self, PyObject *args, PyObject *kwargs) +GzipReader_seek_impl(GzipReader *self, Py_ssize_t offset, Py_ssize_t whence) { - Py_ssize_t offset; - Py_ssize_t whence = SEEK_SET; - static char *keywords[] = {"offset", "whence", NULL}; - static char format[] = {"n|n:GzipReader.seek"}; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, keywords, &offset, &whence)) { - return NULL; - } // Recalculate offset as an absolute file position. if (whence == SEEK_SET) { ; @@ -2117,6 +2111,24 @@ GzipReader_seek(GzipReader *self, PyObject *args, PyObject *kwargs) return PyLong_FromLongLong(self->_pos); } +static PyObject * +GzipReader_seek(GzipReader *self, PyObject *args, PyObject *kwargs) +{ + Py_ssize_t offset; + Py_ssize_t whence = SEEK_SET; + static char *keywords[] = {"offset", "whence", NULL}; + static char format[] = {"n|n:GzipReader.seek"}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, keywords, &offset, &whence)) { + return NULL; + } + /* Take the lock like read() does. Decompression releases the GIL, so + another thread could otherwise read while seek() changes the state. */ + ENTER_ZLIB(self); + PyObject *result = GzipReader_seek_impl(self, offset, whence); + LEAVE_ZLIB(self); + return result; +} + static PyObject * GzipReader_readall(GzipReader *self, PyObject *Py_UNUSED(ignore)) { diff --git a/tests/test_freethreading.py b/tests/test_freethreading.py index 75f113dc..d156b074 100644 --- a/tests/test_freethreading.py +++ b/tests/test_freethreading.py @@ -1,4 +1,5 @@ import concurrent.futures +import io import random import string import threading @@ -243,3 +244,22 @@ def run(index): assert run_in_threads(run)[0] == data assert decompressor.unused_data == b"unused" assert decompressor.crc == isal_zlib.crc32(data) + + +def test_gzip_reader_seek_while_reading(): + # Each number appears once, so a chunk that mixes up parts is not in it. + data = b"".join(b"%08d" % i for i in range(128 * 1024)) + compressed = isal_zlib.compress(data, wbits=31) + reader = isal_zlib._GzipReader(io.BytesIO(compressed)) + + def run(index): + for i in range(NUM_ITERATIONS): + if index % 2: + reader.seek(len(data) // 2 if i % 2 else 0) + else: + # seek() can come between two reads, but not during one. + assert reader.read(64 * 1024) in data + + run_in_threads(run) + reader.seek(0) + assert reader.read() == data From 1b85b9d30536b305a68e49378a2e39fa5db62594 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Wed, 21 Oct 2026 20:29:09 -0400 Subject: [PATCH 47/73] Escape the file path in a generated test program test_threaded_program_can_exit_on_error put the path of its test file between single quotes in a generated program. On Windows, the backslashes in that path start escape sequences, so the program failed with a SyntaxError before it ran any line. A SyntaxError also exits with code 1, so the test passed without testing anything. Use ascii() for the path, and check that the program reached its raise statement. ascii() also escapes non-ASCII characters, so the program stays valid UTF-8 when open() writes it in a locale encoding such as cp1252. The check looks for the exception message alone, because with FORCE_COLOR=1 or PYTHON_COLORS=1, PyPy 3.11 and CPython 3.13 or later put color codes between the exception type and the message. The program now raises Exception('Reached the end'), because a check for 'Error' would also match SyntaxError. --- tests/test_igzip_threaded.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 572ece53..eb52151c 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -342,15 +342,19 @@ def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): test_file.write_bytes(gzip.compress(b"test" * (10 * 1024 * 1024))) with open(program, "wt") as f: f.write("from isal import igzip_threaded\n") + # ascii() escapes backslashes and non-ASCII characters in the path. f.write( - f"f = igzip_threaded.open('{test_file}', " + f"f = igzip_threaded.open({str(test_file)!a}, " f"mode='{mode}', threads={threads})\n" ) - f.write("raise Exception('Error')\n") + f.write("raise Exception('Reached the end')\n") result = subprocess.run([sys.executable, str(program)], capture_output=True) # A sanitizer report changes the exit code. assert result.returncode == 1, result.stderr.decode() + # A SyntaxError or a failed open() also exits with 1. FORCE_COLOR can + # color the exception type, so check only the message. + assert b"Reached the end" in result.stderr, result.stderr.decode() @pytest.mark.parametrize("threads", [1, 2]) From 13e010d4d0170c0ce1d359078a94cc6d65020a4b Mon Sep 17 00:00:00 2001 From: ptweezy Date: Sat, 24 Oct 2026 12:30:59 -0400 Subject: [PATCH 48/73] Remove the temporary files that the tests leave behind test_threaded_write and test_threaded_write_oversized_block_no_error never removed their NamedTemporaryFile(delete=False) files. BaseTest in test_gzip_compliance.py created a file in __init__(), but pytest creates TestCase instances while it collects tests, and only the instances that ran a test removed their file. test_repr and test_seek removed their files only when they passed. Use tmp_path, and create the file name in setUp(). --- tests/test_gzip_compliance.py | 9 +++------ tests/test_igzip.py | 12 ++++-------- tests/test_igzip_threaded.py | 8 ++++---- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index 5d71577c..bca9fd40 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -65,14 +65,11 @@ def seek(self, *args): class BaseTest(unittest.TestCase): - def __init__(self, methodName): + def setUp(self): + # Make a unique file name. The tests create the file themselves. fileno, self.filename = tempfile.mkstemp() os.close(fileno) - super().__init__(methodName) - - def setUp(self): - if os.path.exists(self.filename): - os.unlink(self.filename) + os.unlink(self.filename) def tearDown(self): if os.path.exists(self.filename): diff --git a/tests/test_igzip.py b/tests/test_igzip.py index 4f0e44f3..f185ac80 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -14,7 +14,6 @@ import itertools import os import re -import shutil import struct import subprocess import sys @@ -39,11 +38,9 @@ def test_wrong_compresslevel_igzipfile(): error.match("Compression level should be between 0 and 3") -def test_repr(): - tempdir = tempfile.mkdtemp() - with igzip.IGzipFile(os.path.join(tempdir, "test.gz"), "wb") as test: +def test_repr(tmp_path): + with igzip.IGzipFile(str(tmp_path / "test.gz"), "wb") as test: assert " Date: Sat, 24 Oct 2026 12:30:59 -0400 Subject: [PATCH 50/73] Stop passing "rb" as the queue size in test_close_reader The second parameter of _ThreadedGzipReader is queue_size, so the test created queue.Queue("rb"). If the reader thread got to put() before close(), it failed with TypeError. --- tests/test_igzip_threaded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 572ece53..5a028292 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -211,7 +211,7 @@ def test_threaded_close_error(monkeypatch, threads): def test_close_reader(): tmp = io.BytesIO(Path(TEST_FILE).read_bytes()) - f = igzip_threaded._ThreadedGzipReader(tmp, "rb") + f = igzip_threaded._ThreadedGzipReader(tmp) f.close() assert f.closed # Make sure double closing does not raise errors From 2ac1c9d349f600fedf0df09a2b4a2756335556d7 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Sat, 24 Oct 2026 12:31:00 -0400 Subject: [PATCH 51/73] Run tests/test_compat.py in the compatibility tox environment The environment ran tests/test_isal.py, which 270b6a9 renamed to tests/test_compat.py, so it failed with "file or directory not found". --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c01b469a..0afb1a4c 100644 --- a/tox.ini +++ b/tox.ini @@ -41,7 +41,7 @@ commands= [testenv:compatibility] deps=pytest commands= - pytest tests/test_isal.py + pytest tests/test_compat.py [testenv:lint] deps=flake8 From dcad7cf4e02a33c209f64d9d2fc5f8b6417241b4 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Sat, 24 Oct 2026 12:30:59 -0400 Subject: [PATCH 52/73] Fix the raw deflate unused_data tests when test_offset is 0 no_header[:-test_offset] is empty when test_offset is 0, so the decompressor never reaches the end of the stream and the eof check skips the assertion. Slice up to len(no_header) - test_offset and assert eof. --- tests/test_igzip_lib.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index f2ccce1b..4312d192 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -238,12 +238,13 @@ def test_igzip_decompressor_raw_deflate_unused_data_zlib(test_offset): data = zlib.compress(b"bla") no_header = data[2:] trailer = data[-4:] - raw_deflate_incomplete_trailer = no_header[:-test_offset] - true_unused_data = trailer[:-test_offset] + # no_header[:-test_offset] would be empty when test_offset is 0 + raw_deflate_incomplete_trailer = no_header[:len(no_header) - test_offset] + true_unused_data = trailer[:len(trailer) - test_offset] igzd = IgzipDecompressor(flag=DECOMP_DEFLATE) igzd.decompress(raw_deflate_incomplete_trailer) - if igzd.eof: - assert igzd.unused_data == true_unused_data + assert igzd.eof + assert igzd.unused_data == true_unused_data @pytest.mark.parametrize("test_offset", range(9)) @@ -251,12 +252,13 @@ def test_igzip_decompressor_raw_deflate_unused_data_gzip(test_offset): data = gzip.compress(b"bla") no_header = data[10:] trailer = data[-8:] - raw_deflate_incomplete_trailer = no_header[:-test_offset] - true_unused_data = trailer[:-test_offset] + # no_header[:-test_offset] would be empty when test_offset is 0 + raw_deflate_incomplete_trailer = no_header[:len(no_header) - test_offset] + true_unused_data = trailer[:len(trailer) - test_offset] igzd = IgzipDecompressor(flag=DECOMP_DEFLATE) igzd.decompress(raw_deflate_incomplete_trailer) - if igzd.eof: - assert igzd.unused_data == true_unused_data + assert igzd.eof + assert igzd.unused_data == true_unused_data @pytest.mark.parametrize(["flag", "wbits"], [(DECOMP_DEFLATE, -15), From f3b243e14a5f8da352f490719ae2bdae6040e0aa Mon Sep 17 00:00:00 2001 From: ptweezy Date: Sat, 24 Oct 2026 12:32:27 -0400 Subject: [PATCH 53/73] Add a timeout to the barrier in test_isal_compress_decompress_threaded A failed job made fewer barrier waits than the other jobs, so when the jobs ran out, the other threads waited at the barrier forever and the test never finished. Create the barrier in the test function with a 10 s timeout, because a module-level barrier would stay broken and also fail the next parameter set. --- tests/test_freethreading.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_freethreading.py b/tests/test_freethreading.py index 75f113dc..4e3c155e 100644 --- a/tests/test_freethreading.py +++ b/tests/test_freethreading.py @@ -75,10 +75,9 @@ NUM_THREADS = 10 NUM_ITERATIONS = 20 NUM_JOBS = 50 # To simulate 50 jobs running in 10 threads -barrier = threading.Barrier(parties=NUM_THREADS) -def isal_compress_decompress(compress, decompress): +def isal_compress_decompress(compress, decompress, barrier): for _ in range(NUM_ITERATIONS): barrier.wait() x = compress(HAMLET_SCENE) @@ -101,9 +100,14 @@ def isal_compress_decompress(compress, decompress): ] ) def test_isal_compress_decompress_threaded(compress, decompress): + # A failed job makes fewer barrier waits than the others, so when the + # jobs run out, the other threads would wait at the barrier forever. + # The timeout makes them fail instead. + barrier = threading.Barrier(parties=NUM_THREADS, timeout=10) with concurrent.futures.ThreadPoolExecutor(NUM_THREADS) as executor: futures = [ - executor.submit(isal_compress_decompress, compress, decompress) + executor.submit(isal_compress_decompress, compress, decompress, + barrier) for _ in range(NUM_JOBS) ] for future in concurrent.futures.as_completed(futures): From f1030b90e4a94fc9d79533011c31459ed9953c55 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Sat, 24 Oct 2026 16:21:28 -0400 Subject: [PATCH 54/73] Pass lists instead of itertools.product iterators to parametrize pytest 9.1 deprecated iterators as parametrize argument values, and pytest 10 turns the PytestRemovedIn10Warning into an error, which stops test collection. The call for test_threaded_write is left to #294, which changes the line after it. --- tests/test_compat.py | 30 +++++++++++++++--------------- tests/test_igzip_lib.py | 8 ++++---- tests/test_igzip_threaded.py | 5 +++-- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/tests/test_compat.py b/tests/test_compat.py index 23a79dfb..55487c8e 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -42,14 +42,14 @@ @pytest.mark.parametrize(["data_size", "value"], - itertools.product(DATA_SIZES, SEEDS)) + list(itertools.product(DATA_SIZES, SEEDS))) def test_crc32(data_size, value): data = DATA[:data_size] assert zlib.crc32(data, value) == isal_zlib.crc32(data, value) @pytest.mark.parametrize(["data_size", "value"], - itertools.product(DATA_SIZES, SEEDS)) + list(itertools.product(DATA_SIZES, SEEDS))) def test_adler32(data_size, value): data = DATA[:data_size] assert zlib.adler32(data, value) == isal_zlib.adler32(data, value) @@ -92,7 +92,7 @@ def test_compressobj_decompressobj_no_memory(factory): @pytest.mark.parametrize(["data_size", "level"], - itertools.product(DATA_SIZES, range(4))) + list(itertools.product(DATA_SIZES, range(4)))) def test_compress(data_size, level): data = DATA[:data_size] compressed = isal_zlib.compress(data, level=level) @@ -100,7 +100,7 @@ def test_compress(data_size, level): @pytest.mark.parametrize(["data_size", "level"], - itertools.product(DATA_SIZES, range(10))) + list(itertools.product(DATA_SIZES, range(10)))) def test_decompress_zlib(data_size, level): data = DATA[:data_size] compressed = zlib.compress(data, level=level) @@ -109,8 +109,8 @@ def test_decompress_zlib(data_size, level): @pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel"], - itertools.product([128 * 1024], range(4), - WBITS_RANGE, range(1, 10))) + list(itertools.product([128 * 1024], range(4), + WBITS_RANGE, range(1, 10)))) def test_decompress_wbits(data_size, level, wbits, memLevel): data = DATA[:data_size] compressobj = zlib.compressobj(level=level, wbits=wbits, memLevel=memLevel) @@ -120,7 +120,7 @@ def test_decompress_wbits(data_size, level, wbits, memLevel): @pytest.mark.parametrize(["data_size", "level"], - itertools.product(DATA_SIZES, range(4))) + list(itertools.product(DATA_SIZES, range(4)))) def test_decompress_isal_zlib(data_size, level): data = DATA[:data_size] compressed = isal_zlib.compress(data, level=level) @@ -130,8 +130,8 @@ def test_decompress_isal_zlib(data_size, level): @pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel"], - itertools.product([128 * 1024], range(4), - WBITS_RANGE, range(1, 10))) + list(itertools.product([128 * 1024], range(4), + WBITS_RANGE, range(1, 10)))) def test_compress_compressobj(data_size, level, wbits, memLevel): data = DATA[:data_size] compressobj = isal_zlib.compressobj(level=level, @@ -227,8 +227,8 @@ def test_compressobj_unsupported_strategy(level, strategy): @pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel"], - itertools.product([128 * 1024], range(4), - WBITS_RANGE, range(1, 10))) + list(itertools.product([128 * 1024], range(4), + WBITS_RANGE, range(1, 10)))) def test_decompress_decompressobj(data_size, level, wbits, memLevel): data = DATA[:data_size] compressobj = zlib.compressobj(level=level, wbits=wbits, memLevel=memLevel) @@ -249,7 +249,7 @@ def test_decompressobj_unconsumed_tail(): @pytest.mark.parametrize(["data_size", "level"], - itertools.product(DATA_SIZES, range(4))) + list(itertools.product(DATA_SIZES, range(4)))) def test_igzip_compress(data_size, level): data = DATA[:data_size] compressed = igzip.compress(data, compresslevel=level) @@ -257,7 +257,7 @@ def test_igzip_compress(data_size, level): @pytest.mark.parametrize(["data_size", "level"], - itertools.product(DATA_SIZES, range(10))) + list(itertools.product(DATA_SIZES, range(10)))) def test_decompress_gzip(data_size, level): data = DATA[:data_size] compressed = gzip.compress(data, compresslevel=level) @@ -266,7 +266,7 @@ def test_decompress_gzip(data_size, level): @pytest.mark.parametrize(["data_size", "level"], - itertools.product(DATA_SIZES, range(4))) + list(itertools.product(DATA_SIZES, range(4)))) def test_decompress_igzip(data_size, level): data = DATA[:data_size] compressed = igzip.compress(data, compresslevel=level) @@ -276,7 +276,7 @@ def test_decompress_igzip(data_size, level): @pytest.mark.parametrize(["unused_size", "wbits"], - itertools.product([26], [-15, 15, 31])) + list(itertools.product([26], [-15, 15, 31]))) def test_unused_data(unused_size, wbits): unused_data = b"abcdefghijklmnopqrstuvwxyz"[:unused_size] compressor = zlib.compressobj(wbits=wbits) diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index f2ccce1b..523f7156 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -59,8 +59,8 @@ class Flag(NamedTuple): @pytest.mark.parametrize(["level", "flag", "mem_level", "hist_bits"], - itertools.product( - COMPRESS_LEVELS, FLAGS, MEM_LEVELS, HIST_BITS)) + list(itertools.product( + COMPRESS_LEVELS, FLAGS, MEM_LEVELS, HIST_BITS))) def test_compress_decompress(level, flag: Flag, mem_level, hist_bits): comp = igzip_lib.compress(DATA, level, flag.comp, mem_level, hist_bits) decomp = igzip_lib.decompress(comp, flag.decomp, hist_bits) @@ -349,8 +349,8 @@ def test_igzip_decompressor_decompress_no_memory(): @pytest.mark.parametrize(["unused_size", "flag_pair", "data_size"], - itertools.product([26], FLAGS, - [128 * 1024 - 3, 874, 81923, 9111])) + list(itertools.product( + [26], FLAGS, [128 * 1024 - 3, 874, 81923, 9111]))) def test_decompression_flags(unused_size, flag_pair, data_size): comp_flag, decomp_flag = flag_pair unused_data = b"abcdefghijklmnopqrstuvwxyz"[:unused_size] diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 572ece53..31650f4b 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -105,7 +105,8 @@ def test_threaded_write_oversized_block_no_error(threads): @pytest.mark.timeout(5) @pytest.mark.parametrize(["threads", "block_size"], - itertools.product([1, 3], [8 * 1024, 1024 * 1024])) + list(itertools.product([1, 3], + [8 * 1024, 1024 * 1024]))) def test_threaded_write_incompressible_level_0(tmp_path, threads, block_size): # Level 0 codes all data with one precomputed Huffman table and never # falls back to stored blocks, so random bytes grow by more than 20%. @@ -333,7 +334,7 @@ def test_threaded_writer_does_not_close_stream(): @pytest.mark.timeout(5) @pytest.mark.parametrize( - ["mode", "threads"], itertools.product(["rb", "wb"], [1, 2])) + ["mode", "threads"], list(itertools.product(["rb", "wb"], [1, 2]))) def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): program = tmp_path / "no_context_manager.py" test_file = tmp_path / "output.gz" From d04532df3c0b2eed52eee33a0700b1e2adc6b16f Mon Sep 17 00:00:00 2001 From: ptweezy Date: Sat, 24 Oct 2026 16:23:58 -0400 Subject: [PATCH 55/73] Keep IgzipDecompressor.needs_input False while output is left When the output buffer is full, isal_inflate keeps decoding into its own buffer, so it can read all of the input while output is left. needs_input was then True, and readers such as CPython's DecompressReader raised EOFError on valid data. Set needs_input only when the output buffer was not full, as CPython's LZMADecompressor does. --- CHANGELOG.rst | 5 +++++ src/isal/igzip_libmodule.c | 6 +++++- tests/test_igzip_lib.py | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1fb79cb4..8e90e3fe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -55,6 +55,11 @@ version 1.8.1-dev input buffer when ``max_length`` split the end of a stream. At the end of a stream, ``isal_zlib.Decompress.unconsumed_tail`` is empty, matching the CPython fix for gh-158169. ++ Fix ``igzip_lib.IgzipDecompressor.needs_input``, which could be ``True`` + after ``decompress()`` returned ``max_length`` bytes while more output was + still available. Readers that stop at the end of the file when + ``needs_input`` is ``True``, such as CPython's ``DecompressReader``, raised + ``EOFError`` for valid data. + Fix ``isal_zlib.decompressobj`` with a ``wbits`` of 40 to 47, which detects zlib or gzip data. It failed on gzip data when the first ``decompress()`` call got fewer than 2 bytes, for example, when the data arrived byte by diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index 0f5779f0..d2896bb3 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -260,7 +260,11 @@ decompress(IgzipDecompressor *self, uint8_t *data, size_t len, Py_ssize_t max_le } else if (self->avail_in_real == 0) { self->state.next_in = NULL; - self->needs_input = 1; + /* When the output is full, isal_inflate can still read all of the + input and keep the decompressed data in its own buffer. Ask for + more input only when the output was not full, like CPython's + LZMADecompressor does. */ + self->needs_input = (self->state.avail_out != 0); } else { self->needs_input = 0; diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index f2ccce1b..963d135f 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -284,6 +284,20 @@ def test_igzip_decompressor_unused_data_after_max_length(flag, wbits): assert decompressor.unused_data == compressed[stream_end:position] +@pytest.mark.parametrize("max_length", [100, 8192]) +def test_igzip_decompressor_needs_input_after_all_input_is_read(max_length): + # When the output is full, ISA-L can read all of the input and keep the + # decompressed data in its own buffer. needs_input must stay False until + # decompress() has returned that data. + compressed = igzip_lib.compress(DATA) + decompressor = IgzipDecompressor() + result = [decompressor.decompress(compressed, max_length)] + while not decompressor.eof: + assert not decompressor.needs_input + result.append(decompressor.decompress(b"", max_length)) + assert b"".join(result) == DATA + + @pytest.mark.skipif(not hasattr(sys, "getrefcount"), reason="Needs reference counting") def test_igzip_decompressor_keeps_zdict_reference(): From ce49460bae13f5fb2ca685b35fd937ebffed51a3 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Sat, 24 Oct 2026 16:26:59 -0400 Subject: [PATCH 56/73] Test gzip headers with a header CRC and seek() from the end before a read No test read a valid header with FHCRC, which _GzipReader checks: the headers() helper lost its only test in 2c4de51. test_seek reads before it seeks from the end, so seek() never had to find the size. --- tests/test_igzip.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_igzip.py b/tests/test_igzip.py index 4f0e44f3..0cddfaa5 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -534,6 +534,26 @@ def headers(): yield header + crc.to_bytes(2, "little") +@pytest.mark.parametrize("header", list(headers())) +def test_decompress_header(header): + data = b"hello world" * 100 + compressed = (header + isal_zlib.compress(data, wbits=-15) + + struct.pack(" Date: Sat, 24 Oct 2026 16:47:49 -0400 Subject: [PATCH 57/73] Pass a list instead of an iterator to parametrize in test_threaded_write pytest 10 turns the PytestRemovedIn10Warning for iterators into an error. #297 changes the other calls, but this decorator is next to the line that the previous commit changes. --- tests/test_igzip_threaded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 21dd366e..51581c75 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -36,7 +36,7 @@ def test_threaded_read(): @pytest.mark.parametrize(["mode", "threads"], - itertools.product(["wb", "wt"], [1, 3, -1])) + list(itertools.product(["wb", "wt"], [1, 3, -1]))) def test_threaded_write(tmp_path, mode, threads): with open(tmp_path / "output.gz", "wb") as tmp: # Use a small block size to simulate many writes. From 1080620f4b856d33d02b80e1589860415c039ed4 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 27 Oct 2026 15:43:13 -0400 Subject: [PATCH 58/73] Fix -0 and --fast in the igzip command line interface main() chose the level with `args.compresslevel or _COMPRESS_LEVEL_TRADEOFF`, so level 0 fell back to level 2. Set the default level in the argument parser instead. --- CHANGELOG.rst | 2 ++ src/isal/igzip.py | 5 +++-- tests/test_igzip.py | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fa72ae40..a1e5899d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -131,6 +131,8 @@ version 1.8.1-dev + Fix a crash on macOS and PyPy when there is not enough memory to create the internal lock of an ``isal_zlib.Compress``, ``isal_zlib.Decompress``, or ``igzip_lib.IgzipDecompressor`` object. It raises ``MemoryError`` now. ++ Fix the ``-0`` and ``--fast`` options of ``python -m isal.igzip``, which + compressed at level 2 instead of level 0. version 1.8.0 ----------------- diff --git a/src/isal/igzip.py b/src/isal/igzip.py index 6496a465..eb9b8042 100644 --- a/src/isal/igzip.py +++ b/src/isal/igzip.py @@ -305,7 +305,8 @@ def _argument_parser(): "-3", "--best", action="store_const", dest="compresslevel", const=_COMPRESS_LEVEL_BEST, help="use compression level 3 (best)") - compress_group.set_defaults(compress=True) + compress_group.set_defaults(compress=True, + compresslevel=_COMPRESS_LEVEL_TRADEOFF) compress_group.add_argument( "-d", "--decompress", action="store_const", dest="compress", @@ -333,7 +334,7 @@ def main(): import shutil # Only the command line interface needs it args = _argument_parser().parse_args() - compresslevel = args.compresslevel or _COMPRESS_LEVEL_TRADEOFF + compresslevel = args.compresslevel if args.output: out_filepath = args.output diff --git a/tests/test_igzip.py b/tests/test_igzip.py index b31adb85..ee564f98 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -220,6 +220,8 @@ def test_compress_stdin_stdout(monkeypatch, capsysbinary, level): out, err = capsysbinary.readouterr() assert err == b'' assert gzip.decompress(out) == DATA + # The XFL byte of the gzip header is 4 only at the fastest level. + assert out[8] == (4 if level == "0" else 0) def test_decompress_infile_outfile(monkeypatch, tmp_path, capsysbinary): From f2ae56862a59273da9ed9bd8fa52891121a9f651 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 27 Oct 2026 15:43:13 -0400 Subject: [PATCH 59/73] Check the flag byte for FNAME in test_compress_infile_out_file_no_name The test checked byte 4, the first byte of MTIME. The flags are in byte 3. --- tests/test_igzip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_igzip.py b/tests/test_igzip.py index b31adb85..1cd9517e 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -387,7 +387,7 @@ def test_compress_infile_out_file_no_name(monkeypatch, tmp_path, capsysbinary): assert gzip.decompress(output) == DATA assert err == b'' assert out == b'' - assert output[4] & gzip.FNAME == 0 # No filename set. + assert output[3] & gzip.FNAME == 0 # No filename set. assert output[4:8] == b"\x00\x00\x00\x00" # No timestamp set. From 4976ea825765df28e62eaba39fca57d4f0c8e348 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 27 Oct 2026 15:43:28 -0400 Subject: [PATCH 60/73] Test threaded writes with blocks larger than the deflate window With 8 KiB blocks, the last 32 KiB of the previous block is the whole block, so the tests could not tell whether the writer took the dictionary from the end or the start of it. --- tests/test_igzip_threaded.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index dd26e95e..d7710a68 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -35,13 +35,16 @@ def test_threaded_read(): assert thread_data == data +@pytest.mark.parametrize("block_size", [8 * 1024, 64 * 1024]) @pytest.mark.parametrize(["mode", "threads"], list(itertools.product(["wb", "wt"], [1, 3, -1]))) -def test_threaded_write(tmp_path, mode, threads): +def test_threaded_write(tmp_path, mode, threads, block_size): with open(tmp_path / "output.gz", "wb") as tmp: - # Use a small block size to simulate many writes. + # Use small block sizes to simulate many writes. Each block uses up + # to 32 KiB from the end of the previous block as a dictionary, so + # also test a block size above 32 KiB. with igzip_threaded.open(tmp, mode, threads=threads, - block_size=8*1024) as out_file: + block_size=block_size) as out_file: gzip_open_mode = "rb" if "b" in mode else "rt" with gzip.open(TEST_FILE, gzip_open_mode) as in_file: while True: From ae8e4e5e404ac3c4ca931fc1e8140e606e541c96 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 27 Oct 2026 15:43:35 -0400 Subject: [PATCH 61/73] Check that test_close_writer stops the worker threads Use assert_writer_closed(), which the error tests already use. It also checks that the threads are no longer alive. --- tests/test_igzip_threaded.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index dd26e95e..15d334d7 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -224,9 +224,7 @@ def test_close_writer(threads): f = igzip_threaded._ThreadedGzipWriter( io.BytesIO(), threads=threads) f.close() - assert f.closed - # Make sure double closing does not raise errors - f.close() + assert_writer_closed(f) def test_reader_not_writable(): From dfb8b064261a6c4e9da4ebbc10e1073dc969d60e Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 27 Oct 2026 16:10:44 -0400 Subject: [PATCH 62/73] Check the exception type in test_failure of IgzipDecompressor pytest.raises(Exception) also passes on the SystemError that Python raises when decompress() returns NULL without setting an exception. --- tests/test_igzip_lib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index a336e7f7..32ed3c06 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -220,10 +220,10 @@ def test_decompressor_inputbuf_3(self): def test_failure(self): igzd = IgzipDecompressor() - with pytest.raises(Exception): + with pytest.raises(igzip_lib.IsalError): igzd.decompress(self.BAD_DATA * 30) # Make sure there are no internal consistencies - with pytest.raises(Exception): + with pytest.raises(igzip_lib.IsalError): igzd.decompress(self.BAD_DATA * 30) def test_dictionary(self): From 69a226a98c554fd3c999534cf83729774d51f77a Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 27 Oct 2026 16:11:35 -0400 Subject: [PATCH 63/73] Test bufsize values of 0 and -1 in decompress() isal_zlib.decompress() and igzip_lib.decompress() turn a bufsize of 0 into 1 and reject a negative bufsize, as zlib.decompress() does. No test used those values. Without the first check, a bufsize of 0 makes decompress() loop forever. --- tests/test_compat.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_compat.py b/tests/test_compat.py index 55487c8e..3f339851 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -20,7 +20,7 @@ import zlib from pathlib import Path -from isal import igzip, isal_zlib +from isal import igzip, igzip_lib, isal_zlib import pytest @@ -129,6 +129,16 @@ def test_decompress_isal_zlib(data_size, level): assert decompressed == data +@pytest.mark.parametrize("module", [isal_zlib, igzip_lib]) +def test_decompress_bufsize(module): + data = DATA[:10_000] + compressed = module.compress(data) + # Like zlib.decompress, decompress accepts a bufsize of 0. + assert module.decompress(compressed, bufsize=0) == data + with pytest.raises(ValueError, match="bufsize must be non-negative"): + module.decompress(compressed, bufsize=-1) + + @pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel"], list(itertools.product([128 * 1024], range(4), WBITS_RANGE, range(1, 10)))) From ce8f150a97434f1183f711fdb987a23d301d4dae Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 27 Oct 2026 16:12:26 -0400 Subject: [PATCH 64/73] Check the raw streams in test_reader_not_writable and test_writer_not_readable io.BufferedReader.writable() and io.BufferedWriter.readable() return False without asking the raw stream, so the tests could not fail. --- tests/test_igzip_threaded.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index dd26e95e..806783ad 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -229,14 +229,16 @@ def test_close_writer(threads): f.close() +# io.BufferedReader.writable() and io.BufferedWriter.readable() are always +# False, so check the raw streams. def test_reader_not_writable(): with igzip_threaded.open(TEST_FILE, "rb") as f: - assert not f.writable() + assert not f.raw.writable() def test_writer_not_readable(): with igzip_threaded.open(io.BytesIO(), "wb") as f: - assert not f.readable() + assert not f.raw.readable() def test_writer_wrong_level(): From eaf2d972633ea30ac764edf618251bc2ecc5ad19 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 27 Oct 2026 16:09:31 -0400 Subject: [PATCH 65/73] Make the input buffer tests of IgzipDecompressor reach the input buffer isal_inflate decompresses up to about 64 KiB ahead into its own buffer, so it read the 100 to 300 byte inputs of these tests in full, and the input buffer was never used. Use 512 KiB of random data and multiply the offsets by 1,000. --- tests/test_igzip_lib.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index a336e7f7..6dbb7463 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -12,6 +12,7 @@ import itertools import os import pickle +import random import subprocess import sys import textwrap @@ -72,6 +73,12 @@ class TestIgzipDecompressor(): TEXT = DATA DATA = igzip_lib.compress(DATA) BAD_DATA = b"Not a valid deflate block" + # isal_inflate decompresses up to about 64 KiB ahead into its own buffer + # and reads the input it needs for that. For the input buffer tests to + # leave input over, they use inputs of 100 KB or more of incompressible + # data, which needs about as much input as output. + RANDOM_TEXT = random.Random(0).randbytes(512 * 1024) + RANDOM_DATA = igzip_lib.compress(RANDOM_TEXT) def test_decompress(self): decomp = IgzipDecompressor() @@ -168,19 +175,20 @@ def test_decompressor_inputbuf_1(self): out = [] # Create input buffer and fill it - assert igzd.decompress(self.DATA[:100], max_length=0) == b'' + assert igzd.decompress(self.RANDOM_DATA[:100_000], + max_length=0) == b'' # Retrieve some results, freeing capacity at beginning # of input buffer - out.append(igzd.decompress(b'', 2)) + out.append(igzd.decompress(b'', 80_000)) # Add more data that fits into input buffer after # moving existing data to beginning - out.append(igzd.decompress(self.DATA[100:105], 15)) + out.append(igzd.decompress(self.RANDOM_DATA[100_000:105_000], 15)) # Decompress rest of data - out.append(igzd.decompress(self.DATA[105:])) - assert b''.join(out) == self.TEXT + out.append(igzd.decompress(self.RANDOM_DATA[105_000:])) + assert b''.join(out) == self.RANDOM_TEXT def test_decompressor_inputbuf_2(self): # Test reusing input buffer by appending data at the @@ -189,18 +197,19 @@ def test_decompressor_inputbuf_2(self): out = [] # Create input buffer and empty it - assert igzd.decompress(self.DATA[:200], max_length=0) == b'' + assert igzd.decompress(self.RANDOM_DATA[:200_000], + max_length=0) == b'' out.append(igzd.decompress(b'')) # Fill buffer with new data - out.append(igzd.decompress(self.DATA[200:280], 2)) + out.append(igzd.decompress(self.RANDOM_DATA[200_000:280_000], 2)) # Append some more data, not enough to require resize - out.append(igzd.decompress(self.DATA[280:300], 2)) + out.append(igzd.decompress(self.RANDOM_DATA[280_000:300_000], 2)) # Decompress rest of data - out.append(igzd.decompress(self.DATA[300:])) - assert b''.join(out) == self.TEXT + out.append(igzd.decompress(self.RANDOM_DATA[300_000:])) + assert b''.join(out) == self.RANDOM_TEXT def test_decompressor_inputbuf_3(self): # Test reusing input buffer after extending it @@ -209,14 +218,14 @@ def test_decompressor_inputbuf_3(self): out = [] # Create almost full input buffer - out.append(igzd.decompress(self.DATA[:200], 5)) + out.append(igzd.decompress(self.RANDOM_DATA[:200_000], 5)) # Add even more data to it, requiring resize - out.append(igzd.decompress(self.DATA[200:300], 5)) + out.append(igzd.decompress(self.RANDOM_DATA[200_000:300_000], 5)) # Decompress rest of data - out.append(igzd.decompress(self.DATA[300:])) - assert b''.join(out) == self.TEXT + out.append(igzd.decompress(self.RANDOM_DATA[300_000:])) + assert b''.join(out) == self.RANDOM_TEXT def test_failure(self): igzd = IgzipDecompressor() From b8e0ab7459f9624ebda1d2306e7105faa0dfc461 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 27 Oct 2026 16:12:38 -0400 Subject: [PATCH 66/73] Test that igzip_threaded.open() starts the given number of threads No test checked that open() passes threads to the writer. Without it, the writer uses one thread, and the threaded tests still passed. --- tests/test_igzip_threaded.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index dd26e95e..00e6ba68 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -63,6 +63,13 @@ def test_threaded_open_no_threads(): assert isinstance(klass, gzip.GzipFile) +def test_threaded_open_threads(): + with igzip_threaded.open(io.BytesIO(), "wb", threads=3) as f: + workers = f.raw.compression_workers + assert len(workers) == 3 + assert all(worker.is_alive() for worker in workers) + + def test_threaded_open_not_a_file_or_pathlike(): i_am_a_tuple = (1, 2, 3) with pytest.raises(TypeError) as error: From a9e6da0a1ed17107f10de99aa28727500f8dad29 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 27 Oct 2026 16:12:02 -0400 Subject: [PATCH 67/73] Test that IGzipFile compresses with isal_zlib at the given level When IGzipFile.__init__ does not set self.compress, GzipFile compresses with zlib instead, and the igzip tests still passed. --- tests/test_igzip.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_igzip.py b/tests/test_igzip.py index b31adb85..ff87fabb 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -43,6 +43,23 @@ def test_repr(tmp_path): assert " Date: Mon, 2 Nov 2026 06:11:50 -0500 Subject: [PATCH 70/73] Check the arguments IGzipFile passes to compressobj, not the output The review asked for a test of the parameters instead of a byte-for-byte comparison of the deflate data. The test now records the calls to isal_zlib.compressobj and checks that f.compress is an isal_zlib.Compress. --- tests/test_igzip.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/test_igzip.py b/tests/test_igzip.py index ff87fabb..ea074b1d 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -44,20 +44,20 @@ def test_repr(tmp_path): @pytest.mark.parametrize("level", range(4)) -def test_igzipfile_compresses_with_isal_zlib(level): - # 30,000 bytes fit in the write buffer that GzipFile has on Python 3.12 - # and later (32 KiB or more), so IGzipFile compresses them with one - # compress() call. - with gzip.open(TEST_FILE, "rb") as f: - data = f.read(30_000) - buffer = io.BytesIO() - with igzip.IGzipFile(fileobj=buffer, mode="wb", compresslevel=level, - mtime=0) as f: - f.write(data) - compressor = isal_zlib.compressobj(level, wbits=-isal_zlib.MAX_WBITS) - expected = compressor.compress(data) + compressor.flush() - # Leave out the 10-byte header and the 8-byte trailer. - assert buffer.getvalue()[10:-8] == expected +def test_igzipfile_compressobj_parameters(monkeypatch, level): + calls = [] + compressobj = isal_zlib.compressobj + + def recording_compressobj(*args, **kwargs): + calls.append((args, kwargs)) + return compressobj(*args, **kwargs) + + monkeypatch.setattr(isal_zlib, "compressobj", recording_compressobj) + with igzip.IGzipFile(fileobj=io.BytesIO(), mode="wb", + compresslevel=level) as f: + assert isinstance(f.compress, isal_zlib.Compress) + assert calls == [((level, isal_zlib.DEFLATED, -isal_zlib.MAX_WBITS, + isal_zlib.DEF_MEM_LEVEL, 0), {})] def test_write_readonly_file(): From c6f18bc7f21463bb498ea8c535bbf8f14342ca0b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 2 Nov 2026 13:49:40 +0100 Subject: [PATCH 71/73] Organize changelog and update version statement --- CHANGELOG.rst | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9c103b92..cc1e6938 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,8 +8,22 @@ Changelog .. that users understand how the changes affect the new version. -version 1.8.1-dev +version 1.9.0 ----------------- + +Major changes: + ++ Updated included ISA-L library to 2.32.1. ++ Python 3.10 and PyPy 3.10 are no longer supported. Added support for PyPy + 3.12 and Python 3.15. ++ Restore PyPy wheel builds. ++ Fix builds on macOS with Apple Silicon, which used the portable C code of + ISA-L rather than its optimized aarch64 assembly. On an Apple M4, + decompression is 1.6 to 3 times faster, CRC-32 is 50 times faster, and + Adler-32 is 7 times faster. + +Optimizations: + + ``isal_zlib.compressobj`` supports ``strategy=Z_FIXED`` at compression level 0. Level 0 then uses fixed Huffman codes instead of starting each block with a Huffman table of about 110 bytes. This shrinks small flushed @@ -21,17 +35,19 @@ version 1.8.1-dev + Speed up ``import isal.igzip`` and ``import isal.igzip_threaded`` by importing ``argparse`` and ``shutil`` only when the command line interface runs, and by dropping the ``multiprocessing`` import. -+ Updated ISA-L to 2.32.1. -+ Python 3.10 and PyPy 3.10 are no longer supported. Added support for PyPy - 3.12 and Python 3.15. -+ Restore PyPy wheel builds. + Speed up reading lines from binary ``igzip`` files through iteration, ``readlines()``, and ``readline()``. On an Apple M4, reading FASTQ data is 1.13 times faster, and reading lines of a few bytes is 1.5 times faster. + Speed up each ``write()`` call to binary ``igzip`` files on CPython. On an AMD Ryzen 7 9800X3D with Python 3.14.8, writing FASTQ data or Python source code one line at a time is 1.1 times faster. -+ Fix zlib streams compressed with a preset dictionary so they can be decoded ++ Speed up many small writes to ``igzip.IGzipFile`` on Python 3.12 and later. + ``IGzipFile`` collects these writes in the write buffer of + ``gzip.GzipFile`` and compresses them in large chunks, which makes writing + lines 1.4 times faster. + +Bug fixes: + + Fix zlib streams compressed with a preset dictionary so they can be decoded by standard zlib, including builds linked against older ISA-L releases. + Fix backward seeking in igzip files by discarding buffered compressed data when rewinding the underlying file. @@ -81,14 +97,6 @@ version 1.8.1-dev + Fix ``igzip_threaded`` readers and writers that waited for a polling timeout of up to 50 ms at the end of the stream or when closed. Writing and closing a small file takes less than a millisecond instead of 60 ms. -+ Speed up many small writes to ``igzip.IGzipFile`` on Python 3.12 and later. - ``IGzipFile`` collects these writes in the write buffer of - ``gzip.GzipFile`` and compresses them in large chunks, which makes writing - lines 1.4 times faster. -+ Fix builds on macOS with Apple Silicon, which used the portable C code of - ISA-L rather than its optimized aarch64 assembly. On an Apple M4, - decompression is 1.6 to 3 times faster, CRC-32 is 50 times faster, and - Adler-32 is 7 times faster. + Fix ``igzip_threaded`` writers that waited forever when writing to the underlying file failed, for example, because the disk was full. A later call to ``write()``, ``flush()``, or ``close()`` raises the error. After an error, @@ -108,9 +116,6 @@ version 1.8.1-dev private ``isal_zlib._GzipReader`` class raise ``TypeError`` instead of ``SystemError`` for arguments of the wrong type or number. ``igzip.IGzipFile`` was not affected. -+ Fix crashes and wrong data when one thread calls ``seek()`` of the - private ``isal_zlib._GzipReader`` class while another thread reads from the - same object. ``igzip.IGzipFile`` was not affected. + Fix a crash on PyPy when you create an ``isal_zlib.Compress`` or ``isal_zlib.Decompress`` object by calling the type, for example, ``isal_zlib.Compress()``, and then use it. Calling the type raises @@ -138,6 +143,10 @@ version 1.8.1-dev ``igzip_lib.IgzipDecompressor`` object. It raises ``MemoryError`` now. + Fix the ``-0`` and ``--fast`` options of ``python -m isal.igzip``, which compressed at level 2 instead of level 0. ++ Fix crashes and wrong data when one thread calls ``seek()`` of the + private ``isal_zlib._GzipReader`` class while another thread reads from the + same object. ``igzip.IGzipFile`` was not affected. + version 1.8.0 ----------------- From 2ced63f3c0732ba3257aaad7cbb25b3b1270ba6d Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 2 Nov 2026 13:57:43 +0100 Subject: [PATCH 72/73] Properly credit @ptweezy --- CHANGELOG.rst | 6 +++++- README.rst | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cc1e6938..344f3b2d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,10 @@ Changelog version 1.9.0 ----------------- +By far most of the work on this release has been done by +`@ptweezy `_. Almost all bug fixes and improvements +are their work. + Major changes: + Updated included ISA-L library to 2.32.1. @@ -22,7 +26,7 @@ Major changes: decompression is 1.6 to 3 times faster, CRC-32 is 50 times faster, and Adler-32 is 7 times faster. -Optimizations: +Optimizations abd features: + ``isal_zlib.compressobj`` supports ``strategy=Z_FIXED`` at compression level 0. Level 0 then uses fixed Huffman codes instead of starting each diff --git a/README.rst b/README.rst index 0ac85ac4..fd4ce61b 100644 --- a/README.rst +++ b/README.rst @@ -224,6 +224,8 @@ This project builds upon the software and experience of many. Many thanks to: + The `github actions team `_ for creating the actions CI service that enables building and testing on all three major operating systems. ++ `@ptweezy `_ for discovering and fixing lots and + lots of bugs as well as suggesting and implementing quite a few improvements. + `@animalize `_ for explaining how to test and build python-isal for ARM 64-bit platforms. + And last but not least: everyone who submitted a bug report or a feature From b58a5995801232d7e4fc1972df3e291be735cc48 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 2 Nov 2026 14:11:14 +0100 Subject: [PATCH 73/73] Fix changelog build errors --- CHANGELOG.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 344f3b2d..307b330b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -26,7 +26,7 @@ Major changes: decompression is 1.6 to 3 times faster, CRC-32 is 50 times faster, and Adler-32 is 7 times faster. -Optimizations abd features: +Optimizations and features: + ``isal_zlib.compressobj`` supports ``strategy=Z_FIXED`` at compression level 0. Level 0 then uses fixed Huffman codes instead of starting each @@ -51,7 +51,8 @@ Optimizations abd features: lines 1.4 times faster. Bug fixes: - + Fix zlib streams compressed with a preset dictionary so they can be decoded + ++ Fix zlib streams compressed with a preset dictionary so they can be decoded by standard zlib, including builds linked against older ISA-L releases. + Fix backward seeking in igzip files by discarding buffered compressed data when rewinding the underlying file.