diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e2bc13d..ec279fb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -16,13 +16,13 @@ jobs:
lint:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
with:
submodules: recursive
- - name: Set up Python 3.10
- uses: actions/setup-python@v5
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v7
with:
- python-version: "3.10"
+ python-version: "3.11"
- name: Install tox
run: pip install tox
- name: Lint
@@ -36,13 +36,13 @@ jobs:
- twine_check
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
with:
submodules: recursive
- - name: Set up Python 3.10
- uses: actions/setup-python@v5
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v7
with:
- python-version: "3.10"
+ python-version: "3.11"
- name: Install tox and upgrade setuptools and pip
run: pip install --upgrade tox setuptools pip
- name: Run tox -e ${{ matrix.tox_env }}
@@ -53,28 +53,31 @@ jobs:
strategy:
matrix:
python-version:
- - "3.10"
- "3.11"
- "3.12"
- "3.13"
- "3.14"
- "3.14t"
- - "pypy-3.10"
+ - "3.15"
+ - "3.15t"
+ - "pypy-3.11"
- "pypy-3.11"
os: ["ubuntu-latest"]
include:
- os: "macos-latest" # For m1 macos
python-version: "3.12"
- - os: "macos-13" # for x86 macos
- python-version: "3.10"
+ - os: "macos-15-intel" # for x86 macos
+ python-version: "3.11"
- os: "windows-latest"
- python-version: "3.10"
+ python-version: "3.11"
+ - os: "windows-11-arm"
+ python-version: "3.11"
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
with:
submodules: recursive
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v7
with:
python-version: ${{ matrix.python-version }}
allow-prereleases: true
@@ -83,13 +86,15 @@ jobs:
- name: Set MSVC developer prompt
uses: ilammy/msvc-dev-cmd@v1
if: runner.os == 'Windows'
+ with:
+ arch: ${{ matrix.os == 'windows-11-arm' && 'arm64' || 'x64' }}
- name: Install build dependencies (MacOS)
run: brew install make
if: runner.os == 'macOS'
- name: Run tests
run: tox
- name: Upload coverage report
- uses: codecov/codecov-action@v1
+ uses: codecov/codecov-action@v7
test-arch:
if: startsWith(github.ref, 'refs/tags') || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main'
@@ -97,9 +102,9 @@ jobs:
strategy:
matrix:
python_version:
- - "3.10"
+ - "3.11"
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
with:
submodules: recursive
- uses: uraimo/run-on-arch-action@v3
@@ -129,19 +134,16 @@ jobs:
matrix:
os:
- "ubuntu-latest"
- - "macos-13"
+ - "macos-15-intel"
- "macos-latest"
- "windows-latest"
- python_version: [ "python" ]
- include:
- - os: "ubuntu-latest"
- python_version: "pypy"
+ python_version: [ "python==3.11" ]
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
with:
submodules: recursive
- name: Install miniconda.
- uses: conda-incubator/setup-miniconda@v3 # https://github.com/conda-incubator/setup-miniconda.
+ uses: conda-incubator/setup-miniconda@v4 # https://github.com/conda-incubator/setup-miniconda.
with:
channels: conda-forge,defaults
- name: Install requirements (universal)
@@ -154,7 +156,75 @@ jobs:
env:
PYTHON_ZLIB_NG_LINK_DYNAMIC: True
- deploy:
+ # Run the tests on CPython builds with AddressSanitizer and ThreadSanitizer
+ # from https://github.com/nascheme/cpython_sanity.
+ test-sanitizers:
+ runs-on: ubuntu-latest
+ # A crash under ThreadSanitizer can hang instead of exiting.
+ timeout-minutes: 30
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - sanitizer: address
+ image: ghcr.io/nascheme/cpython-asan:3.14
+ # The free-threaded build runs Python threads in parallel.
+ - sanitizer: thread
+ image: ghcr.io/nascheme/cpython-tsan:3.14t
+ container:
+ image: ${{ matrix.image }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+ - name: Install build dependencies
+ run: apt-get update && apt-get install -y cmake
+ - name: Install python-isal with ${{ matrix.sanitizer }} sanitizer
+ # setuptools-scm needs git, and the container user does not own the
+ # repository.
+ run: |
+ git config --global --add safe.directory "$GITHUB_WORKSPACE"
+ python -m pip install . pytest pytest-timeout
+ env:
+ # The extension modules get -fsanitize from CPython. CFLAGS adds it
+ # to the C code of ISA-L.
+ CFLAGS: -fsanitize=${{ matrix.sanitizer }} -fno-omit-frame-pointer -g
+ - name: Run tests
+ # Do not set PYTHONDEVMODE. Its debug allocator pads each allocation,
+ # so AddressSanitizer misses small overflows. --capture=sys prints
+ # sanitizer reports from C code even when they end the process, and
+ # -v prints the name of the test before its report.
+ run: python -m pytest -v tests --capture=sys
+ env:
+ # Exit at the first data race. A crash after a race can hang.
+ TSAN_OPTIONS: halt_on_error=1
+
+ make_sdist:
+ if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+ needs:
+ - lint
+ - package-checks
+ - test-static
+ - test-dynamic
+ - test-arch
+ - test-sanitizers
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ fetch-depth: 0 # Optional, use if you use setuptools_scm
+ submodules: true # Optional, use if you have submodules
+ persist-credentials: false # Highly recommended as a good security practice to not store credentials in disk, unless you need this for your specific use case
+
+ - name: Build SDist
+ run: pipx run build --sdist
+
+ - uses: actions/upload-artifact@v4
+ with:
+ name: cibw-sdist
+ path: dist/*.tar.gz
+
+ build_wheels:
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
runs-on: ${{ matrix.os }}
needs:
@@ -163,24 +233,26 @@ jobs:
- test-static
- test-dynamic
- test-arch
+ - test-sanitizers
strategy:
matrix:
os:
- ubuntu-latest
- - macos-13
+ - macos-15-intel
- macos-latest
- windows-latest
+ - windows-11-arm
cibw_archs_linux: ["x86_64"]
build_sdist: [true]
include:
- os: "ubuntu-latest"
cibw_archs_linux: "aarch64"
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
with:
submodules: recursive
fetch-depth: 0 # Fetch everything to get accurately versioned tag.
- - uses: actions/setup-python@v2 # Some issues where caused by higher versions.
+ - uses: actions/setup-python@v7
name: Install Python
- name: Install cibuildwheel twine build
run: python -m pip install cibuildwheel twine build
@@ -190,16 +262,19 @@ jobs:
- name: Set MSVC developer prompt
uses: ilammy/msvc-dev-cmd@v1
if: runner.os == 'Windows'
+ with:
+ arch: ${{ matrix.os == 'windows-11-arm' && 'arm64' || 'x64' }}
- name: Set up QEMU
if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'aarch64'}}
- uses: docker/setup-qemu-action@v3
+ uses: docker/setup-qemu-action@v4
with:
platforms: arm64
- name: Build wheels
run: cibuildwheel --output-dir dist
env:
+ CIBW_ENABLE: "pypy"
# Skip 32 bit, macosx_arm64 causes issues on cpython 3.9
- CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_arm64"
+ CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_arm64 cp39-win_arm64 cp310-win_arm64"
CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }}
CIBW_TEST_REQUIRES: "pytest"
# Simple tests that requires the project to be build correctly
@@ -231,21 +306,35 @@ jobs:
if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'x86_64'}}
run:
python -m build --sdist
- - uses: actions/upload-artifact@v4
+ - uses: actions/upload-artifact@v7
with:
- name: "dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cibw_archs_linux }}"
+ name: "cibw-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cibw_archs_linux }}"
path: "dist/"
- - name: Publish package to TestPyPI
- # pypa/gh-action-pypi-publish@master does not work on OSX
+
+ upload_to_pypi:
+ needs: [ build_wheels, make_sdist ]
+ environment: pypi
+ permissions:
+ id-token: write
+ runs-on: ubuntu-latest
+ if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+ steps:
+ - uses: actions/download-artifact@v5
+ with:
+ pattern: cibw-*
+ path: dist
+ merge-multiple: true
+
+ - name: Publish to test PyPI
# Alpha, Beta and dev releases contain a - in the tag.
- if: contains(github.ref, '-') && startsWith(github.ref, 'refs/tags')
- run: twine upload --skip-existing -r testpypi dist/*
- env:
- TWINE_USERNAME: __token__
- TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
- - name: Publish package to PyPI
+ if: "contains(github.ref, '-') && startsWith(github.ref, 'refs/tags')"
+ uses: pypa/gh-action-pypi-publish@release/v1
+ with:
+ repository-url: https://test.pypi.org/legacy/
+ skip-existing: true
+
+ - name: Publish to PyPI
if: "!contains(github.ref, '-') && startsWith(github.ref, 'refs/tags')"
- run: twine upload --skip-existing dist/*
- env:
- TWINE_USERNAME: __token__
- TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+ uses: pypa/gh-action-pypi-publish@release/v1
+ with:
+ skip-existing: true
diff --git a/.readthedocs.yml b/.readthedocs.yml
index cb71cd8..1a83c66 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -14,6 +14,8 @@ build:
os: "ubuntu-22.04"
tools:
python: "3"
+ apt_packages:
+ - cmake
# Make sure zlib-ng is cloned alongside
submodules:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index c067325..11ec8d1 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -7,6 +7,31 @@ Changelog
.. This document is user facing. Please word the changes in such a way
.. that users understand how the changes affect the new version.
+version 1.1.0
+-----------------
++ Update included zlib-ng distribution to 2.3.3.
++ Fix BrokenPipeError when piping ``python -m isal.igzip`` into ``head``,
+ ``less`` or similar programs.
++ Fix ``gzip_ng_threaded`` writers that waited forever when writing to the
+ underlying file failed, for example, because the disk was full. A later call
+ to ``write()``, ``flush()``, or ``close()`` raises the error. After an error,
+ ``close()`` no longer closes a file object that you passed to
+ ``gzip_ng_threaded.open``.
++ Wheels are now built for Windows arm64 architectures.
++ Restore PyPy wheel builds.
++ Fix a crash when calling ``copy()`` on a flushed compress object on Python
+ 3.15. Python 3.15 changed
+ (`gh-134745 `__)
+ ``PyThread_release_lock`` to use ``PyMutex`` internally, which now raises a
+ fatal error when releasing a lock that was never acquired.
++ Fix ``gzip_ng_threaded`` writers at compression level 1, which raised
+ ``OverflowError`` for data that does not compress and consists mostly of
+ bytes 144 to 255.
++ Extraordinarily large gzip headers with either a very long comment or
+ filename will now cause an ``OverflowError`` rather than use more and more
+ memory. The size is limited to the internal buffer size which is currently
+ set at 512 KiB.
+
version 1.0.0
-----------------
The library has been running without issues as a dependency in quite a few
diff --git a/setup.py b/setup.py
index f58cbf8..6d9f267 100644
--- a/setup.py
+++ b/setup.py
@@ -107,15 +107,12 @@ def build_zlib_ng():
subprocess.run([os.path.join(build_dir, "configure")], **run_args)
make_program = "gmake" if shutil.which("gmake") else "make"
subprocess.run([make_program, "libz-ng.a"], **run_args)
- elif sys.platform == "linux":
- subprocess.run([os.path.join(build_dir, "configure")], **run_args)
- subprocess.run(["make", "libz-ng.a", "-j", str(cpu_count)], **run_args)
else:
subprocess.run(["cmake", build_dir], **run_args)
# Do not create test suite and do not perform tests to shorten build times.
# There is no need when stable releases of zlib-ng are used.
subprocess.run(["cmake", "--build", build_dir, "--config", "Release",
- "--target", "zlibstatic",
+ "--target", "zlib-ng-static",
"-j", str(cpu_count)], **run_args)
if BUILD_CACHE:
BUILD_CACHE_FILE.write_text(build_dir)
diff --git a/src/zlib_ng/gzip_ng.py b/src/zlib_ng/gzip_ng.py
index f24bdbe..c2afec2 100644
--- a/src/zlib_ng/gzip_ng.py
+++ b/src/zlib_ng/gzip_ng.py
@@ -304,6 +304,9 @@ def main():
READ_BUFFER_SIZE = args.buffer_size
try:
shutil.copyfileobj(in_file, out_file, args.buffer_size)
+ except BrokenPipeError:
+ # Do not throw errors when a pipe to head or less closes.
+ pass
finally:
if in_file is not sys.stdin.buffer:
in_file.close()
diff --git a/src/zlib_ng/gzip_ng_threaded.py b/src/zlib_ng/gzip_ng_threaded.py
index dee8be7..87c7f6b 100644
--- a/src/zlib_ng/gzip_ng_threaded.py
+++ b/src/zlib_ng/gzip_ng_threaded.py
@@ -226,9 +226,9 @@ def __init__(self,
self.exception: Optional[Exception] = None
self.level = level
self.previous_block = b""
- # Deflating random data results in an output a little larger than the
- # input. Making the output buffer 10% larger is sufficient overkill.
- compress_buffer_size = block_size + max(block_size // 10, 500)
+ # Level 1 never falls back to stored blocks and codes bytes 144 to 255
+ # with 9 bits each, so its output can be an eighth larger.
+ compress_buffer_size = block_size + block_size // 8 + 500
self.block_size = block_size
self.compressors: List[zlib_ng._ParallelCompress] = [
zlib_ng._ParallelCompress(buffersize=compress_buffer_size,
@@ -329,25 +329,32 @@ def flush(self):
# Wait for all data to be written
for out_q in self.output_queues:
out_q.join()
+ if self.exception:
+ raise self.exception
self.raw.flush()
def close(self):
if self._closed:
return
- self.raw.write(zlib_ng.compress(b"", wbits=-15))
- trailer = struct.pack(" bool:
@@ -388,7 +395,7 @@ def _write(self):
continue
self._crc = zlib_ng.crc32_combine(self._crc, crc, data_length)
self._size += data_length
- self.raw.write(compressed)
+ self._write_compressed(compressed)
output_queue.task_done()
index += 1
@@ -413,9 +420,22 @@ def _compress_and_write(self):
data_length = len(data)
self._crc = zlib_ng.crc32_combine(self._crc, crc, data_length)
self._size += data_length
- self.raw.write(compressed)
+ self._write_compressed(compressed)
in_queue.task_done()
+ def _write_compressed(self, compressed: bytes):
+ # After a write error, this thread keeps taking blocks from the queues
+ # and drops them until stop() ends it. That way, no thread waits
+ # forever on a full queue, and write(), flush(), and close() can raise
+ # the error in the calling thread.
+ if self.exception:
+ return
+ try:
+ self.raw.write(compressed)
+ except Exception as e:
+ with self.lock:
+ self.exception = e
+
def _set_error_and_empty_queue(self, error, q):
with self.lock:
self.exception = error
diff --git a/src/zlib_ng/zlib-ng b/src/zlib_ng/zlib-ng
index 4254390..1273109 160000
--- a/src/zlib_ng/zlib-ng
+++ b/src/zlib_ng/zlib-ng
@@ -1 +1 @@
-Subproject commit 425439062b114a0f6cf625022c41d929c7e879f9
+Subproject commit 12731092979c6d07f42da27da673a9f6c7b13586
diff --git a/src/zlib_ng/zlib_ngmodule.c b/src/zlib_ng/zlib_ngmodule.c
index ea71d53..59233e5 100644
--- a/src/zlib_ng/zlib_ngmodule.c
+++ b/src/zlib_ng/zlib_ngmodule.c
@@ -801,7 +801,8 @@ zlib_Compress_copy(compobject *self, PyObject *Py_UNUSED(ignored))
if (!self->is_initialised) {
PyErr_SetString(PyExc_ValueError, "Cannot copy flushed objects.");
- goto error;
+ Py_DECREF(return_value);
+ return NULL;
}
/* Copy the zstream state
@@ -2355,17 +2356,15 @@ GzipReader_read_from_file(GzipReader *self)
const uint8_t *buffer_end = self->buffer_end;
size_t remaining = buffer_end - current_pos;
if (remaining == self->buffer_size) {
- /* Buffer is full but a new read request was issued. This will be due
- to the header being bigger than the header. Enlarge the buffer
- to accommodate the hzip header. */
- size_t new_buffer_size = self->buffer_size * 2;
- uint8_t *tmp_buffer = PyMem_Realloc(self->input_buffer, new_buffer_size);
- if (tmp_buffer == NULL) {
- PyErr_NoMemory();
- return -1;
- }
- self->input_buffer = tmp_buffer;
- self->buffer_size = new_buffer_size;
+ /* Buffer is full but a new read request was issued. This will be due
+ to the header being bigger than the buffer. To prevent adverserial
+ inputs with infinite header sizes an error is thrown here. */
+ PyErr_Format(
+ PyExc_OverflowError,
+ "Gzip header exceeds input buffer size of %lu",
+ self->buffer_size
+ );
+ return -1;
} else if (remaining > 0) {
memmove(self->input_buffer, current_pos, remaining);
}
diff --git a/tests/test_compat.py b/tests/test_compat.py
index 12dda97..871bf84 100644
--- a/tests/test_compat.py
+++ b/tests/test_compat.py
@@ -64,21 +64,21 @@ def limited_zlib_tests(strategies=ZLIB_STRATEGIES):
@pytest.mark.parametrize(["data_size", "value"],
- itertools.product(DATA_SIZES, SEEDS))
+ list(itertools.product(DATA_SIZES, SEEDS)))
def test_crc32(data_size, value):
data = DATA[:data_size]
assert zlib.crc32(data, value) == zlib_ng.crc32(data, value)
@pytest.mark.parametrize(["data_size", "value"],
- itertools.product(DATA_SIZES, SEEDS))
+ list(itertools.product(DATA_SIZES, SEEDS)))
def test_adler32(data_size, value):
data = DATA[:data_size]
assert zlib.adler32(data, value) == zlib_ng.adler32(data, value)
@pytest.mark.parametrize(["data_size", "level", "wbits"],
- itertools.product(DATA_SIZES, range(10), WBITS_RANGE))
+ list(itertools.product(DATA_SIZES, range(10), WBITS_RANGE)))
def test_compress(data_size, level, wbits):
data = DATA[:data_size]
compressed = zlib_ng.compress(data, level=level, wbits=wbits)
@@ -87,7 +87,7 @@ def test_compress(data_size, level, wbits):
@pytest.mark.parametrize(["data_size", "level"],
- itertools.product(DATA_SIZES, range(10)))
+ list(itertools.product(DATA_SIZES, range(10))))
def test_decompress_zlib(data_size, level):
data = DATA[:data_size]
compressed = zlib.compress(data, level=level)
@@ -96,7 +96,7 @@ def test_decompress_zlib(data_size, level):
@pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel", "strategy"],
- limited_zlib_tests(ZLIB_STRATEGIES))
+ list(limited_zlib_tests(ZLIB_STRATEGIES)))
def test_decompress_wbits(data_size, level, wbits, memLevel, strategy):
data = DATA[:data_size]
compressobj = zlib.compressobj(level=level, wbits=wbits, memLevel=memLevel,
@@ -107,7 +107,7 @@ def test_decompress_wbits(data_size, level, wbits, memLevel, strategy):
@pytest.mark.parametrize(["data_size", "level", "wbits"],
- itertools.product([128 * 1024], range(10), WBITS_RANGE),)
+ list(itertools.product([128 * 1024], range(10), WBITS_RANGE),))
def test_decompress_zlib_ng(data_size, level, wbits):
data = DATA[:data_size]
compressed = zlib_ng.compress(data, level=level, wbits=wbits)
@@ -116,7 +116,7 @@ def test_decompress_zlib_ng(data_size, level, wbits):
@pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel", "strategy"],
- limited_zlib_tests(ZLIBNG_STRATEGIES))
+ list(limited_zlib_tests(ZLIBNG_STRATEGIES)))
def test_compress_compressobj(data_size, level, wbits, memLevel, strategy):
data = DATA[:data_size]
compressobj = zlib_ng.compressobj(level=level,
@@ -129,7 +129,7 @@ def test_compress_compressobj(data_size, level, wbits, memLevel, strategy):
@pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel", "strategy"],
- limited_zlib_tests(ZLIB_STRATEGIES))
+ list(limited_zlib_tests(ZLIB_STRATEGIES)))
def test_decompress_decompressobj(data_size, level, wbits, memLevel, strategy):
data = DATA[:data_size]
compressobj = zlib.compressobj(level=level, wbits=wbits, memLevel=memLevel,
@@ -151,7 +151,7 @@ def test_decompressobj_unconsumed_tail():
@pytest.mark.parametrize(["data_size", "level"],
- itertools.product(DATA_SIZES, range(10)))
+ list(itertools.product(DATA_SIZES, range(10))))
def test_gzip_ng_compress(data_size, level):
data = DATA[:data_size]
compressed = gzip_ng.compress(data, compresslevel=level)
@@ -159,7 +159,7 @@ def test_gzip_ng_compress(data_size, level):
@pytest.mark.parametrize(["data_size", "level"],
- itertools.product(DATA_SIZES, range(10)))
+ list(itertools.product(DATA_SIZES, range(10))))
def test_decompress_gzip(data_size, level):
data = DATA[:data_size]
compressed = gzip.compress(data, compresslevel=level)
@@ -168,7 +168,7 @@ def test_decompress_gzip(data_size, level):
@pytest.mark.parametrize(["data_size", "level"],
- itertools.product(DATA_SIZES, range(10)))
+ list(itertools.product(DATA_SIZES, range(10))))
def test_decompress_gzip_ng(data_size, level):
data = DATA[:data_size]
compressed = gzip_ng.compress(data, compresslevel=level)
@@ -177,7 +177,7 @@ def test_decompress_gzip_ng(data_size, level):
@pytest.mark.parametrize(["unused_size", "wbits"],
- itertools.product([26], [-15, 15, 31]))
+ list(itertools.product([26], [-15, 15, 31])))
def test_unused_data(unused_size, wbits):
unused_data = b"abcdefghijklmnopqrstuvwxyz"[:unused_size]
compressor = zlib.compressobj(wbits=wbits)
diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py
index 8ad5441..64ea4b7 100644
--- a/tests/test_gzip_compliance.py
+++ b/tests/test_gzip_compliance.py
@@ -809,7 +809,7 @@ def test_decompress_stdin_stdout(self):
# If the env_vars argument to assert_python_failure or assert_python_ok
# is empty the test will run in isolated mode (-I) which means that the
# PYTHONPATH environment variable will be ignored and the test fails
- # because the isal module can not be found, or the test is run usung the
+ # because the zlib_ng module can not be found, or the test is run using the
# system installed version of the module instead of the newly built
# module that should be tested.
#
diff --git a/tests/test_gzip_ng.py b/tests/test_gzip_ng.py
index abfd283..41eb954 100644
--- a/tests/test_gzip_ng.py
+++ b/tests/test_gzip_ng.py
@@ -14,6 +14,8 @@
import os
import re
import shutil
+import struct
+import subprocess
import sys
import tempfile
import zlib
@@ -65,7 +67,7 @@ def test_GzipNGFile_read_truncated():
"reached")
-@pytest.mark.parametrize("level", range(1, 10))
+@pytest.mark.parametrize("level", list(range(1, 10)))
def test_decompress_stdin_stdout(capsysbinary, level):
"""Test if the command line can decompress data that has been compressed
by gzip at all levels."""
@@ -409,7 +411,7 @@ def test_very_long_header_in_data():
def test_very_long_header_in_file():
# header with a very long filename.
header = (b"\x1f\x8b\x08\x08\x00\x00\x00\x00\x00\xff" +
- gzip_ng.READ_BUFFER_SIZE * 2 * b"A" +
+ (gzip_ng.READ_BUFFER_SIZE - 11) * b"A" +
b"\x00")
compressed = header + zlib_ng.compress(b"", 3, -15) + 8 * b"\00"
f = io.BytesIO(compressed)
@@ -471,3 +473,38 @@ def test_bgzip():
with gzip_ng.open(gzip_file, "rb") as gz:
gz_data = gz.read()
assert bgz_data == gz_data
+
+
+def test_adverserial_header():
+ magic = b"\x1f\x8b"
+ method = 8
+ mtime = 0
+ flags = FCOMMENT
+ _os = 255
+ xfl = 0
+ header = struct.pack("<2sBBIBB", magic, method, flags, mtime, xfl, _os)
+ comment = b"A" * 10 * 1024 * 1024 + b"\00"
+ compressed = zlib_ng.compress(b"", wbits=-15)
+ crc = zlib_ng.crc32(b"")
+ length = 0
+ trailer = struct.pack("