diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..e5f73e9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + groups: + github-actions: + patterns: + - "*" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b5f329..e2bd9cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10"] + python-version: ["3.14"] toxenv: [black, ruff, mypy] steps: - uses: actions/checkout@v6 @@ -34,28 +34,28 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] + python-version: ["3.11", "3.12", "3.13", "3.14", "3.14t"] optional-deps: [true] with-libs: [true] include: - os: macos-latest - python-version: "3.10" + python-version: "3.14" optional-deps: true - os: ubuntu-latest - python-version: "3.10" + python-version: "3.14" with-libs: false optional-deps: false - os: ubuntu-latest - python-version: "3.10" + python-version: "3.14" with-libs: false optional-deps: true - os: ubuntu-latest - python-version: "3.10" + python-version: "3.14" optional-deps: false with-libs: false with-zstandard: true - os: windows-latest - python-version: "3.10" + python-version: "3.14" steps: - name: Install optional tools macOS if: runner.os == 'macOS' && matrix.optional-deps @@ -88,7 +88,7 @@ jobs: if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' && matrix.optional-deps && matrix.with-libs run: tox run -e lz4 - name: Upload coverage report - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v7 deploy: timeout-minutes: 10 @@ -102,7 +102,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.10" + python-version: "3.14" - name: Make distributions run: | python -m pip install build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f902a9a..9046008 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,6 +5,6 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 26.10.0 hooks: - id: black diff --git a/README.rst b/README.rst index 3690fd3..b854367 100644 --- a/README.rst +++ b/README.rst @@ -187,6 +187,15 @@ the ``compression.zstd`` module in the standard library is used. Changelog --------- +development version +~~~~~~~~~~~~~~~~~~~ + +* #181/#168: Added lz4 as supported compression format. + Contributed by @Dev-iL and @gnzsnz. +* PipedCompressionReader no longer has seek/tell methods. This only worked + on Windows before (and only sometimes). +* Dropped support for Python 3.10 + v2.1.0 (2026-06-03) ~~~~~~~~~~~~~~~~~~~ diff --git a/pyproject.toml b/pyproject.toml index 71a6825..0922244 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ "Development Status :: 5 - Production/Stable", "Programming Language :: Python :: 3" ] -requires-python = ">=3.10" +requires-python = ">=3.11" dynamic = ["version"] dependencies = [ 'isal>=1.6.1; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64"', @@ -35,7 +35,12 @@ zstd = [] # Leave this in here for backwards compatibility (Zstandard support u write_to = "src/xopen/_version.py" [tool.ruff.lint] -select = ["E4", "E7", "E9", "F"] +ignore = [ + "B008", # Do not perform function call `_ProgramSettings` in argument defaults; instead, perform the call within the function, or read the default from a module-level singleton variable + "C408", # Unnecessary `dict()` call (rewrite as a literal) + "SIM115", # Use a context manager for opening files + "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements +] [tool.pytest.ini_options] addopts = "--strict-markers" diff --git a/src/xopen/__init__.py b/src/xopen/__init__.py index 7462a9c..356a31c 100644 --- a/src/xopen/__init__.py +++ b/src/xopen/__init__.py @@ -3,35 +3,26 @@ """ __all__ = [ - "xopen", "_PipedCompressionProgram", "__version__", + "xopen", ] +import bz2 import dataclasses import gzip -import stat -import sys import io -import os -import bz2 import lzma +import os import signal +import stat import subprocess +import sys import tempfile import threading import time -from typing import ( - Dict, - Optional, - Union, - IO, - overload, - BinaryIO, - Literal, - Tuple, -) from types import ModuleType +from typing import IO, BinaryIO, Literal, overload from ._version import version as __version__ @@ -43,12 +34,12 @@ XOPEN_DEFAULT_XZ_COMPRESSION = 6 XOPEN_DEFAULT_ZST_COMPRESSION = 3 -igzip: Optional[ModuleType] -isal_zlib: Optional[ModuleType] -igzip_threaded: Optional[ModuleType] -zlib_ng: Optional[ModuleType] -gzip_ng: Optional[ModuleType] -gzip_ng_threaded: Optional[ModuleType] +igzip: ModuleType | None +isal_zlib: ModuleType | None +igzip_threaded: ModuleType | None +zlib_ng: ModuleType | None +gzip_ng: ModuleType | None +gzip_ng_threaded: ModuleType | None try: from isal import igzip, igzip_threaded, isal_zlib @@ -93,23 +84,23 @@ _MAX_PIPE_SIZE = -1 -FilePath = Union[str, bytes, os.PathLike] -FileOrPath = Union[FilePath, IO] +FilePath = str | bytes | os.PathLike +FileOrPath = FilePath | IO @dataclasses.dataclass class _ProgramSettings: - program_args: Tuple[str, ...] - acceptable_compression_levels: Tuple[int, ...] = tuple(range(1, 10)) - threads_flag: Optional[str] = None + program_args: tuple[str, ...] + acceptable_compression_levels: tuple[int, ...] = tuple(range(1, 10)) + threads_flag: str | None = None # This exit code is not interpreted as an error when terminating the process - allowed_exit_code: Optional[int] = -signal.SIGTERM + allowed_exit_code: int | None = -signal.SIGTERM # If this message is printed on stderr on terminating the process, # it is not interpreted as an error - allowed_exit_message: Optional[bytes] = None + allowed_exit_message: bytes | None = None -_PROGRAM_SETTINGS: Dict[str, _ProgramSettings] = { +_PROGRAM_SETTINGS: dict[str, _ProgramSettings] = { "pbzip2": _ProgramSettings( ("pbzip2",), tuple(range(1, 10)), @@ -117,9 +108,9 @@ class _ProgramSettings: allowed_exit_code=None, allowed_exit_message=b"\n *Control-C or similar caught [sig=15], quitting...", ), - "xz": _ProgramSettings(("xz",), tuple(range(0, 10)), "-T"), + "xz": _ProgramSettings(("xz",), tuple(range(10)), "-T"), "zstd": _ProgramSettings(("zstd",), tuple(range(1, 20)), "-T"), - "pigz": _ProgramSettings(("pigz", "--no-name"), tuple(range(0, 10)) + (11,), "-p"), + "pigz": _ProgramSettings(("pigz", "--no-name"), tuple(range(10)) + (11,), "-p"), "gzip": _ProgramSettings(("gzip", "--no-name"), tuple(range(1, 10))), "lz4": _ProgramSettings(("lz4",), tuple(range(_LZ4_LEVEL_MIN, _LZ4_LEVEL_MAX + 1))), } @@ -141,7 +132,7 @@ def _available_cpu_count() -> int: status = f.read() m = re.search(r"(?m)^Cpus_allowed:\s*(.*)$", status) if m: - res = bin(int(m.group(1).replace(",", ""), 16)).count("1") + res = int(m.group(1).replace(",", ""), 16).bit_count() if res > 0: return res except OSError: @@ -159,8 +150,8 @@ def __init__( self, filename: FileOrPath, mode="rb", - compresslevel: Optional[int] = None, - threads: Optional[int] = None, + compresslevel: int | None = None, + threads: int | None = None, program_settings: _ProgramSettings = _ProgramSettings(("gzip", "--no-name")), ): """ @@ -195,7 +186,7 @@ def __init__( self.name: str = str(self._path) self._mode: str = mode self._stderr = tempfile.TemporaryFile("w+b") - self._threads_flag: Optional[str] = program_settings.threads_flag + self._threads_flag: str | None = program_settings.threads_flag if threads is None: if "r" in mode: @@ -300,17 +291,11 @@ def readline(self, *args) -> bytes: return self._file.readline(*args) def seekable(self) -> bool: - return self._file.seekable() - - def tell(self) -> int: - return self._file.tell() + return False - def peek(self, n: Optional[int] = None): + def peek(self, n: int | None = None): return self._file.peek(n) # type: ignore - def seek(self, offset, whence=0) -> int: - return self._file.seek(offset, whence) - def close(self) -> None: if self.closed: return @@ -330,7 +315,6 @@ def close(self) -> None: check_allowed_code_and_message = True self.process.wait() self._feeding = False - self._file.read() if self.in_thread: self.in_thread.join() self._file.close() @@ -438,8 +422,8 @@ def _open_stdin_or_out(mode: str) -> BinaryIO: def _open_bz2( filename: FileOrPath, mode: str, - compresslevel: Optional[int], - threads: Optional[int], + compresslevel: int | None, + threads: int | None, ): assert mode in ("rb", "ab", "wb") if compresslevel is None: @@ -467,8 +451,8 @@ def _open_bz2( def _open_xz( filename: FileOrPath, mode: str, - compresslevel: Optional[int], - threads: Optional[int], + compresslevel: int | None, + threads: int | None, ): assert mode in ("rb", "ab", "wb") if compresslevel is None: @@ -496,8 +480,8 @@ def _open_xz( def _open_zst( filename: FileOrPath, mode: str, - compresslevel: Optional[int], - threads: Optional[int], + compresslevel: int | None, + threads: int | None, ): assert mode in ("rb", "ab", "wb") assert compresslevel != 0 @@ -510,7 +494,7 @@ def _open_zst( if threads != 0: try: # zstd can compress using multiple cores - program_args: Tuple[str, ...] = ("zstd",) + program_args: tuple[str, ...] = ("zstd",) if "r" in mode: # Only use --long=31 for decompression. Using it for # compression overrides level settings for window size and @@ -549,8 +533,8 @@ def _open_zst( def _open_lz4( filename: FileOrPath, mode: str, - compresslevel: Optional[int], - threads: Optional[int], + compresslevel: int | None, + threads: int | None, ): assert mode in ("rb", "ab", "wb") if compresslevel is None: @@ -586,8 +570,8 @@ def _open_lz4( def _open_gz( filename: FileOrPath, mode: str, - compresslevel: Optional[int], - threads: Optional[int], + compresslevel: int | None, + threads: int | None, ): """ Open a gzip file. The ISA-L library is preferred when applicable because @@ -685,7 +669,7 @@ def _open_reproducible_gzip(filename, mode: str, compresslevel: int): return gzip_file -def _detect_format_from_content(filename: FileOrPath) -> Optional[str]: +def _detect_format_from_content(filename: FileOrPath) -> str | None: """ Attempts to detect file format from the content by reading the first 6 bytes. Returns None if no format could be detected. @@ -725,7 +709,7 @@ def _detect_format_from_content(filename: FileOrPath) -> Optional[str]: fileobj.close() -def _detect_format_from_extension(filename: Union[str, bytes]) -> Optional[str]: +def _detect_format_from_extension(filename: str | bytes) -> str | None: """ Attempt to detect file format from the filename extension. Return None if no format could be detected. @@ -742,7 +726,7 @@ def _detect_format_from_extension(filename: Union[str, bytes]) -> Optional[str]: def _file_or_path_to_binary_stream( file_or_path: FileOrPath, binary_mode: str -) -> Tuple[BinaryIO, bool]: +) -> tuple[BinaryIO, bool]: assert binary_mode in ("rb", "wb", "ab") if isinstance(file_or_path, (str, bytes)) or hasattr(file_or_path, "__fspath__"): return open(os.fspath(file_or_path), binary_mode), True # type: ignore @@ -784,42 +768,40 @@ def _file_is_a_socket_or_pipe(filepath): def xopen( filename: FileOrPath, mode: Literal["r", "w", "a", "rt", "wt", "at"] = ..., - compresslevel: Optional[int] = ..., - threads: Optional[int] = ..., + compresslevel: int | None = ..., + threads: int | None = ..., *, encoding: str = ..., - errors: Optional[str] = ..., - newline: Optional[str] = ..., - format: Optional[str] = ..., -) -> io.TextIOWrapper: - ... + errors: str | None = ..., + newline: str | None = ..., + format: str | None = ..., +) -> io.TextIOWrapper: ... @overload def xopen( filename: FileOrPath, mode: Literal["rb", "wb", "ab"], - compresslevel: Optional[int] = ..., - threads: Optional[int] = ..., + compresslevel: int | None = ..., + threads: int | None = ..., *, encoding: str = ..., errors: None = ..., newline: None = ..., - format: Optional[str] = ..., -) -> BinaryIO: - ... + format: str | None = ..., +) -> BinaryIO: ... -def xopen( # noqa: C901 +def xopen( filename: FileOrPath, mode: Literal["r", "w", "a", "rt", "rb", "wt", "wb", "at", "ab"] = "r", - compresslevel: Optional[int] = None, - threads: Optional[int] = None, + compresslevel: int | None = None, + threads: int | None = None, *, encoding: str = "utf-8", - errors: Optional[str] = None, - newline: Optional[str] = None, - format: Optional[str] = None, + errors: str | None = None, + newline: str | None = None, + format: str | None = None, ) -> IO: """ A replacement for the "open" function that can also read and write @@ -870,7 +852,7 @@ def xopen( # noqa: C901 if mode in ("r", "w", "a"): mode += "t" # type: ignore if mode not in ("rt", "rb", "wt", "wb", "at", "ab"): - raise ValueError("Mode '{}' not supported".format(mode)) + raise ValueError(f"Mode '{mode}' not supported") binary_mode = mode[0] + "b" filepath = _filepath_from_path_or_filelike(filename) diff --git a/tests/conftest.py b/tests/conftest.py index 2a63905..c6a7b6b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ import os import random import string + import pytest from xopen import xopen diff --git a/tests/test_piped.py b/tests/test_piped.py index 554676b..3e76fb1 100644 --- a/tests/test_piped.py +++ b/tests/test_piped.py @@ -1,21 +1,23 @@ """ Tests for the PipedCompression classes """ + import gzip import io import os import shutil import sys -import pytest -from pathlib import Path from itertools import cycle +from pathlib import Path + +import pytest from xopen import ( - xopen, - _PipedCompressionProgram, _MAX_PIPE_SIZE, _PROGRAM_SETTINGS, + _PipedCompressionProgram, _ProgramSettings, + xopen, ) extensions = ["", ".gz", ".bz2", ".xz", ".zst", ".lz4"] @@ -24,7 +26,7 @@ import fcntl if not hasattr(fcntl, "F_GETPIPE_SZ") and sys.platform == "linux": - setattr(fcntl, "F_GETPIPE_SZ", 1032) + fcntl.F_GETPIPE_SZ = 1032 except ImportError: fcntl = None @@ -312,19 +314,6 @@ def test_piped_compression_reader_peek_binary(reader): assert read_h.peek(1).startswith(b"T") -@pytest.mark.skipif( - sys.platform != "win32", reason="seeking only works on Windows for now" -) -def test_piped_compression_reader_seek_and_tell(reader): - program_settings, extension = reader - filegz = TEST_DIR / f"file.txt{extension}" - with _PipedCompressionProgram(filegz, "rb", program_settings=program_settings) as f: - original_position = f.tell() - assert f.read(4) == b"Test" - f.seek(original_position) - assert f.read(8) == b"Testing," - - @pytest.mark.parametrize("mode", ["r", "rb"]) def test_piped_compression_reader_peek_text(reader, mode): program_settings, extension = reader diff --git a/tests/test_xopen.py b/tests/test_xopen.py index a9fbb7a..9f68c0f 100644 --- a/tests/test_xopen.py +++ b/tests/test_xopen.py @@ -1,6 +1,7 @@ """ Tests for the xopen.xopen function """ + import bz2 import functools import gzip @@ -14,7 +15,6 @@ from contextlib import contextmanager from pathlib import Path - import pytest from xopen import _detect_format_from_content, xopen @@ -426,7 +426,7 @@ def test_write_gzip_no_threads_no_isal(tmp_path, xopen_without_igzip): import gzip with xopen_without_igzip(tmp_path / "out.gz", "wb", threads=0) as f: - if sys.version_info.major == 3 and sys.version_info.minor >= 12: + if sys.version_info >= (3, 12): assert isinstance(f, gzip.GzipFile), f else: assert isinstance(f.raw, gzip.GzipFile) @@ -619,9 +619,7 @@ def test_lz4_program_without_bindings(tmp_path, monkeypatch): @pytest.mark.parametrize("threads", (0, 1)) def test_xopen_zst_long_window_size(threads): - if threads == 0 and zstd is None: - return - elif threads == 1 and not shutil.which("zstd"): + if threads == 0 and zstd is None or threads == 1 and not shutil.which("zstd"): return # File created with: # cat /dev/zero | head -c 2147483648 > only_zeroes @@ -744,5 +742,6 @@ def test_xopen_dev_stdin_read(threads, ext): shell=True, stdout=subprocess.PIPE, encoding="ascii", + check=True, ) assert result.stdout == CONTENT + "\n" diff --git a/tox.toml b/tox.toml index 015ed52..7f7af1f 100644 --- a/tox.toml +++ b/tox.toml @@ -1,7 +1,7 @@ -env_list = ["black", "ruff", "mypy", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] +env_list = ["black", "ruff", "mypy", "3.11", "3.12", "3.13", "3.14", "3.14t"] [env_run_base] -default_base_python = "3.10" +default_base_python = "3.14" deps = [ "pytest", "pytest-timeout", @@ -32,12 +32,13 @@ commands = [ ] [env.black] -deps = ["black==22.3.0"] +base_python = "3.14" +deps = ["black==26.10"] skip_install = true commands = [["black", "--check", "src/", "tests/"]] [env.ruff] -deps = ["ruff"] +deps = ["ruff==0.17"] commands = [["ruff", "check", "src/", "tests/"]] skip_install = true