diff --git a/src/semble/cli.py b/src/semble/cli.py index 5061e964..4d3657a5 100644 --- a/src/semble/cli.py +++ b/src/semble/cli.py @@ -243,24 +243,8 @@ def _run_clear(clear_type: _CLEAR_CHOICE) -> None: _clear_orphans(cache_folder) -class _CliLogHandler(logging.StreamHandler): - """stderr handler owned by the CLI; setup is idempotent on this type, not on foreign handlers.""" - - -def _configure_cli_logging() -> None: - """Surface semble warnings (e.g. skipped oversized files) on stderr without touching the root logger.""" - package_logger = logging.getLogger("semble") - if any(isinstance(handler, _CliLogHandler) for handler in package_logger.handlers): - return - handler = _CliLogHandler(sys.stderr) - handler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) - package_logger.addHandler(handler) - if package_logger.level == logging.NOTSET: - package_logger.setLevel(logging.WARNING) - - def _cli_main() -> None: - _configure_cli_logging() + logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s") parser = argparse.ArgumentParser(prog="semble") parser.add_argument("-V", "--version", action="version", version=__version__) sub = parser.add_subparsers(dest="command") diff --git a/src/semble/index/create.py b/src/semble/index/create.py index 43d67c55..e2890df5 100644 --- a/src/semble/index/create.py +++ b/src/semble/index/create.py @@ -14,11 +14,11 @@ from semble.index.dense import SelectableBasicBackend, embed_chunks from semble.index.file_walker import walk_files from semble.index.files import ( + MAX_FILE_BYTES, FileStatus, detect_language, get_extensions, get_file_status, - get_max_file_bytes, read_file_text, ) from semble.index.sparse import enrich_for_bm25 @@ -36,7 +36,7 @@ def _warn_skipped_large(skipped_large: list[str]) -> None: "Skipped %d file(s) exceeding the maximum file size of %d bytes " "(raise SEMBLE_MAX_FILE_BYTES to include them): %s%s", len(skipped_large), - get_max_file_bytes(), + MAX_FILE_BYTES, ", ".join(skipped_large[:5]), " ..." if len(skipped_large) > 5 else "", ) diff --git a/src/semble/index/files.py b/src/semble/index/files.py index e3f4f941..0c7740f9 100644 --- a/src/semble/index/files.py +++ b/src/semble/index/files.py @@ -1,4 +1,3 @@ -import logging import os from collections import defaultdict from collections.abc import Sequence @@ -7,10 +6,8 @@ from semble.types import ContentType -_DEFAULT_MAX_FILE_BYTES = 1_000_000 # Default 1 MB max file size to read and index +MAX_FILE_BYTES = int(os.environ.get("SEMBLE_MAX_FILE_BYTES", 1_000_000)) # Max file size to read and index _EMPTY_FILE_BYTES = 128 - -logger = logging.getLogger(__name__) _EXTENSION_TO_LANGUAGE = { ".4th": "forth", ".ada": "ada", @@ -492,25 +489,6 @@ def read_file_text(file_path: Path) -> str: return file_path.read_text(encoding="utf-8", errors="replace") -def get_max_file_bytes() -> int: - """Resolve the maximum file size to index from SEMBLE_MAX_FILE_BYTES, falling back to the default. - - Malformed or nonpositive values warn and fall back to the default rather than crash indexing. - """ - raw = os.environ.get("SEMBLE_MAX_FILE_BYTES") - if raw is None: - return _DEFAULT_MAX_FILE_BYTES - try: - value = int(raw) - except ValueError: - logger.warning("Invalid SEMBLE_MAX_FILE_BYTES %r, using the default of %d bytes", raw, _DEFAULT_MAX_FILE_BYTES) - return _DEFAULT_MAX_FILE_BYTES - if value <= 0: - logger.warning("SEMBLE_MAX_FILE_BYTES must be positive, using the default of %d bytes", _DEFAULT_MAX_FILE_BYTES) - return _DEFAULT_MAX_FILE_BYTES - return value - - def get_file_status(file_path: Path, write_time: float | None) -> FileStatus: """Checks if a file should be indexed based on its size and modification time.""" stat = file_path.stat() @@ -518,7 +496,7 @@ def get_file_status(file_path: Path, write_time: float | None) -> FileStatus: # Index invalid, file invalid return FileStatus.NEWER size = stat.st_size - if size > get_max_file_bytes(): + if size > MAX_FILE_BYTES: # index valid, file invalid return FileStatus.TOO_LARGE if size < _EMPTY_FILE_BYTES and not read_file_text(file_path).strip(): diff --git a/tests/index/test_index.py b/tests/index/test_index.py index 1832d0a8..12926e98 100644 --- a/tests/index/test_index.py +++ b/tests/index/test_index.py @@ -9,7 +9,7 @@ from semble import SembleIndex from semble.index.create import create_index_from_path -from semble.index.files import _DEFAULT_MAX_FILE_BYTES, FileStatus, get_file_status, get_max_file_bytes +from semble.index.files import FileStatus, get_file_status from semble.types import ContentType from tests.conftest import make_chunk @@ -70,45 +70,23 @@ def test_index_empty_returns_zero_chunks(mock_model: StaticModel, tmp_path: Path create_index_from_path(tmp_path, mock_model) -def test_max_file_bytes_resolution(monkeypatch: pytest.MonkeyPatch) -> None: - """The limit resolves from SEMBLE_MAX_FILE_BYTES, falling back to the 1 MB default.""" - monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False) - assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES - monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", str(_DEFAULT_MAX_FILE_BYTES + 5)) - assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES + 5 - - -def test_max_file_bytes_invalid_values_fall_back(monkeypatch: pytest.MonkeyPatch) -> None: - """Malformed or nonpositive SEMBLE_MAX_FILE_BYTES values fall back to the default.""" - monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False) - assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES - for bad in ("not-a-number", "0", "-5"): - monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", bad) - assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES - - -def test_oversized_file_is_skipped_with_warning( - mock_model: StaticModel, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +@pytest.mark.parametrize(("max_file_bytes", "indexed"), [(1_000, False), (10_000, True)]) +def test_max_file_bytes( + mock_model: StaticModel, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + max_file_bytes: int, + indexed: bool, ) -> None: - """Files exceeding the limit are skipped during indexing, with a warning naming them.""" - monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False) - (tmp_path / "big.py").write_bytes(b"x" * (_DEFAULT_MAX_FILE_BYTES + 1)) + """Files over MAX_FILE_BYTES are skipped with a warning naming them; files under it are indexed.""" + monkeypatch.setattr("semble.index.files.MAX_FILE_BYTES", max_file_bytes) + (tmp_path / "small.py").write_text("def f():\n return 1\n") + (tmp_path / "big.py").write_bytes(b"x = 1\n" + b"#" * 5_000) with caplog.at_level(logging.WARNING, logger="semble.index.create"): - with pytest.raises(ValueError): # no indexable content remains - create_index_from_path(tmp_path, mock_model) - assert "big.py" in caplog.text - assert str(_DEFAULT_MAX_FILE_BYTES) in caplog.text - assert "SEMBLE_MAX_FILE_BYTES" in caplog.text - - -def test_oversized_file_indexed_when_limit_raised( - mock_model: StaticModel, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Raising SEMBLE_MAX_FILE_BYTES lets oversized files into the index.""" - monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", str(_DEFAULT_MAX_FILE_BYTES + 1024)) - (tmp_path / "big.py").write_bytes(b"x = 1\n" + b"#" * _DEFAULT_MAX_FILE_BYTES) - _, _, chunks, _ = create_index_from_path(tmp_path, mock_model) - assert any(chunk.file_path.endswith("big.py") for chunk in chunks) + _, _, chunks, _ = create_index_from_path(tmp_path, mock_model) + assert any(chunk.file_path.endswith("big.py") for chunk in chunks) is indexed + assert ("big.py" in caplog.text) is not indexed def test_tiny_invalid_utf8_file_status_does_not_crash(tmp_path: Path) -> None: