From efd7a16a9551b7ef3853c6928e4d7ffa2068eafb Mon Sep 17 00:00:00 2001 From: drknowhow Date: Thu, 10 Sep 2026 09:15:10 -0400 Subject: [PATCH] fix(watcher): file watcher honours the scanner's pruning; gitignored churn no longer rebuilds the index (2.129.1) services/watcher.py kept its own thirteen-entry skip list and tracked any .json/.md/.yaml under the project, ignoring the root .gitignore and linked worktrees that scanner.iter_files already prunes. On a project whose daemons rewrite four state files under a gitignored logs/ every few seconds, the change counter passed IndexStalenessAgent's rebuild threshold every minute: "Index auto-rebuilt" reached a count of 75,968, four agent threads burned ~5 CPU hours in 17 hours on an idle session, and each refresh burst left the MCP server's process heap with pinned 16 MB segments it never released (28 GB commit, <1 GB working set, one heap, ~1,770 segments). Three such servers exhausted the box's commit charge and the C3 Hub answered with MemoryError from werkzeug, which presented as "the hub fails to start". _ChangeHandler now filters with scanner.make_dir_pruner (shared SKIP_DIRS, root .gitignore directory entries and single-segment globs) and is_nested_checkout (cached per directory), before the sub-project excluder and the extension allowlist; editing the root .gitignore reloads the pruner. A change counts only when the file could be in the index. tests/test_watcher_pruning.py covers gitignored dirs, globs, the reload, worktrees, scanner-only skip names, the excluder and the rootless legacy constructor. Claude-Session: https://claude.ai/code/session_01Uw6AAAWX5CVG3QgThjC2UU --- CHANGELOG.md | 35 +++++++++- cli/c3.py | 2 +- pyproject.toml | 2 +- services/watcher.py | 86 +++++++++++++++++++++---- tests/test_watcher_pruning.py | 117 ++++++++++++++++++++++++++++++++++ 5 files changed, 228 insertions(+), 14 deletions(-) create mode 100644 tests/test_watcher_pruning.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f3a9d5..e1f6126 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,40 @@ All notable changes to Code Context Control (C3) are documented here. The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [2.129.0] - 2026-09-09 +## [2.129.1] - 2026-09-10 + +### Fixed — the file watcher counted gitignored churn as source changes, and the MCP server's memory never came back + +`services/watcher.py` kept its own thirteen-entry skip list and tracked +every `.json`/`.md`/`.yaml` under the project — ignoring the root +`.gitignore` and linked worktrees that `scanner.iter_files` already +prunes. On a project whose daemons rewrite four state files under a +gitignored `logs/` every few seconds, that pushed the change counter past +`IndexStalenessAgent`'s rebuild threshold (15) every minute: the +"Index auto-rebuilt" notification on that project reached a count of +**75,968**, and the FileMemory / BranchWatch / IndexStaleness / +EditLedgerEnricher threads burned ~5 CPU hours in 17 hours on an idle +session. + +The memory cost is the part that reached the user. Each refresh burst +leaves the process heap with more 16 MB segments that a few surviving +small blocks pin forever: one MCP server was at 28 GB of commit charge +with under 1 GB of working set (one heap, ~1,770 segments, 99.9 % zero), +and the box had three of them on the same project. Commit ran out, and +the C3 Hub — an innocent bystander — started answering requests with +`MemoryError` from inside werkzeug, which looked exactly like "the hub +fails to start". + +`_ChangeHandler` now filters with the scanner's own pruning: the shared +`SKIP_DIRS` superset, the root `.gitignore`'s directory entries (literal +and single-segment globs), and nested checkouts, before the sub-project +excluder and the extension allowlist. A change counts only when the file +could be *in* the index. Editing the root `.gitignore` reloads the +pruner. Nested-checkout probes are cached per directory, so the filter +costs one stat per directory, not per event. `tests/test_watcher_pruning.py` +covers gitignored dirs, globs, the reload, worktrees, scanner-only skip +names, and the rootless legacy constructor. + ### Fixed — a broken act runner no longer reports "0 parsed failures" (#173) diff --git a/cli/c3.py b/cli/c3.py index d44e3d1..615b191 100644 --- a/cli/c3.py +++ b/cli/c3.py @@ -91,7 +91,7 @@ # Config CONFIG_DIR = ".c3" CONFIG_FILE = ".c3/config.json" -__version__ = "2.129.0" +__version__ = "2.129.1" def _compress_file_cli(compressor, path, mode="smart", **kw): diff --git a/pyproject.toml b/pyproject.toml index 998177a..d7a265a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "code-context-control" -version = "2.129.0" +version = "2.129.1" description = "Local MCP code-intelligence for AI coding tools: surgical search/read/edit, agent-config version history, path-level access + masking guards, and a multi-project hub." readme = "README.md" requires-python = ">=3.10" diff --git a/services/watcher.py b/services/watcher.py index ced6640..7e1667a 100644 --- a/services/watcher.py +++ b/services/watcher.py @@ -7,6 +7,7 @@ - Accumulates changes for session logging - Triggers index rebuild when enough changes accumulate """ +import os import threading from datetime import datetime, timezone from pathlib import Path @@ -14,6 +15,8 @@ from watchdog.events import FileSystemEventHandler from watchdog.observers import Observer +from services.scanner import SKIP_DIRS, is_nested_checkout, make_dir_pruner + # Extensions to watch CODE_EXTENSIONS = { '.py', '.js', '.ts', '.tsx', '.jsx', '.r', '.R', @@ -21,32 +24,93 @@ '.sh', '.sql', '.go', '.rs', '.java', '.cpp', '.c', '.h', } -# Directories to skip -SKIP_DIRS = { - 'node_modules', '.git', '__pycache__', '.c3', 'venv', - 'env', '.venv', 'dist', 'build', '.next', '.cache', - 'coverage', '.pytest_cache', -} +# SKIP_DIRS is re-exported from services.scanner: the watcher and the index +# walker must agree on what is not part of the project (see _ChangeHandler). +__all__ = ['CODE_EXTENSIONS', 'SKIP_DIRS', 'CodeWatcher'] class _ChangeHandler(FileSystemEventHandler): - """Collects file change events.""" + """Collects file change events. + + The filter is the index scanner's, not a private one. A change only + counts when the file could be IN the index: the same directory pruning + as ``scanner.iter_files`` (SKIP_DIRS, the root .gitignore's directory + entries, nested checkouts such as linked worktrees), then the + sub-project excluder, then the extension allowlist. - def __init__(self, excluder=None): + Before 2.129.1 the watcher kept its own shorter skip list and ignored + .gitignore and worktrees, so state files a daemon rewrites every few + seconds under a gitignored ``logs/`` counted as source changes. On one + project that tripped IndexStalenessAgent's rebuild threshold (15) every + single minute — the "Index auto-rebuilt" notification reached a count of + 75,968 — and each refresh burst left the MCP server's process heap with + more pinned 16 MB segments it never gave back: 28 GB of commit charge + per server with under 1 GB touched, three servers on the project, and + the C3 Hub dying of MemoryError when the box ran out of commit. + """ + + def __init__(self, excluder=None, root=None): super().__init__() self._lock = threading.Lock() self._changes = [] self._excluder = excluder # sub-project folders tracked by their own .c3 + self._root = Path(root) if root else None + self._pruned = make_dir_pruner(root) if root else (lambda d: d in SKIP_DIRS) + # dir path -> is another checkout; one stat per directory, not per event + self._checkout_cache: dict = {} + + def _reload_pruner(self): + """Re-read the root .gitignore (called when it changes).""" + if self._root is not None: + self._pruned = make_dir_pruner(self._root) + + def _rel_parts(self, p: Path) -> tuple: + if self._root is None: + return p.parts + try: + return p.relative_to(self._root).parts + except ValueError: + try: + return p.resolve().relative_to(self._root).parts + except (ValueError, OSError): + return p.parts + + def _under_nested_checkout(self, rel_dirs: tuple) -> bool: + if self._root is None: + return False + cur = str(self._root) + for d in rel_dirs: + cur = os.path.join(cur, d) + hit = self._checkout_cache.get(cur) + if hit is None: + if len(self._checkout_cache) > 4096: + self._checkout_cache.clear() + hit = self._checkout_cache[cur] = is_nested_checkout(cur) + if hit: + return True + return False def _should_track(self, path: str) -> bool: p = Path(path) - if any(skip in p.parts for skip in SKIP_DIRS): + if p.suffix.lower() not in CODE_EXTENSIONS: + return False + parts = self._rel_parts(p) + rel_dirs = parts[:-1] + if any(self._pruned(d) for d in rel_dirs): + return False + if self._under_nested_checkout(rel_dirs): return False if self._excluder is not None and self._excluder(path): return False - return p.suffix.lower() in CODE_EXTENSIONS + return True def _record(self, event_type: str, path: str): + if self._root is not None and Path(path).name == '.gitignore': + try: + if Path(path).parent.samefile(self._root): + self._reload_pruner() + except OSError: + pass if not self._should_track(path): return with self._lock: @@ -94,7 +158,7 @@ def __init__(self, project_path: str): excluder = make_excluder(self.project_path) except Exception: excluder = None - self._handler = _ChangeHandler(excluder) + self._handler = _ChangeHandler(excluder, root=self.project_path) self._observer = Observer() self._observer.daemon = True self._file_memory = None diff --git a/tests/test_watcher_pruning.py b/tests/test_watcher_pruning.py new file mode 100644 index 0000000..f69d73d --- /dev/null +++ b/tests/test_watcher_pruning.py @@ -0,0 +1,117 @@ +""" +The file watcher filters events with the index scanner's pruning. + +A change counts toward IndexStalenessAgent's rebuild threshold only when the +file could be in the index. Before 2.129.1 the watcher had its own shorter +skip list and ignored the root .gitignore and nested checkouts, so a daemon +rewriting state JSON under a gitignored ``logs/`` every few seconds tripped +a full refresh every minute (75,968 "Index auto-rebuilt" notifications on +one project) and each refresh burst left the MCP server's process heap with +pinned 16 MB segments it never released — 28 GB of commit per server. +""" +import tempfile +import unittest +from pathlib import Path + +from services.watcher import SKIP_DIRS, _ChangeHandler + + +class _Tree(unittest.TestCase): + def setUp(self): + self._td = tempfile.TemporaryDirectory() + self.root = Path(self._td.name).resolve() + + def tearDown(self): + self._td.cleanup() + + def touch(self, rel, content="x"): + p = self.root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content, encoding="utf-8") + return p + + def handler(self, excluder=None): + return _ChangeHandler(excluder, root=str(self.root)) + + @staticmethod + def recorded(h): + return [Path(c["path"]).name for c in h.get_and_clear()] + + +class TestGitignoredDirectoriesDoNotCount(_Tree): + def test_a_state_file_under_a_gitignored_dir_is_not_a_change(self): + self.touch(".gitignore", "logs/\n.daemon-worktree/\n") + h = self.handler() + h._record("modified", str(self.touch("logs/.health.json", "{}"))) + h._record("modified", str(self.touch( + ".daemon-worktree/logs/restart_failure_state.json", "{}"))) + h._record("modified", str(self.touch("src/real.py"))) + self.assertEqual(self.recorded(h), ["real.py"]) + + def test_a_gitignore_glob_entry_prunes_too(self): + self.touch(".gitignore", "*.egg-info/\n") + h = self.handler() + h._record("created", str(self.touch("c3.egg-info/PKG-INFO.md"))) + h._record("created", str(self.touch("pkg/mod.py"))) + self.assertEqual(self.recorded(h), ["mod.py"]) + + def test_editing_the_root_gitignore_reloads_the_pruner(self): + h = self.handler() + h._record("modified", str(self.touch("logs/state.json", "{}"))) + self.assertEqual(self.recorded(h), ["state.json"]) + gi = self.touch(".gitignore", "logs/\n") + h._record("modified", str(gi)) # the watcher sees the edit + h._record("modified", str(self.touch("logs/state.json", "{}"))) + self.assertEqual(self.recorded(h), []) + + +class TestNestedCheckoutsDoNotCount(_Tree): + def test_a_linked_worktree_copy_is_not_a_change(self): + wt = self.root / "wt" + wt.mkdir() + (wt / ".git").write_text("gitdir: /elsewhere/.git/worktrees/wt\n", + encoding="utf-8") + h = self.handler() + h._record("modified", str(self.touch("wt/src/copy.py"))) + h._record("modified", str(self.touch("src/real.py"))) + self.assertEqual(self.recorded(h), ["real.py"]) + + def test_the_project_root_itself_is_never_treated_as_nested(self): + (self.root / ".git").mkdir() + h = self.handler() + h._record("modified", str(self.touch("src/real.py"))) + self.assertEqual(self.recorded(h), ["real.py"]) + + +class TestSkipListIsTheScannersSuperset(_Tree): + def test_scanner_only_dirs_are_skipped_by_the_watcher(self): + # 'target' and '.mypy_cache' were never in the watcher's own list. + self.assertIn("target", SKIP_DIRS) + h = self.handler() + h._record("modified", str(self.touch("target/debug/build.rs"))) + h._record("modified", str(self.touch(".mypy_cache/3.12/x.json", "{}"))) + h._record("modified", str(self.touch("node_modules/a/index.js"))) + h._record("modified", str(self.touch("src/lib.rs"))) + self.assertEqual(self.recorded(h), ["lib.rs"]) + + def test_extension_allowlist_still_applies(self): + h = self.handler() + h._record("modified", str(self.touch("src/blob.bin"))) + h._record("modified", str(self.touch("src/ok.py"))) + self.assertEqual(self.recorded(h), ["ok.py"]) + + def test_sub_project_excluder_still_applies(self): + h = self.handler(excluder=lambda p: "sub" in Path(p).parts) + h._record("modified", str(self.touch("sub/inner.py"))) + h._record("modified", str(self.touch("src/outer.py"))) + self.assertEqual(self.recorded(h), ["outer.py"]) + + def test_a_rootless_handler_keeps_the_legacy_behaviour(self): + h = _ChangeHandler() + h._record("modified", str(self.touch("node_modules/x/y.js"))) + h._record("modified", str(self.touch("src/z.js"))) + self.assertEqual(self.recorded(h), ["z.js"]) + + +if __name__ == "__main__": + unittest.main()