fix(watcher): file watcher honours the scanner's pruning; gitignored churn no longer rebuilds the index (2.129.1)

CHANGELOG.md

@@ -4,7 +4,40 @@ All notable changes to Code Context Control (C3) are documented here.
 The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [2.129.0] - 2026-09-09
+## [2.129.1] - 2026-09-10
+
+### Fixed — the file watcher counted gitignored churn as source changes, and the MCP server's memory never came back
+
+`services/watcher.py` kept its own thirteen-entry skip list and tracked
+every `.json`/`.md`/`.yaml` under the project — ignoring the root
+`.gitignore` and linked worktrees that `scanner.iter_files` already
+prunes. On a project whose daemons rewrite four state files under a
+gitignored `logs/` every few seconds, that pushed the change counter past
+`IndexStalenessAgent`'s rebuild threshold (15) every minute: the
+"Index auto-rebuilt" notification on that project reached a count of
+**75,968**, and the FileMemory / BranchWatch / IndexStaleness /
+EditLedgerEnricher threads burned ~5 CPU hours in 17 hours on an idle
+session.
+
+The memory cost is the part that reached the user. Each refresh burst
+leaves the process heap with more 16 MB segments that a few surviving
+small blocks pin forever: one MCP server was at 28 GB of commit charge
+with under 1 GB of working set (one heap, ~1,770 segments, 99.9 % zero),
+and the box had three of them on the same project. Commit ran out, and
+the C3 Hub — an innocent bystander — started answering requests with
+`MemoryError` from inside werkzeug, which looked exactly like "the hub
+fails to start".
+
+`_ChangeHandler` now filters with the scanner's own pruning: the shared
+`SKIP_DIRS` superset, the root `.gitignore`'s directory entries (literal
+and single-segment globs), and nested checkouts, before the sub-project
+excluder and the extension allowlist. A change counts only when the file
+could be *in* the index. Editing the root `.gitignore` reloads the
+pruner. Nested-checkout probes are cached per directory, so the filter
+costs one stat per directory, not per event. `tests/test_watcher_pruning.py`
+covers gitignored dirs, globs, the reload, worktrees, scanner-only skip
+names, and the rootless legacy constructor.
+
 
 ### Fixed — a broken act runner no longer reports "0 parsed failures" (#173)
 

cli/c3.py

@@ -91,7 +91,7 @@
 # Config
 CONFIG_DIR = ".c3"
 CONFIG_FILE = ".c3/config.json"
-__version__ = "2.129.0"
+__version__ = "2.129.1"
 
 
 def _compress_file_cli(compressor, path, mode="smart", **kw):

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "code-context-control"
-version = "2.129.0"
+version = "2.129.1"
 description = "Local MCP code-intelligence for AI coding tools: surgical search/read/edit, agent-config version history, path-level access + masking guards, and a multi-project hub."
 readme = "README.md"
 requires-python = ">=3.10"

services/watcher.py

@@ -7,46 +7,110 @@
 - Accumulates changes for session logging
 - Triggers index rebuild when enough changes accumulate
 """
+import os
 import threading
 from datetime import datetime, timezone
 from pathlib import Path
 
 from watchdog.events import FileSystemEventHandler
 from watchdog.observers import Observer
 
+from services.scanner import SKIP_DIRS, is_nested_checkout, make_dir_pruner
+
 # Extensions to watch
 CODE_EXTENSIONS = {
     '.py', '.js', '.ts', '.tsx', '.jsx', '.r', '.R',
     '.css', '.html', '.json', '.yaml', '.yml', '.md',
     '.sh', '.sql', '.go', '.rs', '.java', '.cpp', '.c', '.h',
 }
 
-# Directories to skip
-SKIP_DIRS = {
-    'node_modules', '.git', '__pycache__', '.c3', 'venv',
-    'env', '.venv', 'dist', 'build', '.next', '.cache',
-    'coverage', '.pytest_cache',
-}
+# SKIP_DIRS is re-exported from services.scanner: the watcher and the index
+# walker must agree on what is not part of the project (see _ChangeHandler).
+__all__ = ['CODE_EXTENSIONS', 'SKIP_DIRS', 'CodeWatcher']
 
 
 class _ChangeHandler(FileSystemEventHandler):
-    """Collects file change events."""
+    """Collects file change events.
+
+    The filter is the index scanner's, not a private one. A change only
+    counts when the file could be IN the index: the same directory pruning
+    as ``scanner.iter_files`` (SKIP_DIRS, the root .gitignore's directory
+    entries, nested checkouts such as linked worktrees), then the
+    sub-project excluder, then the extension allowlist.
 
-    def __init__(self, excluder=None):
+    Before 2.129.1 the watcher kept its own shorter skip list and ignored
+    .gitignore and worktrees, so state files a daemon rewrites every few
+    seconds under a gitignored ``logs/`` counted as source changes. On one
+    project that tripped IndexStalenessAgent's rebuild threshold (15) every
+    single minute — the "Index auto-rebuilt" notification reached a count of
+    75,968 — and each refresh burst left the MCP server's process heap with
+    more pinned 16 MB segments it never gave back: 28 GB of commit charge
+    per server with under 1 GB touched, three servers on the project, and
+    the C3 Hub dying of MemoryError when the box ran out of commit.
+    """
+
+    def __init__(self, excluder=None, root=None):
         super().__init__()
         self._lock = threading.Lock()
         self._changes = []
         self._excluder = excluder  # sub-project folders tracked by their own .c3
+        self._root = Path(root) if root else None
+        self._pruned = make_dir_pruner(root) if root else (lambda d: d in SKIP_DIRS)
+        # dir path -> is another checkout; one stat per directory, not per event
+        self._checkout_cache: dict = {}
+
+    def _reload_pruner(self):
+        """Re-read the root .gitignore (called when it changes)."""
+        if self._root is not None:
+            self._pruned = make_dir_pruner(self._root)
+
+    def _rel_parts(self, p: Path) -> tuple:
+        if self._root is None:
+            return p.parts
+        try:
+            return p.relative_to(self._root).parts
+        except ValueError:
+            try:
+                return p.resolve().relative_to(self._root).parts
+            except (ValueError, OSError):
+                return p.parts
+
+    def _under_nested_checkout(self, rel_dirs: tuple) -> bool:
+        if self._root is None:
+            return False
+        cur = str(self._root)
+        for d in rel_dirs:
+            cur = os.path.join(cur, d)
+            hit = self._checkout_cache.get(cur)
+            if hit is None:
+                if len(self._checkout_cache) > 4096:
+                    self._checkout_cache.clear()
+                hit = self._checkout_cache[cur] = is_nested_checkout(cur)
+            if hit:
+                return True
+        return False
 
     def _should_track(self, path: str) -> bool:
         p = Path(path)
-        if any(skip in p.parts for skip in SKIP_DIRS):
+        if p.suffix.lower() not in CODE_EXTENSIONS:
+            return False
+        parts = self._rel_parts(p)
+        rel_dirs = parts[:-1]
+        if any(self._pruned(d) for d in rel_dirs):
+            return False
+        if self._under_nested_checkout(rel_dirs):
             return False
         if self._excluder is not None and self._excluder(path):
             return False
-        return p.suffix.lower() in CODE_EXTENSIONS
+        return True
 
     def _record(self, event_type: str, path: str):
+        if self._root is not None and Path(path).name == '.gitignore':
+            try:
+                if Path(path).parent.samefile(self._root):
+                    self._reload_pruner()
+            except OSError:
+                pass
         if not self._should_track(path):
             return
         with self._lock:
@@ -94,7 +158,7 @@ def __init__(self, project_path: str):
             excluder = make_excluder(self.project_path)
         except Exception:
             excluder = None
-        self._handler = _ChangeHandler(excluder)
+        self._handler = _ChangeHandler(excluder, root=self.project_path)
         self._observer = Observer()
         self._observer.daemon = True
         self._file_memory = None

tests/test_watcher_pruning.py

@@ -0,0 +1,117 @@
+"""
+The file watcher filters events with the index scanner's pruning.
+
+A change counts toward IndexStalenessAgent's rebuild threshold only when the
+file could be in the index. Before 2.129.1 the watcher had its own shorter
+skip list and ignored the root .gitignore and nested checkouts, so a daemon
+rewriting state JSON under a gitignored ``logs/`` every few seconds tripped
+a full refresh every minute (75,968 "Index auto-rebuilt" notifications on
+one project) and each refresh burst left the MCP server's process heap with
+pinned 16 MB segments it never released — 28 GB of commit per server.
+"""
+import tempfile
+import unittest
+from pathlib import Path
+
+from services.watcher import SKIP_DIRS, _ChangeHandler
+
+
+class _Tree(unittest.TestCase):
+    def setUp(self):
+        self._td = tempfile.TemporaryDirectory()
+        self.root = Path(self._td.name).resolve()
+
+    def tearDown(self):
+        self._td.cleanup()
+
+    def touch(self, rel, content="x"):
+        p = self.root / rel
+        p.parent.mkdir(parents=True, exist_ok=True)
+        p.write_text(content, encoding="utf-8")
+        return p
+
+    def handler(self, excluder=None):
+        return _ChangeHandler(excluder, root=str(self.root))
+
+    @staticmethod
+    def recorded(h):
+        return [Path(c["path"]).name for c in h.get_and_clear()]
+
+
+class TestGitignoredDirectoriesDoNotCount(_Tree):
+    def test_a_state_file_under_a_gitignored_dir_is_not_a_change(self):
+        self.touch(".gitignore", "logs/\n.daemon-worktree/\n")
+        h = self.handler()
+        h._record("modified", str(self.touch("logs/.health.json", "{}")))
+        h._record("modified", str(self.touch(
+            ".daemon-worktree/logs/restart_failure_state.json", "{}")))
+        h._record("modified", str(self.touch("src/real.py")))
+        self.assertEqual(self.recorded(h), ["real.py"])
+
+    def test_a_gitignore_glob_entry_prunes_too(self):
+        self.touch(".gitignore", "*.egg-info/\n")
+        h = self.handler()
+        h._record("created", str(self.touch("c3.egg-info/PKG-INFO.md")))
+        h._record("created", str(self.touch("pkg/mod.py")))
+        self.assertEqual(self.recorded(h), ["mod.py"])
+
+    def test_editing_the_root_gitignore_reloads_the_pruner(self):
+        h = self.handler()
+        h._record("modified", str(self.touch("logs/state.json", "{}")))
+        self.assertEqual(self.recorded(h), ["state.json"])
+        gi = self.touch(".gitignore", "logs/\n")
+        h._record("modified", str(gi))  # the watcher sees the edit
+        h._record("modified", str(self.touch("logs/state.json", "{}")))
+        self.assertEqual(self.recorded(h), [])
+
+
+class TestNestedCheckoutsDoNotCount(_Tree):
+    def test_a_linked_worktree_copy_is_not_a_change(self):
+        wt = self.root / "wt"
+        wt.mkdir()
+        (wt / ".git").write_text("gitdir: /elsewhere/.git/worktrees/wt\n",
+                                 encoding="utf-8")
+        h = self.handler()
+        h._record("modified", str(self.touch("wt/src/copy.py")))
+        h._record("modified", str(self.touch("src/real.py")))
+        self.assertEqual(self.recorded(h), ["real.py"])
+
+    def test_the_project_root_itself_is_never_treated_as_nested(self):
+        (self.root / ".git").mkdir()
+        h = self.handler()
+        h._record("modified", str(self.touch("src/real.py")))
+        self.assertEqual(self.recorded(h), ["real.py"])
+
+
+class TestSkipListIsTheScannersSuperset(_Tree):
+    def test_scanner_only_dirs_are_skipped_by_the_watcher(self):
+        # 'target' and '.mypy_cache' were never in the watcher's own list.
+        self.assertIn("target", SKIP_DIRS)
+        h = self.handler()
+        h._record("modified", str(self.touch("target/debug/build.rs")))
+        h._record("modified", str(self.touch(".mypy_cache/3.12/x.json", "{}")))
+        h._record("modified", str(self.touch("node_modules/a/index.js")))
+        h._record("modified", str(self.touch("src/lib.rs")))
+        self.assertEqual(self.recorded(h), ["lib.rs"])
+
+    def test_extension_allowlist_still_applies(self):
+        h = self.handler()
+        h._record("modified", str(self.touch("src/blob.bin")))
+        h._record("modified", str(self.touch("src/ok.py")))
+        self.assertEqual(self.recorded(h), ["ok.py"])
+
+    def test_sub_project_excluder_still_applies(self):
+        h = self.handler(excluder=lambda p: "sub" in Path(p).parts)
+        h._record("modified", str(self.touch("sub/inner.py")))
+        h._record("modified", str(self.touch("src/outer.py")))
+        self.assertEqual(self.recorded(h), ["outer.py"])
+
+    def test_a_rootless_handler_keeps_the_legacy_behaviour(self):
+        h = _ChangeHandler()
+        h._record("modified", str(self.touch("node_modules/x/y.js")))
+        h._record("modified", str(self.touch("src/z.js")))
+        self.assertEqual(self.recorded(h), ["z.js"])
+
+
+if __name__ == "__main__":
+    unittest.main()