perf(file_memory): stop re-reading every file map on each poll (2.158.1)

CHANGELOG.md

@@ -4,6 +4,21 @@ All notable changes to Code Context Control (C3) are documented here.
 The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.158.1] - 2026-10-21
+
+### Fixed — FileMemory and BranchWatch stop re-reading every file map
+
+`list_tracked()` JSON-loaded every record in `.c3/file_memory` to read one
+field, and two background agents called it on a timer. On the Yep project
+that was 64,126 records (292 MB) per call, 92% of them maps of files inside
+nested checkouts, and the project's 14 MCP servers used about 4.5 CPU cores
+between them. `list_tracked()` now lists the store and reads only records it
+has not seen before; a record's filename is the md5 of its path, so a name
+already read cannot change meaning. BranchWatch looks each dirty file up with
+the new `is_tracked()` instead of listing the store every 30 s, and the
+FileMemory stale scan skips records inside nested checkouts, which the
+watcher never reports changes for.
+
 ## [2.158.0] - 2026-10-20
 
 ### Added — `format="json"` for c3_jira and c3_bitbucket

cli/c3.py

@@ -91,7 +91,7 @@
 # Config
 CONFIG_DIR = ".c3"
 CONFIG_FILE = ".c3/config.json"
-__version__ = "2.158.0"
+__version__ = "2.158.1"
 
 # The PreToolUse matcher for native subagent calls (installer and hub migration).
 AGENT_MATCHER = "Agent|Task"

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "code-context-control"
-version = "2.158.0"
+version = "2.158.1"
 description = "Local MCP code-intelligence for AI coding tools: surgical search/read/edit, agent-config version history, path-level access + masking guards, and a multi-project hub."
 readme = "README.md"
 requires-python = ">=3.10"

services/agents.py

@@ -903,8 +903,12 @@ def check(self):
         change_count = self.watcher._handler.change_count
         if change_count != self._last_change_count:
             self._last_change_count = change_count
-            # Check tracked files for staleness
+            # Check tracked files for staleness. The watcher never reports a
+            # change inside a nested checkout, so records there are skipped.
+            handler = self.watcher._handler
             for rel_path in self.file_memory.list_tracked():
+                if handler.under_nested_checkout(rel_path):
+                    continue
                 if self.file_memory.needs_update(rel_path):
                     files_to_process.append(rel_path)
 
@@ -1007,11 +1011,10 @@ def _queue_changed(self, paths) -> int:
         """
         if not paths:
             return 0
-        tracked = {t.replace("\\", "/"): t for t in self.file_memory.list_tracked()}
         queued = 0
         for p in paths:
-            canonical = tracked.get(p.replace("\\", "/"))
-            if canonical is None:
+            canonical = p.replace("\\", "/")
+            if not self.file_memory.is_tracked(canonical):
                 continue
             try:
                 if not self.file_memory.needs_update(canonical):

services/file_memory.py

@@ -8,6 +8,7 @@
 """
 import hashlib
 import json
+import os
 import re
 import threading
 import time
@@ -56,6 +57,10 @@ def __init__(self, project_path: str):
         # update()'s add_or_update on the shared TextIndex. Reentrant so
         # search() may hold it across _ensure_search_index() + the query.
         self._search_lock = threading.RLock()
+        # list_tracked(): record filename -> stored path. A filename is the
+        # md5 of its path, so a name already read never maps to another path.
+        self._tracked_paths: dict[str, str] = {}
+        self._tracked_lock = threading.Lock()
         # One-time cleanup of pre-2.121.0 generated summaries (marker-gated).
         try:
             self.purge_summaries()
@@ -101,6 +106,10 @@ def purge_summaries(self) -> int:
             pass
         return rewritten
 
+    def is_tracked(self, rel_path: str) -> bool:
+        """True if a record exists for ``rel_path``, without reading it."""
+        return self._store_path(str(rel_path).replace("\\", "/")).exists()
+
     def get(self, rel_path: str) -> Optional[dict]:
         """Load a file's memory record, or None if not tracked."""
         rel_path = str(rel_path).replace("\\", "/")
@@ -391,18 +400,29 @@ def search_sections(sections, parent_name=""):
         return unique_ranges
 
     def list_tracked(self) -> list:
-        """Return relative paths of all tracked files."""
-        tracked = []
-        for f in self.store_dir.glob("*.json"):
-            if f.name.startswith("_"):
-                continue
-            try:
-                with open(f, encoding="utf-8") as fh:
-                    data = json.load(fh)
-                tracked.append(data.get("path", ""))
-            except Exception:
-                continue
-        return [p for p in tracked if p]
+        """Return relative paths of all tracked files.
+
+        Lists the store directory on every call but reads only records whose
+        filename it has not seen before; removed records drop out.
+        """
+        try:
+            names = {e.name for e in os.scandir(self.store_dir)
+                     if e.name.endswith(".json") and not e.name.startswith("_")}
+        except OSError:
+            return []
+        with self._tracked_lock:
+            known = self._tracked_paths
+            for gone in known.keys() - names:
+                del known[gone]
+            for name in names - known.keys():
+                try:
+                    with open(self.store_dir / name, encoding="utf-8") as fh:
+                        path = json.load(fh).get("path", "")
+                except (OSError, ValueError, AttributeError):
+                    continue
+                if path:
+                    known[name] = path
+            return list(known.values())
 
     def prune_stale(self) -> list[str]:
         """Remove records whose source file no longer exists on disk.

services/watcher.py

@@ -90,6 +90,10 @@ def _under_nested_checkout(self, rel_dirs: tuple) -> bool:
                 return True
         return False
 
+    def under_nested_checkout(self, rel_path: str) -> bool:
+        """True if ``rel_path`` (relative to the root) sits in a nested checkout."""
+        return self._under_nested_checkout(Path(rel_path).parts[:-1])
+
     def _should_track(self, path: str) -> bool:
         p = Path(path)
         if p.suffix.lower() not in CODE_EXTENSIONS:

tests/test_file_memory_fast_paths.py

@@ -0,0 +1,106 @@
+"""list_tracked reads each record once, BranchWatch looks dirty files up one
+by one, and the FileMemory stale scan skips records in nested checkouts."""
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+
+from services.agents import BranchWatchAgent, FileMemoryAgent
+from services.file_memory import FileMemoryStore
+from services.notifications import NotificationStore
+from services.watcher import _ChangeHandler
+
+
+def _write(root: Path, rel: str, text: str) -> None:
+    p = root / rel
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(text, encoding="utf-8")
+
+
+class TestListTracked(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.root = Path(self.tmp.name)
+        _write(self.root, "a.py", "def a():\n    return 1\n")
+        _write(self.root, "pkg/b.py", "def b():\n    return 2\n")
+        self.fm = FileMemoryStore(str(self.root))
+        self.fm.update("a.py")
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def test_new_and_dropped_records_show_up(self):
+        self.assertEqual(self.fm.list_tracked(), ["a.py"])
+        self.fm.update("pkg/b.py")
+        self.assertEqual(sorted(self.fm.list_tracked()), ["a.py", "pkg/b.py"])
+        self.fm.drop("a.py")
+        self.assertEqual(self.fm.list_tracked(), ["pkg/b.py"])
+
+    def test_a_record_already_listed_is_not_read_again(self):
+        self.fm.list_tracked()
+        store_file = self.fm._store_path("a.py")
+        record = json.loads(store_file.read_text(encoding="utf-8"))
+        record["path"] = "changed.py"
+        store_file.write_text(json.dumps(record), encoding="utf-8")
+        self.assertEqual(self.fm.list_tracked(), ["a.py"])
+
+    def test_unreadable_record_is_skipped_and_retried(self):
+        broken = self.fm.store_dir / ("0" * 32 + ".json")
+        broken.write_text("{not json", encoding="utf-8")
+        self.assertEqual(self.fm.list_tracked(), ["a.py"])
+        broken.write_text(json.dumps({"path": "late.py"}), encoding="utf-8")
+        self.assertEqual(sorted(self.fm.list_tracked()), ["a.py", "late.py"])
+
+    def test_is_tracked_is_slash_agnostic(self):
+        self.fm.update("pkg/b.py")
+        self.assertTrue(self.fm.is_tracked("pkg/b.py"))
+        self.assertTrue(self.fm.is_tracked("pkg\\b.py"))
+        self.assertFalse(self.fm.is_tracked("pkg/missing.py"))
+
+
+class TestAgentsSkipFullScans(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.root = Path(self.tmp.name)
+        (self.root / ".c3").mkdir()
+        _write(self.root, "main.py", "def m():\n    return 1\n")
+        # A linked worktree: a directory whose .git is a FILE.
+        _write(self.root, "wt/copy.py", "def c():\n    return 1\n")
+        _write(self.root, "wt/.git", "gitdir: elsewhere/.git/worktrees/wt\n")
+        self.fm = FileMemoryStore(str(self.root))
+        self.fm.update("main.py")
+        self.fm.update("wt/copy.py")
+        self.notifs = NotificationStore(str(self.root))
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def test_stale_scan_skips_records_in_nested_checkouts(self):
+        handler = _ChangeHandler(root=str(self.root))
+        agent = FileMemoryAgent(self.fm, SimpleNamespace(_handler=handler),
+                                self.notifs, enabled=False)
+        _write(self.root, "main.py", "def m():\n    return 2\n")
+        _write(self.root, "wt/copy.py", "def c():\n    return 2\n")
+        handler._changes.append({"type": "modified", "path": str(self.root / "main.py")})
+        updated = []
+        real_update = self.fm.update
+        self.fm.update = lambda rel, *a, **k: (updated.append(rel), real_update(rel))[1]
+        agent.check()
+        self.assertEqual(updated, ["main.py"])
+
+    def test_branch_watch_looks_up_dirty_files_without_listing_the_store(self):
+        agent = BranchWatchAgent(self.fm, self.notifs, str(self.root), enabled=False)
+
+        def _no_full_scan():
+            raise AssertionError("list_tracked must not run for a dirty-file lookup")
+
+        self.fm.list_tracked = _no_full_scan
+        _write(self.root, "main.py", "def m():\n    return 3\n")
+        queued = agent._queue_changed(["main.py", "not_tracked.py"])
+        self.assertEqual(queued, 1)
+        self.assertIn("main.py", self.fm.drain_queue())
+
+
+if __name__ == "__main__":
+    unittest.main()