From 5305e1749e8c2f34f92bb0b49cc8943352f849a4 Mon Sep 17 00:00:00 2001 From: yepgent Date: Wed, 21 Oct 2026 07:40:18 -0400 Subject: [PATCH] perf(file_memory): stop re-reading every file map on each poll (2.158.1) list_tracked() JSON-loaded every record to read its path, and FileMemory and BranchWatch called it on timers: 64,126 records (292 MB) per call on the Yep project, 92% of them maps inside nested checkouts. It now reads only records it has not seen; BranchWatch uses is_tracked() per dirty file; the stale scan skips nested checkouts. Tests: tests/test_file_memory_fast_paths.py (4 of 6 fail on the old code). --- CHANGELOG.md | 15 ++++ cli/c3.py | 2 +- pyproject.toml | 2 +- services/agents.py | 11 ++- services/file_memory.py | 44 ++++++++--- services/watcher.py | 4 + tests/test_file_memory_fast_paths.py | 106 +++++++++++++++++++++++++++ 7 files changed, 166 insertions(+), 18 deletions(-) create mode 100644 tests/test_file_memory_fast_paths.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ee27f2..adaa887 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to Code Context Control (C3) are documented here. The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.158.1] - 2026-10-21 + +### Fixed — FileMemory and BranchWatch stop re-reading every file map + +`list_tracked()` JSON-loaded every record in `.c3/file_memory` to read one +field, and two background agents called it on a timer. On the Yep project +that was 64,126 records (292 MB) per call, 92% of them maps of files inside +nested checkouts, and the project's 14 MCP servers used about 4.5 CPU cores +between them. `list_tracked()` now lists the store and reads only records it +has not seen before; a record's filename is the md5 of its path, so a name +already read cannot change meaning. BranchWatch looks each dirty file up with +the new `is_tracked()` instead of listing the store every 30 s, and the +FileMemory stale scan skips records inside nested checkouts, which the +watcher never reports changes for. + ## [2.158.0] - 2026-10-20 ### Added — `format="json"` for c3_jira and c3_bitbucket diff --git a/cli/c3.py b/cli/c3.py index ce74d55..20f7be8 100644 --- a/cli/c3.py +++ b/cli/c3.py @@ -91,7 +91,7 @@ # Config CONFIG_DIR = ".c3" CONFIG_FILE = ".c3/config.json" -__version__ = "2.158.0" +__version__ = "2.158.1" # The PreToolUse matcher for native subagent calls (installer and hub migration). AGENT_MATCHER = "Agent|Task" diff --git a/pyproject.toml b/pyproject.toml index 0aea095..51fcf49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "code-context-control" -version = "2.158.0" +version = "2.158.1" description = "Local MCP code-intelligence for AI coding tools: surgical search/read/edit, agent-config version history, path-level access + masking guards, and a multi-project hub." readme = "README.md" requires-python = ">=3.10" diff --git a/services/agents.py b/services/agents.py index 4c8e304..991cf4d 100644 --- a/services/agents.py +++ b/services/agents.py @@ -903,8 +903,12 @@ def check(self): change_count = self.watcher._handler.change_count if change_count != self._last_change_count: self._last_change_count = change_count - # Check tracked files for staleness + # Check tracked files for staleness. The watcher never reports a + # change inside a nested checkout, so records there are skipped. + handler = self.watcher._handler for rel_path in self.file_memory.list_tracked(): + if handler.under_nested_checkout(rel_path): + continue if self.file_memory.needs_update(rel_path): files_to_process.append(rel_path) @@ -1007,11 +1011,10 @@ def _queue_changed(self, paths) -> int: """ if not paths: return 0 - tracked = {t.replace("\\", "/"): t for t in self.file_memory.list_tracked()} queued = 0 for p in paths: - canonical = tracked.get(p.replace("\\", "/")) - if canonical is None: + canonical = p.replace("\\", "/") + if not self.file_memory.is_tracked(canonical): continue try: if not self.file_memory.needs_update(canonical): diff --git a/services/file_memory.py b/services/file_memory.py index c2da06a..a79b190 100644 --- a/services/file_memory.py +++ b/services/file_memory.py @@ -8,6 +8,7 @@ """ import hashlib import json +import os import re import threading import time @@ -56,6 +57,10 @@ def __init__(self, project_path: str): # update()'s add_or_update on the shared TextIndex. Reentrant so # search() may hold it across _ensure_search_index() + the query. self._search_lock = threading.RLock() + # list_tracked(): record filename -> stored path. A filename is the + # md5 of its path, so a name already read never maps to another path. + self._tracked_paths: dict[str, str] = {} + self._tracked_lock = threading.Lock() # One-time cleanup of pre-2.121.0 generated summaries (marker-gated). try: self.purge_summaries() @@ -101,6 +106,10 @@ def purge_summaries(self) -> int: pass return rewritten + def is_tracked(self, rel_path: str) -> bool: + """True if a record exists for ``rel_path``, without reading it.""" + return self._store_path(str(rel_path).replace("\\", "/")).exists() + def get(self, rel_path: str) -> Optional[dict]: """Load a file's memory record, or None if not tracked.""" rel_path = str(rel_path).replace("\\", "/") @@ -391,18 +400,29 @@ def search_sections(sections, parent_name=""): return unique_ranges def list_tracked(self) -> list: - """Return relative paths of all tracked files.""" - tracked = [] - for f in self.store_dir.glob("*.json"): - if f.name.startswith("_"): - continue - try: - with open(f, encoding="utf-8") as fh: - data = json.load(fh) - tracked.append(data.get("path", "")) - except Exception: - continue - return [p for p in tracked if p] + """Return relative paths of all tracked files. + + Lists the store directory on every call but reads only records whose + filename it has not seen before; removed records drop out. + """ + try: + names = {e.name for e in os.scandir(self.store_dir) + if e.name.endswith(".json") and not e.name.startswith("_")} + except OSError: + return [] + with self._tracked_lock: + known = self._tracked_paths + for gone in known.keys() - names: + del known[gone] + for name in names - known.keys(): + try: + with open(self.store_dir / name, encoding="utf-8") as fh: + path = json.load(fh).get("path", "") + except (OSError, ValueError, AttributeError): + continue + if path: + known[name] = path + return list(known.values()) def prune_stale(self) -> list[str]: """Remove records whose source file no longer exists on disk. diff --git a/services/watcher.py b/services/watcher.py index 7e1667a..f7900b6 100644 --- a/services/watcher.py +++ b/services/watcher.py @@ -90,6 +90,10 @@ def _under_nested_checkout(self, rel_dirs: tuple) -> bool: return True return False + def under_nested_checkout(self, rel_path: str) -> bool: + """True if ``rel_path`` (relative to the root) sits in a nested checkout.""" + return self._under_nested_checkout(Path(rel_path).parts[:-1]) + def _should_track(self, path: str) -> bool: p = Path(path) if p.suffix.lower() not in CODE_EXTENSIONS: diff --git a/tests/test_file_memory_fast_paths.py b/tests/test_file_memory_fast_paths.py new file mode 100644 index 0000000..c7a8f2e --- /dev/null +++ b/tests/test_file_memory_fast_paths.py @@ -0,0 +1,106 @@ +"""list_tracked reads each record once, BranchWatch looks dirty files up one +by one, and the FileMemory stale scan skips records in nested checkouts.""" +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +from services.agents import BranchWatchAgent, FileMemoryAgent +from services.file_memory import FileMemoryStore +from services.notifications import NotificationStore +from services.watcher import _ChangeHandler + + +def _write(root: Path, rel: str, text: str) -> None: + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text, encoding="utf-8") + + +class TestListTracked(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + _write(self.root, "a.py", "def a():\n return 1\n") + _write(self.root, "pkg/b.py", "def b():\n return 2\n") + self.fm = FileMemoryStore(str(self.root)) + self.fm.update("a.py") + + def tearDown(self): + self.tmp.cleanup() + + def test_new_and_dropped_records_show_up(self): + self.assertEqual(self.fm.list_tracked(), ["a.py"]) + self.fm.update("pkg/b.py") + self.assertEqual(sorted(self.fm.list_tracked()), ["a.py", "pkg/b.py"]) + self.fm.drop("a.py") + self.assertEqual(self.fm.list_tracked(), ["pkg/b.py"]) + + def test_a_record_already_listed_is_not_read_again(self): + self.fm.list_tracked() + store_file = self.fm._store_path("a.py") + record = json.loads(store_file.read_text(encoding="utf-8")) + record["path"] = "changed.py" + store_file.write_text(json.dumps(record), encoding="utf-8") + self.assertEqual(self.fm.list_tracked(), ["a.py"]) + + def test_unreadable_record_is_skipped_and_retried(self): + broken = self.fm.store_dir / ("0" * 32 + ".json") + broken.write_text("{not json", encoding="utf-8") + self.assertEqual(self.fm.list_tracked(), ["a.py"]) + broken.write_text(json.dumps({"path": "late.py"}), encoding="utf-8") + self.assertEqual(sorted(self.fm.list_tracked()), ["a.py", "late.py"]) + + def test_is_tracked_is_slash_agnostic(self): + self.fm.update("pkg/b.py") + self.assertTrue(self.fm.is_tracked("pkg/b.py")) + self.assertTrue(self.fm.is_tracked("pkg\\b.py")) + self.assertFalse(self.fm.is_tracked("pkg/missing.py")) + + +class TestAgentsSkipFullScans(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + (self.root / ".c3").mkdir() + _write(self.root, "main.py", "def m():\n return 1\n") + # A linked worktree: a directory whose .git is a FILE. + _write(self.root, "wt/copy.py", "def c():\n return 1\n") + _write(self.root, "wt/.git", "gitdir: elsewhere/.git/worktrees/wt\n") + self.fm = FileMemoryStore(str(self.root)) + self.fm.update("main.py") + self.fm.update("wt/copy.py") + self.notifs = NotificationStore(str(self.root)) + + def tearDown(self): + self.tmp.cleanup() + + def test_stale_scan_skips_records_in_nested_checkouts(self): + handler = _ChangeHandler(root=str(self.root)) + agent = FileMemoryAgent(self.fm, SimpleNamespace(_handler=handler), + self.notifs, enabled=False) + _write(self.root, "main.py", "def m():\n return 2\n") + _write(self.root, "wt/copy.py", "def c():\n return 2\n") + handler._changes.append({"type": "modified", "path": str(self.root / "main.py")}) + updated = [] + real_update = self.fm.update + self.fm.update = lambda rel, *a, **k: (updated.append(rel), real_update(rel))[1] + agent.check() + self.assertEqual(updated, ["main.py"]) + + def test_branch_watch_looks_up_dirty_files_without_listing_the_store(self): + agent = BranchWatchAgent(self.fm, self.notifs, str(self.root), enabled=False) + + def _no_full_scan(): + raise AssertionError("list_tracked must not run for a dirty-file lookup") + + self.fm.list_tracked = _no_full_scan + _write(self.root, "main.py", "def m():\n return 3\n") + queued = agent._queue_changed(["main.py", "not_tracked.py"]) + self.assertEqual(queued, 1) + self.assertIn("main.py", self.fm.drain_queue()) + + +if __name__ == "__main__": + unittest.main()