fix(git): include untracked new files in diff snapshots (DEBUG #57)

CHANGELOG.md

@@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Fixed
+- **失败现场 diff 包含未跟踪新文件（DEBUG #57）**：`capture_diff_snapshot` 在 `git diff HEAD` 之后追加未跟踪新文件的 unified diff。失败侧 `_failure_diff` 与成功侧共用这一 helper。依赖树、缓存、运行态目录、受保护路径和含 NUL 的二进制不进入这份补充。不改固化门，不写 Parent baseline，不开 val 密封。
+
 ### Docs
 - **README 对齐软默认与 MCP 口味（英/中）**：soft-accept 为 MCP 默认路径（经验写入 `~/.evolver/workspaces/<id>/`，不自动 git commit，不写 Parent baseline；`no_baseline` / `pack_absent` / `unmeasured` / `drop` / `flat` / `regression` 为警告）。硬阻断仍为密钥、`~/.evolver/`、`val_seal`。补开箱 `setup` / `doctor`、soft T0、前 5 tick 核心循环、`pending_solidify`、安装后重启 MCP，以及本 fork 的 clone URL（`linkerlin/evolver.py`）。顺手修正仍写「缺基线即停 / 过门即 commit / drop 仍硬拒绝」的 quickstart、AGENTS.md 与教程句子。
 

DEBUG.md

@@ -1082,18 +1082,25 @@
   后再谈归因，截断输出里的百分比进度不是证据；**`lastfailed` 不是线索库，
   是考古层**——`--lf` 前先 `--collect-only` 验 id 存活，零命中会静默变成全量。
 
-### 57. 失败现场 diff 看不见未跟踪文件（MCP 验证脚本实证，未修）
+### 57. 失败现场 diff 看不见未跟踪文件（MCP 验证脚本实证，已修）
 
 - **症状**：MCP 闭环验证脚本里，宿主新建 `notes.md`（未跟踪）后走
   `hypothesis_missing` 拒绝，episode 的 diff 为空，断言失败；把同一改动
   落在已跟踪的 `README.md` 上后 diff 正常，脚本 PASS。
 - **根因**：`_failure_diff` → `capture_diff_snapshot` 即 `git diff HEAD`，
   天然不含未跟踪文件。成功侧走同一 helper、同盲区——行为一致，没有偏袒
   任一臂，但两边都可能丢掉"新建文件"类候选的关键现场。
-- **修复**：未修。改快照口径牵连成功侧与既有 diff 断言，需单独一轮（快照
-  未跟踪内容 + 钉住两臂一致 + 全量回归）。
-- **经验**：**验证脚本模拟宿主改动必须落在已跟踪文件上**；凡断言 diff
-  内容，先确认文件跟踪状态——`git diff HEAD` 类快照默认看不见新文件。
+- **修复**：`capture_diff_snapshot` 仍先取 `git diff HEAD`（已跟踪文件的
+  暂存与未暂存改动，含已 `git add` 的新文件），再把从未入库的未跟踪新文件
+  收成 `diff --git` 新文件块接在后面。失败侧 `_failure_diff` 与成功侧
+  都走这一 helper，两臂同一口径。补充块跳过爆炸半径不计的路径（`.venv`、
+  `node_modules`、`__pycache__`、`.git`、`.env` 等受保护文件）、运行态目录
+  （`memory/`、`.evolver/` 等，与 `_is_runtime_state` 同口径）以及文件头
+  含 NUL 的二进制；读入量不超过快照剩余预算。不改固化门，不写 Parent
+  baseline，不开 val 密封。
+- **经验**：**凡断言 diff 内容，先确认文件是不是未跟踪**——修之前
+  `git diff HEAD` 看不见新文件。快照要看见新文件，就改共用 helper，
+  不要给失败侧单开一条路。运行态与依赖树不能跟候选正文抢那 2000 字。
 
 ### 58. 外部 MCP 评审 P1 四项（评估会话，全修）
 

src/evolver/gep/git_ops.py

@@ -11,7 +11,7 @@
 import time
 from collections.abc import Sequence
 from pathlib import Path
-from typing import Any
+from typing import Any, Final
 
 from evolver.solo.git_guard import guard_git_subcommand
 
@@ -95,12 +95,120 @@ def git_list_untracked_files(cwd: Path | str | None = None) -> list[str]:
     return [line for line in raw.splitlines() if line.strip()]
 
 
+# Path components that are dependency or cache trees, including nested ones.
+# Top-level prefixes are also dropped by is_constraint_counted_path.
+_SNAPSHOT_NOISE_PARTS: Final[frozenset[str]] = frozenset(
+    {
+        ".git",
+        "node_modules",
+        ".venv",
+        "venv",
+        "__pycache__",
+        ".pytest_cache",
+    }
+)
+# Top-level runtime heads. Same set as solidify._is_runtime_state: the
+# engine's own write domain, not the mutation scene. Only the untracked
+# supplement is filtered; `git diff HEAD` is left unchanged.
+_SNAPSHOT_RUNTIME_HEADS: Final[frozenset[str]] = frozenset(
+    {
+        ".evolver",
+        ".evolver_settings",
+        ".evomap",
+        ".pytest_cache",
+        "logs",
+        "memory",
+    }
+)
+_BINARY_SNIFF_BYTES: Final = 8192
+
+
+def _untracked_snapshot_candidate(rel: str) -> bool:
+    """True when an untracked path belongs in the mutation diff scene."""
+    norm = normalize_rel_path(rel)
+    if not norm or not is_constraint_counted_path(norm):
+        return False
+    if norm.startswith("evolver/.config/") or norm.endswith(".pyc"):
+        return False
+    parts = norm.split("/")
+    if parts[0] in _SNAPSHOT_RUNTIME_HEADS:
+        return False
+    return all(part not in _SNAPSHOT_NOISE_PARTS for part in parts)
+
+
+def _format_new_file_diff(rel: str, text: str) -> str:
+    """Unified diff for a file that exists only in the worktree."""
+    path = normalize_rel_path(rel)
+    header = f"diff --git a/{path} b/{path}\nnew file mode 100644\n--- /dev/null\n+++ b/{path}\n"
+    if text == "":
+        return header
+    lines = text.splitlines()
+    body = "\n".join(f"+{line}" for line in lines)
+    if text.endswith("\n"):
+        body += "\n"
+    else:
+        body += "\n\\ No newline at end of file\n"
+    return f"{header}@@ -0,0 +1,{len(lines)} @@\n{body}"
+
+
+def _read_text_prefix(path: Path, limit: int) -> str | None:
+    """UTF-8 prefix of a regular file, or None when it is missing or binary."""
+    if limit <= 0 or not path.is_file():
+        return None
+    try:
+        with path.open("rb") as handle:
+            sniff = handle.read(_BINARY_SNIFF_BYTES)
+            if b"\x00" in sniff:
+                return None
+            blob = sniff[:limit] if len(sniff) >= limit else sniff + handle.read(limit - len(sniff))
+    except OSError:
+        return None
+    return blob.decode("utf-8", errors="replace")
+
+
+def _untracked_diff_text(cwd: Path, *, budget: int) -> str:
+    """New-file diffs for untracked candidates, capped at ``budget`` chars."""
+    if budget <= 0:
+        return ""
+    chunks: list[str] = []
+    used = 0
+    for rel in git_list_untracked_files(cwd):
+        if used >= budget or not _untracked_snapshot_candidate(rel):
+            continue
+        remaining = budget - used
+        text = _read_text_prefix(cwd / rel, remaining)
+        if text is None:
+            continue
+        chunk = _format_new_file_diff(rel, text)
+        if len(chunk) > remaining:
+            chunk = chunk[:remaining]
+        chunks.append(chunk)
+        used += len(chunk)
+    return "".join(chunks)
+
+
 def capture_diff_snapshot(
     cwd: Path | str | None = None,
     max_chars: int = 80_000,
 ) -> str:
-    """Capture current diff vs HEAD, truncated to max_chars."""
-    raw = try_run_cmd(["diff", "HEAD"], cwd=cwd)
+    """Capture the worktree vs HEAD, including untracked new files.
+
+    ``git diff HEAD`` already contains staged and unstaged edits to tracked
+    paths. A file that was never ``git add``ed is absent from that diff, so
+    its content is appended as a ``diff --git`` new-file hunk. Failure and
+    success snapshots share this helper. Noise trees, runtime-state dirs,
+    protected paths, and binaries are left out of the untracked supplement.
+    The result is truncated to ``max_chars``.
+    """
+    root = Path(cwd) if cwd is not None else Path.cwd()
+    raw = try_run_cmd(["diff", "HEAD"], cwd=root)
+    if len(raw) > max_chars:
+        return raw[:max_chars] + "\n...[truncated]\n"
+    extra = _untracked_diff_text(root, budget=max_chars - len(raw))
+    if extra:
+        if raw and not raw.endswith("\n"):
+            raw += "\n"
+        raw += extra
     if len(raw) > max_chars:
         return raw[:max_chars] + "\n...[truncated]\n"
     return raw

tests/gep/test_solidify.py

@@ -231,6 +231,52 @@ def test_run_validations_timeout(git_ws: Path, monkeypatch: pytest.MonkeyPatch)
     assert res["results"][0]["stderr"]
 
 
+def test_hypothesis_rejection_records_untracked_new_file(git_ws: Path) -> None:
+    """DEBUG #57: a gate reject must keep an untracked new file in the scene.
+
+    The file is rolled back afterwards; the episode is the only copy left.
+    """
+    (git_ws / "notes.md").write_text("untracked-notes-marker\n", encoding="utf-8")
+    write_state_for_solidify(_last_run())
+    result = solidify()
+    assert result["ok"] is False
+    assert result["error"] == "hypothesis_missing"
+    from evolver.gep.episode_record import load_episode, record_round
+
+    stored = record_round(event_id=str(result["event_id"]))
+    body = load_episode(stored["id"])
+    assert body is not None
+    assert "diff --git a/notes.md b/notes.md" in body["diff"]
+    assert "untracked-notes-marker" in body["diff"]
+    assert not (git_ws / "notes.md").exists()
+
+
+def test_success_diff_snapshot_includes_untracked_new_file(
+    git_ws: Path,
+    armed_pack: dict[str, Any],
+    declared_hypothesis: dict[str, Any],
+) -> None:
+    """Success uses the same snapshot helper, so a new file is recorded there too."""
+    _ = armed_pack, declared_hypothesis
+    set_flag("enable_fitness_cascade", False, persist=False)
+    (git_ws / "notes.md").write_text("untracked-notes-marker\n", encoding="utf-8")
+    write_state_for_solidify(
+        _last_run(
+            mutation={
+                "id": "mut_untracked_ok",
+                "validation": [[sys.executable, "-c", "print(1)"]],
+            }
+        )
+    )
+    result = solidify()
+    assert result["ok"] is True
+    evt = json.loads(
+        (get_gep_assets_dir() / "events.jsonl").read_text(encoding="utf-8").strip().splitlines()[-1]
+    )
+    assert "untracked-notes-marker" in str(evt.get("diff_snapshot") or "")
+    assert "diff --git a/notes.md" in str(evt.get("diff_snapshot") or "")
+
+
 def test_hypothesis_rejection_leaves_a_recordable_scene(git_ws: Path) -> None:
     """A settled Reject is an engine scene: gene, diff, and the gate verdict."""
     (git_ws / "README.md").write_text("changed by the candidate\n", encoding="utf-8")

tests/test_git_ops.py

@@ -89,6 +89,80 @@ def test_is_git_repo_true_and_false(tmp_path: Path) -> None:
     assert is_git_repo(tmp_path) is True
 
 
+def _init_repo(path: Path) -> None:
+    subprocess.run(["git", "init"], cwd=path, check=True, capture_output=True)
+    subprocess.run(
+        ["git", "config", "user.email", "t@t.com"], cwd=path, check=True, capture_output=True
+    )
+    subprocess.run(["git", "config", "user.name", "T"], cwd=path, check=True, capture_output=True)
+    (path / "README.md").write_text("init\n", encoding="utf-8")
+    subprocess.run(["git", "add", "README.md"], cwd=path, check=True, capture_output=True)
+    subprocess.run(
+        ["git", "-c", "commit.gpgsign=false", "commit", "-m", "init"],
+        cwd=path,
+        check=True,
+        capture_output=True,
+    )
+
+
+def test_capture_diff_snapshot_includes_untracked_new_file(tmp_path: Path) -> None:
+    """DEBUG #57: a new file that was never git-added is part of the scene."""
+    _init_repo(tmp_path)
+    (tmp_path / "notes.md").write_text("untracked-notes-marker\n", encoding="utf-8")
+    snap = capture_diff_snapshot(tmp_path)
+    assert "diff --git a/notes.md b/notes.md" in snap
+    assert "+untracked-notes-marker" in snap
+
+
+def test_capture_diff_snapshot_includes_staged_new_file(tmp_path: Path) -> None:
+    """A new file already in the index stays visible via git diff HEAD."""
+    _init_repo(tmp_path)
+    (tmp_path / "added.py").write_text("staged-new-marker\n", encoding="utf-8")
+    subprocess.run(["git", "add", "added.py"], cwd=tmp_path, check=True, capture_output=True)
+    snap = capture_diff_snapshot(tmp_path)
+    assert "staged-new-marker" in snap
+
+
+def test_capture_diff_snapshot_keeps_tracked_edit_beside_untracked(tmp_path: Path) -> None:
+    _init_repo(tmp_path)
+    (tmp_path / "README.md").write_text("tracked-edit-marker\n", encoding="utf-8")
+    (tmp_path / "notes.md").write_text("untracked-notes-marker\n", encoding="utf-8")
+    snap = capture_diff_snapshot(tmp_path)
+    assert "tracked-edit-marker" in snap
+    assert "untracked-notes-marker" in snap
+
+
+def test_capture_diff_snapshot_skips_noise_and_binaries(tmp_path: Path) -> None:
+    """Explosion-radius noise and binaries must not crowd the mutation scene."""
+    _init_repo(tmp_path)
+    (tmp_path / "notes.md").write_text("untracked-notes-marker\n", encoding="utf-8")
+    noise = {
+        tmp_path / ".venv" / "lib" / "x.py": "noise-venv-marker\n",
+        tmp_path / "pkg" / "node_modules" / "leftpad.js": "noise-node-modules-marker\n",
+        tmp_path / "src" / "__pycache__" / "x.pyc": "noise-pycache-marker\n",
+        tmp_path / "memory" / "evolution_solidify_state.json": "noise-memory-marker\n",
+        tmp_path / ".evolver" / "gep" / "events.jsonl": "noise-evolver-marker\n",
+        tmp_path / ".env": "API_KEY=noise-env-marker\n",
+    }
+    for path, text in noise.items():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(text, encoding="utf-8")
+    blob = tmp_path / "payload.bin"
+    blob.write_bytes(b"\x00noise-binary-marker")
+    snap = capture_diff_snapshot(tmp_path)
+    assert "untracked-notes-marker" in snap
+    for marker in (
+        "noise-venv-marker",
+        "noise-node-modules-marker",
+        "noise-pycache-marker",
+        "noise-memory-marker",
+        "noise-evolver-marker",
+        "noise-env-marker",
+        "noise-binary-marker",
+    ):
+        assert marker not in snap
+
+
 def test_capture_diff_snapshot_truncates(tmp_path: Path) -> None:
     subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True)
     subprocess.run(