diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 6ff04cf..771a305 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -12,7 +12,7 @@ "name": "lcd", "source": "./", "description": "Lean Context Development — adaptive-weight agentic dev methodology (Quick / Standard / Deep lanes).", - "version": "0.16.0" + "version": "0.17.0" } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 80c58e9..f639946 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "lcd", "displayName": "Lean Context Development", - "version": "0.16.0", + "version": "0.17.0", "description": "Context-economy agentic dev methodology: process weight adapts to the work (Quick / Standard / Deep lanes) and durable artifacts make a context reset cheap.", "author": { "name": "stid" diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e80e6c..4e70a54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,30 @@ All notable changes to this project are documented here. Format loosely follows Development is tracked by decision IDs (`D-NNN`) in the project's internal decision log; entries cite them for traceability. +## [0.17.0] — 2026-08-22 + +The benchmark gains its missing comparison: a bare-agent baseline arm, so "context economy +pays" is a measured claim, not an argument (dev-harness decision D-025). + +### Added +- **`evals/run-eval.sh --baseline`** — bare-agent arm: no plugin loads, the workspace copy is + stripped of the fixture's methodology onboarding (strip verified fail-closed), the run is + driven by a new frozen prompt (`evals/feature-prompt-baseline.md`) whose feature paragraph + is byte-identical to the LCD prompt's (test-enforced), and the finish line is an `EVAL-DONE` + marker. `--baseline` + `--plugin-dir` is refused as a contradiction. +- **`evals/grade.sh --baseline`** — grades bare workspaces with the identical row shape + (`audit: n/a`); fail-closed both ways (residue under `--baseline` refused; bare workspace + without it still refused). Golden-locked grading criteria unchanged. +- **`tests/test-eval-baseline.sh`** — fixture tests for all of the above. + +### Changed +- **Both eval arms now run config-isolated, symmetrically:** `CLAUDE_CONFIG_DIR` points at a + fresh empty dir and `--settings '{"enabledPlugins":[]}'` is passed, so user-level installed + plugins and the personal `~/.claude/CLAUDE.md` no longer leak into benchmark runs (they did + by default; `--plugin-dir` adds to, not replaces, the installed set). The arms differ by + exactly one flag: `--plugin-dir`. Rows recorded before this isolation are not comparable + with rows after it. + ## [0.16.0] — 2026-08-22 From a full review of the plugin's prompts against Anthropic's current published guidance diff --git a/evals/feature-prompt-baseline.md b/evals/feature-prompt-baseline.md new file mode 100644 index 0000000..18b87b5 --- /dev/null +++ b/evals/feature-prompt-baseline.md @@ -0,0 +1,22 @@ +# Scripted feature — baseline arm (FROZEN — the benchmark's constant) + +> This prompt is handed verbatim to `claude -p` inside a fresh bare-project workspace +> derived from `evals/fixture/` with the methodology onboarding stripped. Its feature +> paragraph is byte-identical to `feature-prompt.md`'s (enforced by +> `tests/test-eval-baseline.sh`); only the process wrapper differs — that difference IS +> the experiment. Identical wording across runs of one experiment — do not edit between +> runs. + +You are working in a small Node.js project. Build the following feature end-to-end, +writing tests for everything you add and committing as you go: + +Add a `stats` operation exposed via both the CLI and HTTP: it returns the record +count, a last-updated indicator, and a checksum over the records; validation and +error semantics must be identical on both surfaces, with the shared logic living in +the core module both surfaces already use. + +Finish with the full test suite green. Work autonomously — do not ask questions, and do +not stop between steps: keep going in this same turn until the feature is complete and +the suite is green. When, and only when, that holds, create a file named `EVAL-DONE` at +the repository root as your final action. If hard-blocked, record why in `BLOCKED.md` at +the repository root and stop. diff --git a/evals/grade.sh b/evals/grade.sh index a065ac6..8d14aa4 100755 --- a/evals/grade.sh +++ b/evals/grade.sh @@ -13,12 +13,19 @@ # verdict, not the grader's exit code); 2 on unusable input (fail closed: this is an # instrument, unlike the plugin's fail-open hooks). # -# Usage: grade.sh [arm] +# Usage: grade.sh [--baseline] [arm] +# --baseline: grade a bare-agent workspace (D-025 in the dev harness). Fail-closed in +# BOTH directions: --baseline refuses a workspace that still contains docs/lcd (residue +# = silently-wrong baseline), and its absence keeps refusing non-LCD workspaces. +# The audit column short-circuits to n/a; every other column grades identically. set -uo pipefail +baseline=0 +if [[ "${1:-}" == "--baseline" ]]; then baseline=1; shift; fi + if [[ $# -lt 2 ]]; then - echo "usage: $0 [arm]" >&2 + echo "usage: $0 [--baseline] [arm]" >&2 exit 2 fi @@ -27,13 +34,20 @@ script_dir="$(cd "$(dirname "$0")" && pwd)" plugin_bin="$script_dir/../bin" [[ -d "$ws" ]] || { echo "error: workspace not found: $ws" >&2; exit 2; } -[[ -d "$ws/docs/lcd" ]] || { echo "error: $ws is not an LCD workspace" >&2; exit 2; } +if [[ $baseline -eq 1 ]]; then + [[ -d "$ws/docs/lcd" ]] && { echo "error: --baseline but workspace contains docs/lcd (methodology residue): $ws" >&2; exit 2; } +else + [[ -d "$ws/docs/lcd" ]] || { echo "error: $ws is not an LCD workspace" >&2; exit 2; } +fi work="$ws/docs/lcd/work/$slug" # --- audit (the same cross-path gate the methodology ships) ------------------------ # A run that stopped before producing spec+plan is a valid (failed) result, not a -# grader error: record it as INCOMPLETE so the experiment row still lands. -if [[ -f "$work/spec.md" && -f "$work/plan.md" ]]; then +# grader error: record it as INCOMPLETE so the experiment row still lands. A baseline +# workspace has no audit by definition: n/a, not a failure. +if [[ $baseline -eq 1 ]]; then + audit_res="n/a"; nonok="n/a" +elif [[ -f "$work/spec.md" && -f "$work/plan.md" ]]; then table="$(LCD_ROOT="$ws" LCD_SPECS_DIR="docs/lcd/work" bash "$plugin_bin/audit-crosspath.sh" "$slug" 2>/dev/null)" if [[ $? -eq 0 ]]; then audit_res="PASS"; else audit_res="BLOCKED"; fi nonok="$(printf '%s\n' "$table" \ diff --git a/evals/run-eval.sh b/evals/run-eval.sh index 66a8738..8c042bd 100755 --- a/evals/run-eval.sh +++ b/evals/run-eval.sh @@ -14,9 +14,19 @@ # CONTRIBUTING for the evidence-in-PR policy (contributors run locally with their own key). # # Usage: -# evals/run-eval.sh --arm