diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 771a305..f6b44a0 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -12,7 +12,7 @@ "name": "lcd", "source": "./", "description": "Lean Context Development — adaptive-weight agentic dev methodology (Quick / Standard / Deep lanes).", - "version": "0.17.0" + "version": "0.17.1" } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index f639946..9c8b1ee 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "lcd", "displayName": "Lean Context Development", - "version": "0.17.0", + "version": "0.17.1", "description": "Context-economy agentic dev methodology: process weight adapts to the work (Quick / Standard / Deep lanes) and durable artifacts make a context reset cheap.", "author": { "name": "stid" diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e70a54..e0350d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,23 @@ All notable changes to this project are documented here. Format loosely follows Development is tracked by decision IDs (`D-NNN`) in the project's internal decision log; entries cite them for traceability. +## [0.17.1] — 2026-08-28 + +Repair of the 0.17.0 isolation contract, found by the independent closeout evaluator +refuting the first baseline campaign: the runs it guarded were not isolated. + +### Fixed +- **The plugin-disable flag was schema-invalid and inert.** `--settings '{"enabledPlugins":[]}'` + passed an array where the CLI expects an object map with an explicit `false` per plugin — + so user-installed plugins (including lcd itself) still loaded into "baseline" runs. The + runner now builds the disable map from the operator's real `~/.claude/settings.json` + (`{"": false}` for every enabled plugin), failing closed when the file exists but + cannot be parsed, or when `jq` is missing. +- **Tests now assert the model call's real argv**, via an argv-recording stub standing in as + the model CLI: baseline legs must carry no `--plugin-dir` and the schema-valid disable + map; plugin-arm legs must carry `--plugin-dir` plus the same map. The previous tests only + checked the runner's printed plan — the exact gap the evaluator named. + ## [0.17.0] — 2026-08-22 The benchmark gains its missing comparison: a bare-agent baseline arm, so "context economy diff --git a/evals/run-eval.sh b/evals/run-eval.sh index 8c042bd..ee73788 100755 --- a/evals/run-eval.sh +++ b/evals/run-eval.sh @@ -24,9 +24,10 @@ # is an EVAL-DONE marker instead of the audit artifact. Contradicts --plugin-dir. # # Isolation (BOTH arms, symmetric): every model call runs with CLAUDE_CONFIG_DIR pointed -# at a fresh empty dir (bypasses user-level plugins, settings and CLAUDE.md; auth survives -# via the OS keychain) and --settings '{"enabledPlugins":[]}'. The arms differ by exactly -# one flag: the plugin arm adds --plugin-dir. +# at a fresh empty dir (bypasses user-level plugins, settings and CLAUDE.md where the host +# supports it) and --settings carrying a plugin-disable map built from the operator's real +# settings (object map, explicit false per plugin — the schema-valid form; an empty array +# is inert). The arms differ by exactly one flag: the plugin arm adds --plugin-dir. # Env: # EVAL_CLAUDE_BIN model CLI (default: claude) # EVAL_CLAUDE_MODEL model for the run (default: claude-fable-5 — the experiment's premise @@ -80,11 +81,28 @@ else fi [[ -n "$workspace" ]] || workspace="$(mktemp -d "${TMPDIR:-/tmp}/lcd-eval-XXXXXX")/$arm" config_dir="$(dirname "$workspace")/claude-config-$arm" -isolation_flags=(--settings '{"enabledPlugins":[]}') # --- preflight: fail closed, naming the precondition ------------------------------- refuse() { echo "refused: $1" >&2; exit 2; } +# Plugin-disable map (0.17.1, evaluator finding): enabledPlugins takes an OBJECT map with +# an explicit false per plugin — an empty array is schema-invalid and silently inert, and +# --plugin-dir ADDS to the installed set rather than replacing it. Build the map from the +# operator's real settings so every user-enabled plugin is disabled for the run; fail +# closed if the file exists but can't be parsed (an instrument must not guess). +user_settings="${HOME}/.claude/settings.json" +if [[ -f "$user_settings" ]]; then + command -v jq >/dev/null 2>&1 \ + || refuse "jq required to build the plugin-disable map from $user_settings" + disable_map="$(jq -c '{enabledPlugins: ((.enabledPlugins // {}) | with_entries(.value = false))}' \ + "$user_settings" 2>/dev/null)" \ + || refuse "cannot parse $user_settings to build the plugin-disable map" + [[ -n "$disable_map" ]] || refuse "cannot parse $user_settings to build the plugin-disable map" +else + disable_map='{"enabledPlugins":{}}' +fi +isolation_flags=(--settings "$disable_map") + [[ $baseline -eq 1 && $plugin_dir_explicit -eq 1 ]] \ && refuse "--baseline contradicts --plugin-dir (the baseline arm runs with no plugin)" [[ -d "$fixture" && -f "$fixture/package.json" ]] \ @@ -111,7 +129,7 @@ if [[ $dry_run -eq 1 ]]; then echo " prompt: $prompt_file" echo " workspace: $workspace" echo " results: $results" - echo " isolation: CLAUDE_CONFIG_DIR=$config_dir · --settings '{\"enabledPlugins\":[]}'" + echo " isolation: CLAUDE_CONFIG_DIR=$config_dir · --settings $disable_map" echo " model: $claude_bin --model $claude_model $claude_flags (max $max_legs legs)" exit 0 fi diff --git a/tests/test-eval-baseline.sh b/tests/test-eval-baseline.sh index 5540b07..bcb8c15 100755 --- a/tests/test-eval-baseline.sh +++ b/tests/test-eval-baseline.sh @@ -115,4 +115,58 @@ cols_b="$(grep ' · none · ' "$results" | head -1 | awk -F' · ' '{print NF}')" [[ -n "$cols_b" ]] || fail "AC-4 (EVAL): baseline row present and labelled with its arm" [[ "$cols_a" -eq "$cols_b" ]] || fail "AC-4 (EVAL): rows are column-identical across arms ($cols_a vs $cols_b)" +# --- AC-2 (CLI): the model call's REAL argv is isolated (evaluator finding, 0.17.1) ----- +# An argv-recording stub stands in for the model CLI: these tests assert what the leg loop +# actually executes, not what the runner prints. Fake HOME provides a user settings.json +# with enabled plugins; the runner must pass --settings with an OBJECT map disabling each. +stub="$tmp/claude-stub.sh" +cat > "$stub" << 'EOS' +#!/usr/bin/env bash +printf '%s\n' "$@" > "${ARGV_OUT:?}" +echo '{}' +EOS +chmod +x "$stub" +fakehome="$tmp/fakehome" +mkdir -p "$fakehome/.claude" +echo '{"enabledPlugins":{"lcd@lcd":true,"other@mkt":true}}' > "$fakehome/.claude/settings.json" + +extract_settings() { # print the value following --settings in the recorded argv + awk 'found {print; exit} $0=="--settings" {found=1}' "$1" +} + +export ARGV_OUT="$tmp/argv-none.txt" +out="$(HOME="$fakehome" EVAL_CLAUDE_BIN="$stub" EVAL_MAX_LEGS=1 bash "$runner" --arm none --baseline \ + --fixture "$fixture" --results "$tmp/r-argv.md" --workspace "$tmp/ws-argv-none" 2>&1)"; code=$? +assert_exit 0 "$code" "AC-2 (CLI): one-leg baseline run with argv stub completes" +[[ -f "$ARGV_OUT" ]] || fail "AC-2 (CLI): baseline leg actually invoked the model CLI" +argv="$(cat "$ARGV_OUT")" +assert_not_contains "$argv" '--plugin-dir' "AC-2 (CLI): baseline model call carries no --plugin-dir" +settings_json="$(extract_settings "$ARGV_OUT")" +[[ -n "$settings_json" ]] || fail "AC-2 (CLI): baseline model call passes --settings" +echo "$settings_json" | jq -e '.enabledPlugins | type == "object"' >/dev/null \ + || fail "AC-2 (CLI): enabledPlugins is an object map (schema-valid), got: $settings_json" +echo "$settings_json" | jq -e '.enabledPlugins["lcd@lcd"] == false and .enabledPlugins["other@mkt"] == false' >/dev/null \ + || fail "AC-2 (CLI): every user-enabled plugin explicitly disabled, got: $settings_json" + +export ARGV_OUT="$tmp/argv-lcd.txt" +out="$(HOME="$fakehome" EVAL_CLAUDE_BIN="$stub" EVAL_MAX_LEGS=1 bash "$runner" --arm A \ + --fixture "$fixture" --plugin-dir "$plugin_root" --results "$tmp/r-argv.md" \ + --workspace "$tmp/ws-argv-lcd" 2>&1)"; code=$? +assert_exit 0 "$code" "AC-2 (CLI): one-leg plugin-arm run with argv stub completes" +argv="$(cat "$ARGV_OUT")" +assert_contains "$argv" '--plugin-dir' "AC-2 (CLI): plugin-arm model call carries --plugin-dir" +settings_json="$(extract_settings "$ARGV_OUT")" +echo "$settings_json" | jq -e '.enabledPlugins["lcd@lcd"] == false' >/dev/null \ + || fail "AC-2 (CLI): plugin arm disables user-level plugins identically, got: $settings_json" +unset ARGV_OUT + +# --- AC-2 (CLI): unparseable user settings fail closed ---------------------------------- +badhome="$tmp/badhome" +mkdir -p "$badhome/.claude" +echo 'not json {' > "$badhome/.claude/settings.json" +out="$(HOME="$badhome" EVAL_CLAUDE_BIN=bash EVAL_MAX_LEGS=0 bash "$runner" --arm none --baseline \ + --fixture "$fixture" --results "$tmp/r-bad.md" --workspace "$tmp/ws-bad" 2>&1)"; code=$? +assert_exit 2 "$code" "AC-2 (CLI): unparseable user settings.json refused (no silent inert isolation)" +assert_contains "$out" 'settings' "AC-2 (CLI): refusal names the settings file" + exit 0