fix: eval isolation was inert — schema-valid plugin-disable map + real-argv tests

.claude-plugin/marketplace.json

@@ -12,7 +12,7 @@
       "name": "lcd",
       "source": "./",
       "description": "Lean Context Development — adaptive-weight agentic dev methodology (Quick / Standard / Deep lanes).",
-      "version": "0.17.0"
+      "version": "0.17.1"
     }
   ]
 }

.claude-plugin/plugin.json

@@ -2,7 +2,7 @@
   "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
   "name": "lcd",
   "displayName": "Lean Context Development",
-  "version": "0.17.0",
+  "version": "0.17.1",
   "description": "Context-economy agentic dev methodology: process weight adapts to the work (Quick / Standard / Deep lanes) and durable artifacts make a context reset cheap.",
   "author": {
     "name": "stid"

CHANGELOG.md

@@ -6,6 +6,23 @@ All notable changes to this project are documented here. Format loosely follows
 Development is tracked by decision IDs (`D-NNN`) in the project's internal decision log; entries
 cite them for traceability.
 
+## [0.17.1] — 2026-08-28
+
+Repair of the 0.17.0 isolation contract, found by the independent closeout evaluator
+refuting the first baseline campaign: the runs it guarded were not isolated.
+
+### Fixed
+- **The plugin-disable flag was schema-invalid and inert.** `--settings '{"enabledPlugins":[]}'`
+  passed an array where the CLI expects an object map with an explicit `false` per plugin —
+  so user-installed plugins (including lcd itself) still loaded into "baseline" runs. The
+  runner now builds the disable map from the operator's real `~/.claude/settings.json`
+  (`{"<plugin>": false}` for every enabled plugin), failing closed when the file exists but
+  cannot be parsed, or when `jq` is missing.
+- **Tests now assert the model call's real argv**, via an argv-recording stub standing in as
+  the model CLI: baseline legs must carry no `--plugin-dir` and the schema-valid disable
+  map; plugin-arm legs must carry `--plugin-dir` plus the same map. The previous tests only
+  checked the runner's printed plan — the exact gap the evaluator named.
+
 ## [0.17.0] — 2026-08-22
 
 The benchmark gains its missing comparison: a bare-agent baseline arm, so "context economy

evals/run-eval.sh

@@ -24,9 +24,10 @@
 # is an EVAL-DONE marker instead of the audit artifact. Contradicts --plugin-dir.
 #
 # Isolation (BOTH arms, symmetric): every model call runs with CLAUDE_CONFIG_DIR pointed
-# at a fresh empty dir (bypasses user-level plugins, settings and CLAUDE.md; auth survives
-# via the OS keychain) and --settings '{"enabledPlugins":[]}'. The arms differ by exactly
-# one flag: the plugin arm adds --plugin-dir.
+# at a fresh empty dir (bypasses user-level plugins, settings and CLAUDE.md where the host
+# supports it) and --settings carrying a plugin-disable map built from the operator's real
+# settings (object map, explicit false per plugin — the schema-valid form; an empty array
+# is inert). The arms differ by exactly one flag: the plugin arm adds --plugin-dir.
 # Env:
 #   EVAL_CLAUDE_BIN    model CLI (default: claude)
 #   EVAL_CLAUDE_MODEL  model for the run (default: claude-fable-5 — the experiment's premise
@@ -80,11 +81,28 @@ else
 fi
 [[ -n "$workspace" ]] || workspace="$(mktemp -d "${TMPDIR:-/tmp}/lcd-eval-XXXXXX")/$arm"
 config_dir="$(dirname "$workspace")/claude-config-$arm"
-isolation_flags=(--settings '{"enabledPlugins":[]}')
 
 # --- preflight: fail closed, naming the precondition -------------------------------
 refuse() { echo "refused: $1" >&2; exit 2; }
 
+# Plugin-disable map (0.17.1, evaluator finding): enabledPlugins takes an OBJECT map with
+# an explicit false per plugin — an empty array is schema-invalid and silently inert, and
+# --plugin-dir ADDS to the installed set rather than replacing it. Build the map from the
+# operator's real settings so every user-enabled plugin is disabled for the run; fail
+# closed if the file exists but can't be parsed (an instrument must not guess).
+user_settings="${HOME}/.claude/settings.json"
+if [[ -f "$user_settings" ]]; then
+  command -v jq >/dev/null 2>&1 \
+    || refuse "jq required to build the plugin-disable map from $user_settings"
+  disable_map="$(jq -c '{enabledPlugins: ((.enabledPlugins // {}) | with_entries(.value = false))}' \
+      "$user_settings" 2>/dev/null)" \
+    || refuse "cannot parse $user_settings to build the plugin-disable map"
+  [[ -n "$disable_map" ]] || refuse "cannot parse $user_settings to build the plugin-disable map"
+else
+  disable_map='{"enabledPlugins":{}}'
+fi
+isolation_flags=(--settings "$disable_map")
+
 [[ $baseline -eq 1 && $plugin_dir_explicit -eq 1 ]] \
   && refuse "--baseline contradicts --plugin-dir (the baseline arm runs with no plugin)"
 [[ -d "$fixture" && -f "$fixture/package.json" ]] \
@@ -111,7 +129,7 @@ if [[ $dry_run -eq 1 ]]; then
   echo "  prompt: $prompt_file"
   echo "  workspace: $workspace"
   echo "  results: $results"
-  echo "  isolation: CLAUDE_CONFIG_DIR=$config_dir · --settings '{\"enabledPlugins\":[]}'"
+  echo "  isolation: CLAUDE_CONFIG_DIR=$config_dir · --settings $disable_map"
   echo "  model: $claude_bin --model $claude_model $claude_flags (max $max_legs legs)"
   exit 0
 fi

tests/test-eval-baseline.sh

@@ -115,4 +115,58 @@ cols_b="$(grep ' · none · ' "$results" | head -1 | awk -F' · ' '{print NF}')"
 [[ -n "$cols_b" ]] || fail "AC-4 (EVAL): baseline row present and labelled with its arm"
 [[ "$cols_a" -eq "$cols_b" ]] || fail "AC-4 (EVAL): rows are column-identical across arms ($cols_a vs $cols_b)"
 
+# --- AC-2 (CLI): the model call's REAL argv is isolated (evaluator finding, 0.17.1) -----
+# An argv-recording stub stands in for the model CLI: these tests assert what the leg loop
+# actually executes, not what the runner prints. Fake HOME provides a user settings.json
+# with enabled plugins; the runner must pass --settings with an OBJECT map disabling each.
+stub="$tmp/claude-stub.sh"
+cat > "$stub" << 'EOS'
+#!/usr/bin/env bash
+printf '%s\n' "$@" > "${ARGV_OUT:?}"
+echo '{}'
+EOS
+chmod +x "$stub"
+fakehome="$tmp/fakehome"
+mkdir -p "$fakehome/.claude"
+echo '{"enabledPlugins":{"lcd@lcd":true,"other@mkt":true}}' > "$fakehome/.claude/settings.json"
+
+extract_settings() {  # print the value following --settings in the recorded argv
+  awk 'found {print; exit} $0=="--settings" {found=1}' "$1"
+}
+
+export ARGV_OUT="$tmp/argv-none.txt"
+out="$(HOME="$fakehome" EVAL_CLAUDE_BIN="$stub" EVAL_MAX_LEGS=1 bash "$runner" --arm none --baseline \
+        --fixture "$fixture" --results "$tmp/r-argv.md" --workspace "$tmp/ws-argv-none" 2>&1)"; code=$?
+assert_exit 0 "$code" "AC-2 (CLI): one-leg baseline run with argv stub completes"
+[[ -f "$ARGV_OUT" ]] || fail "AC-2 (CLI): baseline leg actually invoked the model CLI"
+argv="$(cat "$ARGV_OUT")"
+assert_not_contains "$argv" '--plugin-dir' "AC-2 (CLI): baseline model call carries no --plugin-dir"
+settings_json="$(extract_settings "$ARGV_OUT")"
+[[ -n "$settings_json" ]] || fail "AC-2 (CLI): baseline model call passes --settings"
+echo "$settings_json" | jq -e '.enabledPlugins | type == "object"' >/dev/null \
+  || fail "AC-2 (CLI): enabledPlugins is an object map (schema-valid), got: $settings_json"
+echo "$settings_json" | jq -e '.enabledPlugins["lcd@lcd"] == false and .enabledPlugins["other@mkt"] == false' >/dev/null \
+  || fail "AC-2 (CLI): every user-enabled plugin explicitly disabled, got: $settings_json"
+
+export ARGV_OUT="$tmp/argv-lcd.txt"
+out="$(HOME="$fakehome" EVAL_CLAUDE_BIN="$stub" EVAL_MAX_LEGS=1 bash "$runner" --arm A \
+        --fixture "$fixture" --plugin-dir "$plugin_root" --results "$tmp/r-argv.md" \
+        --workspace "$tmp/ws-argv-lcd" 2>&1)"; code=$?
+assert_exit 0 "$code" "AC-2 (CLI): one-leg plugin-arm run with argv stub completes"
+argv="$(cat "$ARGV_OUT")"
+assert_contains "$argv" '--plugin-dir' "AC-2 (CLI): plugin-arm model call carries --plugin-dir"
+settings_json="$(extract_settings "$ARGV_OUT")"
+echo "$settings_json" | jq -e '.enabledPlugins["lcd@lcd"] == false' >/dev/null \
+  || fail "AC-2 (CLI): plugin arm disables user-level plugins identically, got: $settings_json"
+unset ARGV_OUT
+
+# --- AC-2 (CLI): unparseable user settings fail closed ----------------------------------
+badhome="$tmp/badhome"
+mkdir -p "$badhome/.claude"
+echo 'not json {' > "$badhome/.claude/settings.json"
+out="$(HOME="$badhome" EVAL_CLAUDE_BIN=bash EVAL_MAX_LEGS=0 bash "$runner" --arm none --baseline \
+        --fixture "$fixture" --results "$tmp/r-bad.md" --workspace "$tmp/ws-bad" 2>&1)"; code=$?
+assert_exit 2 "$code" "AC-2 (CLI): unparseable user settings.json refused (no silent inert isolation)"
+assert_contains "$out" 'settings' "AC-2 (CLI): refusal names the settings file"
+
 exit 0