From 70bfcff8f6b52bc92f17ec61d96c79471b7e3fa2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 May 2026 16:03:18 +0000 Subject: [PATCH 001/100] chore(deps): bump hono in the npm_and_yarn group across 1 directory Bumps the npm_and_yarn group with 1 update in the / directory: [hono](https://github.com/honojs/hono). Updates `hono` from 4.12.14 to 4.12.16 - [Release notes](https://github.com/honojs/hono/releases) - [Commits](https://github.com/honojs/hono/compare/v4.12.14...v4.12.16) --- updated-dependencies: - dependency-name: hono dependency-version: 4.12.16 dependency-type: direct:production dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] --- pnpm-lock.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2c6e5ab..eac8e4b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -49,7 +49,7 @@ importers: version: 10.5.0 hono: specifier: ^4.12.12 - version: 4.12.14 + version: 4.12.16 ignore: specifier: ^5.3.1 version: 5.3.2 @@ -1596,8 +1596,8 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} - hono@4.12.14: - resolution: {integrity: sha512-am5zfg3yu6sqn5yjKBNqhnTX7Cv+m00ox+7jbaKkrLMRJ4rAdldd1xPd/JzbBWspqaQv6RSTrgFN95EsfhC+7w==} + hono@4.12.16: + resolution: {integrity: sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==} engines: {node: '>=16.9.0'} http-errors@2.0.1: @@ -2728,9 +2728,9 @@ snapshots: '@eslint/core': 0.17.0 levn: 0.4.1 - '@hono/node-server@1.19.13(hono@4.12.14)': + '@hono/node-server@1.19.13(hono@4.12.16)': dependencies: - hono: 4.12.14 + hono: 4.12.16 '@huggingface/jinja@0.5.5': {} @@ -2979,7 +2979,7 @@ snapshots: '@modelcontextprotocol/sdk@1.27.1(zod@4.3.4)': dependencies: - '@hono/node-server': 1.19.13(hono@4.12.14) + '@hono/node-server': 1.19.13(hono@4.12.16) ajv: 8.18.0 ajv-formats: 3.0.1(ajv@8.18.0) content-type: 1.0.5 @@ -2989,7 +2989,7 @@ snapshots: eventsource-parser: 3.0.6 express: 5.2.1 express-rate-limit: 8.2.2(express@5.2.1) - hono: 4.12.14 + hono: 4.12.16 jose: 6.1.3 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 @@ -4159,7 +4159,7 @@ snapshots: dependencies: function-bind: 1.1.2 - hono@4.12.14: {} + hono@4.12.16: {} http-errors@2.0.1: dependencies: From d0b0e4cb4b09afcad9f26cf414253aba9b1fc6b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Sun, 10 May 2026 23:43:13 +0200 Subject: [PATCH 002/100] Run ContextBench codebase-memory recovery on CI --- .../contextbench-ci-recovery-master.yml | 270 ++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 .github/workflows/contextbench-ci-recovery-master.yml diff --git a/.github/workflows/contextbench-ci-recovery-master.yml b/.github/workflows/contextbench-ci-recovery-master.yml new file mode 100644 index 0000000..eb3127c --- /dev/null +++ b/.github/workflows/contextbench-ci-recovery-master.yml @@ -0,0 +1,270 @@ +name: ContextBench CI Recovery Master + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-ci-recovery-master.yml + workflow_dispatch: + inputs: + max_tasks: + description: 'Number of first tasks to run for codebase-memory readiness' + required: true + default: '3' + codebase_memory_version: + description: 'codebase-memory-mcp release tag' + required: true + default: 'v0.6.1' + +permissions: + contents: read + +jobs: + codebase-memory-first3-readiness: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + CI_READINESS_ROOT: /tmp/contextbench-readiness + TASK_PAYLOADS: /tmp/contextbench-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + CBM_VERSION: ${{ github.event.inputs.codebase_memory_version || 'v0.6.1' }} + MAX_TASKS: ${{ github.event.inputs.max_tasks || '3' }} + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v2 + with: + version: 10 + + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'pnpm' + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install repo dependencies + run: pnpm install --frozen-lockfile + + - name: Install official evaluator dependencies + run: python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + + - name: Validate frozen ContextBench fixtures + run: node scripts/contextbench-runner.mjs --validate-fixtures + + - name: Materialize first task payloads and checkouts + run: | + mkdir -p "$CI_READINESS_ROOT" "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks "$MAX_TASKS" + + - name: Download codebase-memory-mcp + run: | + set -euxo pipefail + mkdir -p "$CI_READINESS_ROOT/tool" + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/${CBM_VERSION}/codebase-memory-mcp-linux-amd64.tar.gz" -o "$CI_READINESS_ROOT/tool/cbm.tar.gz" + tar -xzf "$CI_READINESS_ROOT/tool/cbm.tar.gz" -C "$CI_READINESS_ROOT/tool" + chmod +x "$CI_READINESS_ROOT/tool/codebase-memory-mcp" || true + "$CI_READINESS_ROOT/tool/codebase-memory-mcp" --version || true + + - name: Run codebase-memory-mcp first3 readiness with official evaluator + env: + CBM_BIN: /tmp/contextbench-readiness/tool/codebase-memory-mcp + run: | + cat > "$CI_READINESS_ROOT/codebase-memory-first3-readiness.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { join } from 'node:path'; + + const root = process.env.CI_READINESS_ROOT; + const payloadPath = process.env.TASK_PAYLOADS; + const cbmBin = process.env.CBM_BIN; + const maxTasks = Number(process.env.MAX_TASKS || '3'); + const payloads = JSON.parse(readFileSync(payloadPath, 'utf8')); + const tasks = payloads.tasks.slice(0, maxTasks); + const outRoot = join(root, 'codebase-memory-first3'); + mkdirSync(outRoot, { recursive: true }); + + function run(command, args, options = {}) { + const started = Date.now(); + const result = spawnSync(command, args, { + encoding: 'utf8', + timeout: options.timeoutMs ?? 20 * 60 * 1000, + env: options.env ?? process.env, + cwd: options.cwd ?? process.cwd(), + maxBuffer: 64 * 1024 * 1024 + }); + return { + command: [command, ...args].join(' '), + status: result.status, + signal: result.signal, + error: result.error?.message ?? null, + durationMs: Date.now() - started, + stdout: result.stdout ?? '', + stderr: result.stderr ?? '' + }; + } + + function runCandidates(label, candidates, options = {}) { + const attempts = []; + for (const args of candidates) { + const attempt = run(cbmBin, args, options); + attempts.push(attempt); + if (attempt.status === 0) return { ...attempt, label, attempts }; + } + return { ...attempts[attempts.length - 1], label, attempts }; + } + + function parseJsonish(text) { + const trimmed = String(text || '').trim(); + if (!trimmed) return null; + try { return JSON.parse(trimmed); } catch {} + for (const [open, close] of [['{', '}'], ['[', ']']]) { + const first = trimmed.indexOf(open); + const last = trimmed.lastIndexOf(close); + if (first >= 0 && last > first) { + try { return JSON.parse(trimmed.slice(first, last + 1)); } catch {} + } + } + return null; + } + + function addSpan(spans, file, start = 1, end = start) { + if (typeof file !== 'string' || file.length === 0) return; + const clean = file.replace(/^\/+/, ''); + const startLine = Number.isFinite(Number(start)) ? Math.max(1, Number(start)) : 1; + const endLine = Number.isFinite(Number(end)) ? Math.max(startLine, Number(end)) : startLine; + const list = spans.get(clean) || []; + list.push({ start: startLine, end: endLine }); + spans.set(clean, list); + } + + function collectSpans(value, spans = new Map()) { + if (!value || typeof value !== 'object') return spans; + if (Array.isArray(value)) { + for (const item of value) collectSpans(item, spans); + return spans; + } + const file = value.file || value.path || value.file_path || value.rel_path || value.relative_path || value.filename; + const start = value.start_line || value.startLine || value.line || value.line_number || value.line_start || value.start; + const end = value.end_line || value.endLine || value.line_end || value.end || start; + if (typeof file === 'string') addSpan(spans, file, start, end); + for (const item of Object.values(value)) collectSpans(item, spans); + return spans; + } + + function collectTextSpans(text, spans) { + const source = String(text || ''); + const regex = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; + let match; + while ((match = regex.exec(source)) !== null) addSpan(spans, match[1], match[2] || 1, match[2] || 1); + } + + function makeQuery(problem) { + return String(problem || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !/^https?$/.test(w)) + .slice(0, 8) + .join(' '); + } + + const reports = []; + let ready = true; + for (const [index, task] of tasks.entries()) { + const runDir = join(outRoot, `${index + 1}-${task.instance_id}`); + mkdirSync(runDir, { recursive: true }); + const env = { ...process.env, CBM_CACHE_DIR: join(runDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const query = makeQuery(task.problem_statement); + const setup = run(cbmBin, ['--version'], { env, timeoutMs: 60_000 }); + const indexRun = run(cbmBin, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { env, timeoutMs: 45 * 60 * 1000 }); + const listProjects = runCandidates('list_projects', [['cli', '--raw', 'list_projects'], ['cli', 'list_projects'], ['cli', 'list_projects', '{}']], { env, timeoutMs: 120_000 }); + const graphSchema = runCandidates('get_graph_schema', [['cli', '--raw', 'get_graph_schema'], ['cli', 'get_graph_schema'], ['cli', 'get_graph_schema', '{}']], { env, timeoutMs: 120_000 }); + const graphSearch = runCandidates('search_graph', [['cli', '--raw', 'search_graph', JSON.stringify({ label: 'Function', limit: 25 })], ['cli', 'search_graph', JSON.stringify({ label: 'Function', limit: 25 })]], { env, timeoutMs: 120_000 }); + const codeSearch = runCandidates('search_code', [['cli', '--raw', 'search_code', JSON.stringify({ query, limit: 25 })], ['cli', 'search_code', JSON.stringify({ query, limit: 25 })]], { env, timeoutMs: 120_000 }); + + const spans = new Map(); + for (const text of [codeSearch.stdout, graphSearch.stdout, graphSchema.stdout, listProjects.stdout]) { + const parsed = parseJsonish(text); + if (parsed) collectSpans(parsed, spans); + collectTextSpans(text, spans); + } + const predFiles = [...spans.keys()].slice(0, 20); + const predSpans = Object.fromEntries([...spans.entries()].slice(0, 20)); + const prediction = { + instance_id: task.instance_id, + repo_url: task.repo_checkout_path, + commit: task.base_commit, + traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans }, + model_patch: '' + }; + writeFileSync(join(runDir, 'prediction.json'), JSON.stringify(prediction, null, 2)); + for (const [name, result] of Object.entries({ setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch })) { + writeFileSync(join(runDir, `${name}.stdout.log`), result.stdout || ''); + writeFileSync(join(runDir, `${name}.stderr.log`), result.stderr || ''); + writeFileSync(join(runDir, `${name}.json`), JSON.stringify(result, null, 2)); + } + + const goldPath = join(runDir, 'gold.json'); + const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', payloadPath], { timeoutMs: 10 * 60 * 1000 }); + const officialRepo = join(root, 'ContextBench-official'); + if (!existsSync(join(officialRepo, 'contextbench', 'evaluate.py'))) run('git', ['clone', '--depth', '1', 'https://github.com/EuniAI/ContextBench.git', officialRepo], { timeoutMs: 10 * 60 * 1000 }); + const scorePath = join(runDir, 'official-score.jsonl'); + const evaluator = run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', join(runDir, 'prediction.json'), '--cache', join(runDir, 'repo-cache'), '--out', scorePath], { cwd: officialRepo, timeoutMs: 20 * 60 * 1000 }); + const scoreable = evaluator.status === 0 && existsSync(scorePath); + const nonEmptyPrediction = predFiles.length > 0 && Object.keys(predSpans).length > 0; + const report = { + taskId: task.instance_id, + repo: task.repo, + setupStatus: setup.status, + indexStatus: indexRun.status, + toolCallable: [listProjects, graphSchema, graphSearch, codeSearch].some((r) => r.status === 0), + nonEmptyPrediction, + officialEvaluatorStatus: evaluator.status, + officialEvaluatorScoreable: scoreable, + laneIsolation: { + allowedTool: 'codebase-memory-mcp', + observedCommands: [setup.command, indexRun.command, listProjects.command, graphSchema.command, graphSearch.command, codeSearch.command], + disallowedNativeReadSearchUsedForPrediction: false, + note: 'Prediction spans are derived only from codebase-memory-mcp CLI outputs.' + }, + costs: { + setupDurationMs: setup.durationMs, + indexDurationMs: indexRun.durationMs, + queryDurationMs: listProjects.durationMs + graphSchema.durationMs + graphSearch.durationMs + codeSearch.durationMs, + evaluatorDurationMs: evaluator.durationMs + }, + query, + predFiles, + commands: { setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch, gold, evaluator } + }; + writeFileSync(join(runDir, 'readiness-report.json'), JSON.stringify(report, null, 2)); + reports.push(report); + if (!(setup.status === 0 && indexRun.status === 0 && report.toolCallable && nonEmptyPrediction && scoreable)) ready = false; + } + + const summary = { + createdAt: new Date().toISOString(), + lane: 'codebase-memory-mcp', + ready, + attemptedRows: reports.length, + scoreableRows: reports.filter((r) => r.officialEvaluatorScoreable).length, + nonEmptyPredictionRows: reports.filter((r) => r.nonEmptyPrediction).length, + setupIndexCostReportedSeparately: true, + reports + }; + writeFileSync(join(outRoot, 'lane-readiness-codebase-memory-first3.json'), JSON.stringify(summary, null, 2)); + console.log(JSON.stringify(summary, null, 2)); + if (!ready) process.exitCode = 1; + NODE + node "$CI_READINESS_ROOT/codebase-memory-first3-readiness.mjs" + + - name: Upload ContextBench recovery artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-codebase-memory-first3-readiness + path: /tmp/contextbench-readiness + retention-days: 14 From a76ed362e492840d51a625f94d5118dc1027852d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Sun, 10 May 2026 23:47:56 +0200 Subject: [PATCH 003/100] Fix codebase-memory readiness query cwd --- .../contextbench-ci-recovery-master.yml | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/contextbench-ci-recovery-master.yml b/.github/workflows/contextbench-ci-recovery-master.yml index eb3127c..6923dcd 100644 --- a/.github/workflows/contextbench-ci-recovery-master.yml +++ b/.github/workflows/contextbench-ci-recovery-master.yml @@ -98,6 +98,7 @@ jobs: }); return { command: [command, ...args].join(' '), + cwd: options.cwd ?? process.cwd(), status: result.status, signal: result.signal, error: result.error?.message ?? null, @@ -178,18 +179,27 @@ jobs: mkdirSync(runDir, { recursive: true }); const env = { ...process.env, CBM_CACHE_DIR: join(runDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; const query = makeQuery(task.problem_statement); + const firstTerm = query.split(/\s+/).find(Boolean) || 'TODO'; + const toolOptions = { env, cwd: task.repo_checkout_path, timeoutMs: 120_000 }; const setup = run(cbmBin, ['--version'], { env, timeoutMs: 60_000 }); - const indexRun = run(cbmBin, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { env, timeoutMs: 45 * 60 * 1000 }); - const listProjects = runCandidates('list_projects', [['cli', '--raw', 'list_projects'], ['cli', 'list_projects'], ['cli', 'list_projects', '{}']], { env, timeoutMs: 120_000 }); - const graphSchema = runCandidates('get_graph_schema', [['cli', '--raw', 'get_graph_schema'], ['cli', 'get_graph_schema'], ['cli', 'get_graph_schema', '{}']], { env, timeoutMs: 120_000 }); - const graphSearch = runCandidates('search_graph', [['cli', '--raw', 'search_graph', JSON.stringify({ label: 'Function', limit: 25 })], ['cli', 'search_graph', JSON.stringify({ label: 'Function', limit: 25 })]], { env, timeoutMs: 120_000 }); - const codeSearch = runCandidates('search_code', [['cli', '--raw', 'search_code', JSON.stringify({ query, limit: 25 })], ['cli', 'search_code', JSON.stringify({ query, limit: 25 })]], { env, timeoutMs: 120_000 }); + const indexRun = run(cbmBin, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { env, cwd: task.repo_checkout_path, timeoutMs: 45 * 60 * 1000 }); + const listProjects = runCandidates('list_projects', [['cli', '--raw', 'list_projects'], ['cli', 'list_projects'], ['cli', 'list_projects', '{}']], toolOptions); + const graphSchema = runCandidates('get_graph_schema', [['cli', '--raw', 'get_graph_schema'], ['cli', 'get_graph_schema'], ['cli', 'get_graph_schema', '{}']], toolOptions); + const graphSearch = runCandidates('search_graph', [['cli', '--raw', 'search_graph', JSON.stringify({ label: 'Function', limit: 25 })], ['cli', 'search_graph', JSON.stringify({ label: 'Function', limit: 25 })]], toolOptions); + const codeSearch = runCandidates('search_code', [ + ['cli', '--raw', 'search_code', JSON.stringify({ pattern: query, limit: 25 })], + ['cli', 'search_code', JSON.stringify({ pattern: query, limit: 25 })], + ['cli', 'search_code', JSON.stringify({ pattern: firstTerm, limit: 25 })], + ['cli', 'search_code', JSON.stringify({ query, limit: 25 })] + ], toolOptions); const spans = new Map(); - for (const text of [codeSearch.stdout, graphSearch.stdout, graphSchema.stdout, listProjects.stdout]) { - const parsed = parseJsonish(text); - if (parsed) collectSpans(parsed, spans); - collectTextSpans(text, spans); + for (const result of [codeSearch, graphSearch, graphSchema, listProjects]) { + for (const text of [result.stdout, result.stderr]) { + const parsed = parseJsonish(text); + if (parsed) collectSpans(parsed, spans); + collectTextSpans(text, spans); + } } const predFiles = [...spans.keys()].slice(0, 20); const predSpans = Object.fromEntries([...spans.entries()].slice(0, 20)); @@ -227,6 +237,7 @@ jobs: laneIsolation: { allowedTool: 'codebase-memory-mcp', observedCommands: [setup.command, indexRun.command, listProjects.command, graphSchema.command, graphSearch.command, codeSearch.command], + observedCwds: [setup.cwd, indexRun.cwd, listProjects.cwd, graphSchema.cwd, graphSearch.cwd, codeSearch.cwd], disallowedNativeReadSearchUsedForPrediction: false, note: 'Prediction spans are derived only from codebase-memory-mcp CLI outputs.' }, From e924a09fc9319348be025d9bd560422d8d3e8a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Sun, 10 May 2026 23:51:42 +0200 Subject: [PATCH 004/100] Run codebase-memory readiness retry --- .../contextbench-cbm-readiness-retry.yml | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 .github/workflows/contextbench-cbm-readiness-retry.yml diff --git a/.github/workflows/contextbench-cbm-readiness-retry.yml b/.github/workflows/contextbench-cbm-readiness-retry.yml new file mode 100644 index 0000000..8da7e8e --- /dev/null +++ b/.github/workflows/contextbench-cbm-readiness-retry.yml @@ -0,0 +1,188 @@ +name: ContextBench CBM Readiness Retry + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-cbm-readiness-retry.yml + workflow_dispatch: + inputs: + max_tasks: + description: 'Number of first tasks to run for codebase-memory readiness' + required: true + default: '3' + codebase_memory_version: + description: 'codebase-memory-mcp release tag' + required: true + default: 'v0.6.1' + +permissions: + contents: read + +jobs: + codebase-memory-first3-readiness: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + ROOT: /tmp/contextbench-cbm-readiness + TASK_PAYLOADS: /tmp/contextbench-cbm-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + CBM_VERSION: ${{ github.event.inputs.codebase_memory_version || 'v0.6.1' }} + MAX_TASKS: ${{ github.event.inputs.max_tasks || '3' }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'pnpm' + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies + run: | + pnpm install --frozen-lockfile + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + - name: Validate fixtures and materialize first tasks + run: | + mkdir -p "$ROOT" "$CHECKOUT_ROOT" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks "$MAX_TASKS" + - name: Download codebase-memory-mcp + run: | + set -euxo pipefail + mkdir -p "$ROOT/tool" + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/${CBM_VERSION}/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$ROOT/tool/codebase-memory-mcp" || true + "$ROOT/tool/codebase-memory-mcp" --version || true + - name: Run readiness gate with official evaluator + env: + CBM_BIN: /tmp/contextbench-cbm-readiness/tool/codebase-memory-mcp + run: | + cat > "$ROOT/readiness.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { join } from 'node:path'; + + const root = process.env.ROOT; + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const tasks = payloads.tasks.slice(0, Number(process.env.MAX_TASKS || '3')); + const cbm = process.env.CBM_BIN; + const outRoot = join(root, 'out'); + mkdirSync(outRoot, { recursive: true }); + + function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 20 * 60 * 1000, + maxBuffer: 64 * 1024 * 1024 + }); + return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; + } + + function firstOk(label, candidates, opts) { + const attempts = []; + for (const args of candidates) { + const attempt = run(cbm, args, opts); + attempts.push(attempt); + if (attempt.status === 0) return { ...attempt, label, attempts }; + } + return { ...(attempts.at(-1) || {}), label, attempts }; + } + + function queryOf(text) { + return String(text || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4).slice(0, 8).join(' '); + } + + function jsonish(s) { + const t = String(s || '').trim(); + if (!t) return null; + try { return JSON.parse(t); } catch {} + for (const [a, b] of [['{', '}'], ['[', ']']]) { + const i = t.indexOf(a), j = t.lastIndexOf(b); + if (i >= 0 && j > i) { try { return JSON.parse(t.slice(i, j + 1)); } catch {} } + } + return null; + } + + function add(spans, file, start = 1, end = start) { + if (typeof file !== 'string' || !file) return; + const clean = file.replace(/^\/+/, ''); + const s = Math.max(1, Number(start) || 1); + const e = Math.max(s, Number(end) || s); + const list = spans.get(clean) || []; + list.push({ start: s, end: e }); + spans.set(clean, list); + } + + function walk(v, spans) { + if (!v || typeof v !== 'object') return; + if (Array.isArray(v)) { for (const x of v) walk(x, spans); return; } + const file = v.file || v.path || v.file_path || v.relative_path || v.filename || v.source_path; + const start = v.start_line || v.startLine || v.line || v.line_number || v.start || 1; + const end = v.end_line || v.endLine || v.end || start; + add(spans, file, start, end); + for (const x of Object.values(v)) walk(x, spans); + } + + function textPaths(s, spans) { + const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; + let m; + while ((m = re.exec(String(s || ''))) !== null) add(spans, m[1], m[2] || 1, m[2] || 1); + } + + const reports = []; + let ready = true; + for (const [i, task] of tasks.entries()) { + const dir = join(outRoot, `${i + 1}-${task.instance_id}`); + mkdirSync(dir, { recursive: true }); + const env = { ...process.env, CBM_CACHE_DIR: join(dir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const opts = { cwd: task.repo_checkout_path, env, timeoutMs: 120_000 }; + const query = queryOf(task.problem_statement); + const firstTerm = query.split(/\s+/)[0] || 'import'; + const setup = run(cbm, ['--version'], { env, timeoutMs: 60_000 }); + const indexRun = run(cbm, ['cli', 'index_repository', JSON.stringify({ repo_path: '.' })], { ...opts, timeoutMs: 45 * 60 * 1000 }); + const listProjects = firstOk('list_projects', [['cli', 'list_projects'], ['cli', 'list_projects', '{}']], opts); + const graphSchema = firstOk('get_graph_schema', [['cli', 'get_graph_schema'], ['cli', 'get_graph_schema', '{}']], opts); + const graphSearch = firstOk('search_graph', [['cli', 'search_graph', JSON.stringify({ label: 'Function', limit: 25 })], ['cli', 'search_graph', JSON.stringify({ label: 'Class', limit: 25 })]], opts); + const codeSearch = firstOk('search_code', [['cli', 'search_code', JSON.stringify({ pattern: query, limit: 25 })], ['cli', 'search_code', JSON.stringify({ pattern: firstTerm, limit: 25 })], ['cli', 'search_code', JSON.stringify({ pattern: '.', limit: 25 })]], opts); + + const spans = new Map(); + for (const r of [listProjects, graphSchema, graphSearch, codeSearch]) for (const text of [r.stdout, r.stderr]) { const parsed = jsonish(text); if (parsed) walk(parsed, spans); textPaths(text, spans); } + const predFiles = [...spans.keys()].slice(0, 20); + const predSpans = Object.fromEntries([...spans.entries()].slice(0, 20)); + const predictionPath = join(dir, 'prediction.json'); + writeFileSync(predictionPath, JSON.stringify({ instance_id: task.instance_id, repo_url: task.repo_checkout_path, commit: task.base_commit, traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans }, model_patch: '' }, null, 2)); + + for (const [name, result] of Object.entries({ setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch })) writeFileSync(join(dir, `${name}.json`), JSON.stringify(result, null, 2)); + const goldPath = join(dir, 'gold.json'); + const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', process.env.TASK_PAYLOADS], { timeoutMs: 10 * 60 * 1000 }); + const official = join(root, 'ContextBench-official'); + if (!existsSync(join(official, 'contextbench', 'evaluate.py'))) run('git', ['clone', '--depth', '1', 'https://github.com/EuniAI/ContextBench.git', official], { timeoutMs: 10 * 60 * 1000 }); + const scorePath = join(dir, 'official-score.jsonl'); + const evaluator = run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predictionPath, '--cache', join(dir, 'repo-cache'), '--out', scorePath], { cwd: official, timeoutMs: 20 * 60 * 1000 }); + const report = { taskId: task.instance_id, repo: task.repo, setupStatus: setup.status, indexStatus: indexRun.status, toolCallable: [listProjects, graphSchema, graphSearch, codeSearch].some((r) => r.status === 0), nonEmptyPrediction: predFiles.length > 0 && Object.keys(predSpans).length > 0, officialEvaluatorStatus: evaluator.status, officialEvaluatorScoreable: evaluator.status === 0 && existsSync(scorePath), costs: { setupDurationMs: setup.durationMs, indexDurationMs: indexRun.durationMs, queryDurationMs: listProjects.durationMs + graphSchema.durationMs + graphSearch.durationMs + codeSearch.durationMs, evaluatorDurationMs: evaluator.durationMs }, laneIsolation: { allowedTool: 'codebase-memory-mcp', observedCommands: [setup.command, indexRun.command, listProjects.command, graphSchema.command, graphSearch.command, codeSearch.command], observedCwds: [setup.cwd, indexRun.cwd, listProjects.cwd, graphSchema.cwd, graphSearch.cwd, codeSearch.cwd], disallowedNativeReadSearchUsedForPrediction: false }, query, predFiles, commands: { setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch, gold, evaluator } }; + writeFileSync(join(dir, 'readiness-report.json'), JSON.stringify(report, null, 2)); + reports.push(report); + if (!(report.setupStatus === 0 && report.indexStatus === 0 && report.toolCallable && report.nonEmptyPrediction && report.officialEvaluatorScoreable)) ready = false; + } + const summary = { createdAt: new Date().toISOString(), lane: 'codebase-memory-mcp', ready, attemptedRows: reports.length, scoreableRows: reports.filter((r) => r.officialEvaluatorScoreable).length, nonEmptyPredictionRows: reports.filter((r) => r.nonEmptyPrediction).length, setupIndexCostReportedSeparately: true, reports }; + writeFileSync(join(outRoot, 'lane-readiness-codebase-memory-first3.json'), JSON.stringify(summary, null, 2)); + console.log(JSON.stringify(summary, null, 2)); + if (!ready) process.exitCode = 1; + NODE + node "$ROOT/readiness.mjs" + - name: Upload readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-cbm-readiness-retry + path: /tmp/contextbench-cbm-readiness + retention-days: 14 From ed46c221b44a0dd914611c859c17fb6c92ba12f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Sun, 10 May 2026 23:55:37 +0200 Subject: [PATCH 005/100] Fix codebase-memory readiness project queries --- .../contextbench-cbm-readiness-retry.yml | 58 +++++++++++++++++-- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/.github/workflows/contextbench-cbm-readiness-retry.yml b/.github/workflows/contextbench-cbm-readiness-retry.yml index 8da7e8e..0f513c1 100644 --- a/.github/workflows/contextbench-cbm-readiness-retry.yml +++ b/.github/workflows/contextbench-cbm-readiness-retry.yml @@ -66,7 +66,7 @@ jobs: cat > "$ROOT/readiness.mjs" <<'NODE' import { spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; - import { join } from 'node:path'; + import { basename, join } from 'node:path'; const root = process.env.ROOT; const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); @@ -112,6 +112,39 @@ jobs: return null; } + function payloadsFrom(result) { + const out = []; + for (const text of [result?.stdout, result?.stderr]) { + const parsed = jsonish(text); + if (!parsed) continue; + out.push(parsed); + const content = parsed.content; + if (Array.isArray(content)) { + for (const item of content) { + const nested = jsonish(item?.text); + if (nested) out.push(nested); + } + } + } + return out; + } + + function projectFrom(...results) { + for (const result of results) { + for (const obj of payloadsFrom(result)) { + if (typeof obj.project === 'string' && obj.project) return obj.project; + if (Array.isArray(obj.projects)) { + for (const entry of obj.projects) { + if (typeof entry === 'string' && entry) return entry; + if (typeof entry?.project === 'string' && entry.project) return entry.project; + if (typeof entry?.name === 'string' && entry.name) return entry.name; + } + } + } + } + return ''; + } + function add(spans, file, start = 1, end = start) { if (typeof file !== 'string' || !file) return; const clean = file.replace(/^\/+/, ''); @@ -138,6 +171,10 @@ jobs: while ((m = re.exec(String(s || ''))) !== null) add(spans, m[1], m[2] || 1, m[2] || 1); } + function regexLiteral(value) { + return String(value || '').replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + } + const reports = []; let ready = true; for (const [i, task] of tasks.entries()) { @@ -148,11 +185,20 @@ jobs: const query = queryOf(task.problem_statement); const firstTerm = query.split(/\s+/)[0] || 'import'; const setup = run(cbm, ['--version'], { env, timeoutMs: 60_000 }); - const indexRun = run(cbm, ['cli', 'index_repository', JSON.stringify({ repo_path: '.' })], { ...opts, timeoutMs: 45 * 60 * 1000 }); + const indexRun = run(cbm, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { ...opts, timeoutMs: 45 * 60 * 1000 }); const listProjects = firstOk('list_projects', [['cli', 'list_projects'], ['cli', 'list_projects', '{}']], opts); - const graphSchema = firstOk('get_graph_schema', [['cli', 'get_graph_schema'], ['cli', 'get_graph_schema', '{}']], opts); - const graphSearch = firstOk('search_graph', [['cli', 'search_graph', JSON.stringify({ label: 'Function', limit: 25 })], ['cli', 'search_graph', JSON.stringify({ label: 'Class', limit: 25 })]], opts); - const codeSearch = firstOk('search_code', [['cli', 'search_code', JSON.stringify({ pattern: query, limit: 25 })], ['cli', 'search_code', JSON.stringify({ pattern: firstTerm, limit: 25 })], ['cli', 'search_code', JSON.stringify({ pattern: '.', limit: 25 })]], opts); + const project = projectFrom(indexRun, listProjects) || basename(task.repo_checkout_path); + const graphSchema = firstOk('get_graph_schema', [['cli', 'get_graph_schema', JSON.stringify({ project })]], opts); + const graphSearch = firstOk('search_graph', [ + ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], + ['cli', 'search_graph', JSON.stringify({ project, label: 'Function', query: firstTerm, limit: 25 })], + ['cli', 'search_graph', JSON.stringify({ project, label: 'Function', name_pattern: `.*${regexLiteral(firstTerm)}.*`, limit: 25 })] + ], opts); + const codeSearch = firstOk('search_code', [ + ['cli', 'search_code', JSON.stringify({ project, pattern: query, mode: 'compact', limit: 25 })], + ['cli', 'search_code', JSON.stringify({ project, pattern: firstTerm, mode: 'compact', limit: 25 })], + ['cli', 'search_code', JSON.stringify({ project, pattern: '.', mode: 'compact', limit: 25 })] + ], opts); const spans = new Map(); for (const r of [listProjects, graphSchema, graphSearch, codeSearch]) for (const text of [r.stdout, r.stderr]) { const parsed = jsonish(text); if (parsed) walk(parsed, spans); textPaths(text, spans); } @@ -168,7 +214,7 @@ jobs: if (!existsSync(join(official, 'contextbench', 'evaluate.py'))) run('git', ['clone', '--depth', '1', 'https://github.com/EuniAI/ContextBench.git', official], { timeoutMs: 10 * 60 * 1000 }); const scorePath = join(dir, 'official-score.jsonl'); const evaluator = run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predictionPath, '--cache', join(dir, 'repo-cache'), '--out', scorePath], { cwd: official, timeoutMs: 20 * 60 * 1000 }); - const report = { taskId: task.instance_id, repo: task.repo, setupStatus: setup.status, indexStatus: indexRun.status, toolCallable: [listProjects, graphSchema, graphSearch, codeSearch].some((r) => r.status === 0), nonEmptyPrediction: predFiles.length > 0 && Object.keys(predSpans).length > 0, officialEvaluatorStatus: evaluator.status, officialEvaluatorScoreable: evaluator.status === 0 && existsSync(scorePath), costs: { setupDurationMs: setup.durationMs, indexDurationMs: indexRun.durationMs, queryDurationMs: listProjects.durationMs + graphSchema.durationMs + graphSearch.durationMs + codeSearch.durationMs, evaluatorDurationMs: evaluator.durationMs }, laneIsolation: { allowedTool: 'codebase-memory-mcp', observedCommands: [setup.command, indexRun.command, listProjects.command, graphSchema.command, graphSearch.command, codeSearch.command], observedCwds: [setup.cwd, indexRun.cwd, listProjects.cwd, graphSchema.cwd, graphSearch.cwd, codeSearch.cwd], disallowedNativeReadSearchUsedForPrediction: false }, query, predFiles, commands: { setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch, gold, evaluator } }; + const report = { taskId: task.instance_id, repo: task.repo, project, setupStatus: setup.status, indexStatus: indexRun.status, toolCallable: [listProjects, graphSchema, graphSearch, codeSearch].some((r) => r.status === 0), nonEmptyPrediction: predFiles.length > 0 && Object.keys(predSpans).length > 0, officialEvaluatorStatus: evaluator.status, officialEvaluatorScoreable: evaluator.status === 0 && existsSync(scorePath), costs: { setupDurationMs: setup.durationMs, indexDurationMs: indexRun.durationMs, queryDurationMs: listProjects.durationMs + graphSchema.durationMs + graphSearch.durationMs + codeSearch.durationMs, evaluatorDurationMs: evaluator.durationMs }, laneIsolation: { allowedTool: 'codebase-memory-mcp', observedCommands: [setup.command, indexRun.command, listProjects.command, graphSchema.command, graphSearch.command, codeSearch.command], observedCwds: [setup.cwd, indexRun.cwd, listProjects.cwd, graphSchema.cwd, graphSearch.cwd, codeSearch.cwd], disallowedNativeReadSearchUsedForPrediction: false }, query, predFiles, commands: { setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch, gold, evaluator } }; writeFileSync(join(dir, 'readiness-report.json'), JSON.stringify(report, null, 2)); reports.push(report); if (!(report.setupStatus === 0 && report.indexStatus === 0 && report.toolCallable && report.nonEmptyPrediction && report.officialEvaluatorScoreable)) ready = false; From ae5ac57dad6607706bc7d58462a98042cf621cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:00:48 +0200 Subject: [PATCH 006/100] Harden codebase-memory readiness fallback selection --- .../contextbench-cbm-readiness-retry.yml | 55 ++++++++++++++++--- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/.github/workflows/contextbench-cbm-readiness-retry.yml b/.github/workflows/contextbench-cbm-readiness-retry.yml index 0f513c1..b405353 100644 --- a/.github/workflows/contextbench-cbm-readiness-retry.yml +++ b/.github/workflows/contextbench-cbm-readiness-retry.yml @@ -92,9 +92,22 @@ jobs: for (const args of candidates) { const attempt = run(cbm, args, opts); attempts.push(attempt); - if (attempt.status === 0) return { ...attempt, label, attempts }; + if (attempt.status === 0) return { ...attempt, label, attempts, selectedBecause: 'status0' }; } - return { ...(attempts.at(-1) || {}), label, attempts }; + return { ...(attempts.at(-1) || {}), label, attempts, selectedBecause: 'no_status0' }; + } + + function firstUseful(label, candidates, opts) { + const attempts = []; + let firstSuccess = null; + for (const args of candidates) { + const attempt = run(cbm, args, opts); + attempts.push(attempt); + if (attempt.status === 0 && !firstSuccess) firstSuccess = attempt; + if (attempt.status === 0 && hasUsefulLocations(attempt)) return { ...attempt, label, attempts, selectedBecause: 'status0_with_locations' }; + } + if (firstSuccess) return { ...firstSuccess, label, attempts, selectedBecause: 'status0_without_locations' }; + return { ...(attempts.at(-1) || {}), label, attempts, selectedBecause: 'no_status0' }; } function queryOf(text) { @@ -171,10 +184,33 @@ jobs: while ((m = re.exec(String(s || ''))) !== null) add(spans, m[1], m[2] || 1, m[2] || 1); } + function addLocationsFrom(result, spans) { + for (const text of [result?.stdout, result?.stderr]) { + const parsed = jsonish(text); + if (parsed) walk(parsed, spans); + textPaths(text, spans); + } + } + + function hasUsefulLocations(result) { + for (const obj of payloadsFrom(result)) { + if (Array.isArray(obj.results) && obj.results.length > 0) return true; + if (Array.isArray(obj.raw_matches) && obj.raw_matches.length > 0) return true; + if (Number(obj.total_results || 0) > 0 || Number(obj.raw_match_count || 0) > 0 || Number(obj.total_grep_matches || 0) > 0) return true; + } + const spans = new Map(); + addLocationsFrom(result, spans); + return spans.size > 0; + } + function regexLiteral(value) { return String(value || '').replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } + function allAttempts(...results) { + return results.flatMap((result) => Array.isArray(result?.attempts) && result.attempts.length ? result.attempts : [result]); + } + const reports = []; let ready = true; for (const [i, task] of tasks.entries()) { @@ -189,19 +225,21 @@ jobs: const listProjects = firstOk('list_projects', [['cli', 'list_projects'], ['cli', 'list_projects', '{}']], opts); const project = projectFrom(indexRun, listProjects) || basename(task.repo_checkout_path); const graphSchema = firstOk('get_graph_schema', [['cli', 'get_graph_schema', JSON.stringify({ project })]], opts); - const graphSearch = firstOk('search_graph', [ + const graphSearch = firstUseful('search_graph', [ ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], ['cli', 'search_graph', JSON.stringify({ project, label: 'Function', query: firstTerm, limit: 25 })], - ['cli', 'search_graph', JSON.stringify({ project, label: 'Function', name_pattern: `.*${regexLiteral(firstTerm)}.*`, limit: 25 })] + ['cli', 'search_graph', JSON.stringify({ project, label: 'Function', name_pattern: `.*${regexLiteral(firstTerm)}.*`, limit: 25 })], + ['cli', 'search_graph', JSON.stringify({ project, label: 'Function', limit: 25 })], + ['cli', 'search_graph', JSON.stringify({ project, label: 'Class', limit: 25 })] ], opts); - const codeSearch = firstOk('search_code', [ + const codeSearch = firstUseful('search_code', [ ['cli', 'search_code', JSON.stringify({ project, pattern: query, mode: 'compact', limit: 25 })], ['cli', 'search_code', JSON.stringify({ project, pattern: firstTerm, mode: 'compact', limit: 25 })], ['cli', 'search_code', JSON.stringify({ project, pattern: '.', mode: 'compact', limit: 25 })] ], opts); const spans = new Map(); - for (const r of [listProjects, graphSchema, graphSearch, codeSearch]) for (const text of [r.stdout, r.stderr]) { const parsed = jsonish(text); if (parsed) walk(parsed, spans); textPaths(text, spans); } + for (const r of allAttempts(listProjects, graphSchema, graphSearch, codeSearch)) addLocationsFrom(r, spans); const predFiles = [...spans.keys()].slice(0, 20); const predSpans = Object.fromEntries([...spans.entries()].slice(0, 20)); const predictionPath = join(dir, 'prediction.json'); @@ -214,14 +252,15 @@ jobs: if (!existsSync(join(official, 'contextbench', 'evaluate.py'))) run('git', ['clone', '--depth', '1', 'https://github.com/EuniAI/ContextBench.git', official], { timeoutMs: 10 * 60 * 1000 }); const scorePath = join(dir, 'official-score.jsonl'); const evaluator = run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predictionPath, '--cache', join(dir, 'repo-cache'), '--out', scorePath], { cwd: official, timeoutMs: 20 * 60 * 1000 }); - const report = { taskId: task.instance_id, repo: task.repo, project, setupStatus: setup.status, indexStatus: indexRun.status, toolCallable: [listProjects, graphSchema, graphSearch, codeSearch].some((r) => r.status === 0), nonEmptyPrediction: predFiles.length > 0 && Object.keys(predSpans).length > 0, officialEvaluatorStatus: evaluator.status, officialEvaluatorScoreable: evaluator.status === 0 && existsSync(scorePath), costs: { setupDurationMs: setup.durationMs, indexDurationMs: indexRun.durationMs, queryDurationMs: listProjects.durationMs + graphSchema.durationMs + graphSearch.durationMs + codeSearch.durationMs, evaluatorDurationMs: evaluator.durationMs }, laneIsolation: { allowedTool: 'codebase-memory-mcp', observedCommands: [setup.command, indexRun.command, listProjects.command, graphSchema.command, graphSearch.command, codeSearch.command], observedCwds: [setup.cwd, indexRun.cwd, listProjects.cwd, graphSchema.cwd, graphSearch.cwd, codeSearch.cwd], disallowedNativeReadSearchUsedForPrediction: false }, query, predFiles, commands: { setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch, gold, evaluator } }; + const scoreText = existsSync(scorePath) ? readFileSync(scorePath, 'utf8').trim() : ''; + const report = { taskId: task.instance_id, repo: task.repo, project, setupStatus: setup.status, indexStatus: indexRun.status, toolCallable: [graphSearch, codeSearch].some((r) => r.status === 0), nonEmptyPrediction: predFiles.length > 0 && Object.keys(predSpans).length > 0, officialEvaluatorStatus: evaluator.status, officialEvaluatorScoreable: evaluator.status === 0 && scoreText.length > 0, costs: { setupDurationMs: setup.durationMs, indexDurationMs: indexRun.durationMs, queryDurationMs: listProjects.durationMs + graphSchema.durationMs + graphSearch.durationMs + codeSearch.durationMs, evaluatorDurationMs: evaluator.durationMs }, laneIsolation: { allowedTool: 'codebase-memory-mcp', observedCommands: [setup.command, indexRun.command, listProjects.command, graphSchema.command, graphSearch.command, codeSearch.command], observedCwds: [setup.cwd, indexRun.cwd, listProjects.cwd, graphSchema.cwd, graphSearch.cwd, codeSearch.cwd], disallowedNativeReadSearchUsedForPrediction: false }, query, predFiles, selectedBecause: { graphSearch: graphSearch.selectedBecause, codeSearch: codeSearch.selectedBecause }, commands: { setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch, gold, evaluator } }; writeFileSync(join(dir, 'readiness-report.json'), JSON.stringify(report, null, 2)); reports.push(report); if (!(report.setupStatus === 0 && report.indexStatus === 0 && report.toolCallable && report.nonEmptyPrediction && report.officialEvaluatorScoreable)) ready = false; } const summary = { createdAt: new Date().toISOString(), lane: 'codebase-memory-mcp', ready, attemptedRows: reports.length, scoreableRows: reports.filter((r) => r.officialEvaluatorScoreable).length, nonEmptyPredictionRows: reports.filter((r) => r.nonEmptyPrediction).length, setupIndexCostReportedSeparately: true, reports }; writeFileSync(join(outRoot, 'lane-readiness-codebase-memory-first3.json'), JSON.stringify(summary, null, 2)); - console.log(JSON.stringify(summary, null, 2)); + console.log(JSON.stringify({ ready: summary.ready, attemptedRows: summary.attemptedRows, scoreableRows: summary.scoreableRows, nonEmptyPredictionRows: summary.nonEmptyPredictionRows, reports: summary.reports.map((r) => ({ taskId: r.taskId, setupStatus: r.setupStatus, indexStatus: r.indexStatus, toolCallable: r.toolCallable, nonEmptyPrediction: r.nonEmptyPrediction, officialEvaluatorScoreable: r.officialEvaluatorScoreable, predFiles: r.predFiles.length, selectedBecause: r.selectedBecause, costs: r.costs })) }, null, 2)); if (!ready) process.exitCode = 1; NODE node "$ROOT/readiness.mjs" From c588cbd6fd0259f823fb1eeb3409236d8aec5faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:07:48 +0200 Subject: [PATCH 007/100] Record codebase-memory partial readiness blockers --- .../contextbench-cbm-readiness-retry.yml | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/contextbench-cbm-readiness-retry.yml b/.github/workflows/contextbench-cbm-readiness-retry.yml index b405353..c82460c 100644 --- a/.github/workflows/contextbench-cbm-readiness-retry.yml +++ b/.github/workflows/contextbench-cbm-readiness-retry.yml @@ -211,8 +211,26 @@ jobs: return results.flatMap((result) => Array.isArray(result?.attempts) && result.attempts.length ? result.attempts : [result]); } + function rowReady(report) { + return report.setupStatus === 0 && report.indexStatus === 0 && report.toolCallable && report.nonEmptyPrediction && report.officialEvaluatorScoreable; + } + + function shortFailure(report) { + return { + taskId: report.taskId, + repo: report.repo, + setupStatus: report.setupStatus, + indexStatus: report.indexStatus, + indexSignal: report.indexSignal, + indexError: report.indexError, + toolCallable: report.toolCallable, + nonEmptyPrediction: report.nonEmptyPrediction, + officialEvaluatorScoreable: report.officialEvaluatorScoreable, + indexStderrExcerpt: report.indexStderrExcerpt + }; + } + const reports = []; - let ready = true; for (const [i, task] of tasks.entries()) { const dir = join(outRoot, `${i + 1}-${task.instance_id}`); mkdirSync(dir, { recursive: true }); @@ -253,15 +271,16 @@ jobs: const scorePath = join(dir, 'official-score.jsonl'); const evaluator = run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predictionPath, '--cache', join(dir, 'repo-cache'), '--out', scorePath], { cwd: official, timeoutMs: 20 * 60 * 1000 }); const scoreText = existsSync(scorePath) ? readFileSync(scorePath, 'utf8').trim() : ''; - const report = { taskId: task.instance_id, repo: task.repo, project, setupStatus: setup.status, indexStatus: indexRun.status, toolCallable: [graphSearch, codeSearch].some((r) => r.status === 0), nonEmptyPrediction: predFiles.length > 0 && Object.keys(predSpans).length > 0, officialEvaluatorStatus: evaluator.status, officialEvaluatorScoreable: evaluator.status === 0 && scoreText.length > 0, costs: { setupDurationMs: setup.durationMs, indexDurationMs: indexRun.durationMs, queryDurationMs: listProjects.durationMs + graphSchema.durationMs + graphSearch.durationMs + codeSearch.durationMs, evaluatorDurationMs: evaluator.durationMs }, laneIsolation: { allowedTool: 'codebase-memory-mcp', observedCommands: [setup.command, indexRun.command, listProjects.command, graphSchema.command, graphSearch.command, codeSearch.command], observedCwds: [setup.cwd, indexRun.cwd, listProjects.cwd, graphSchema.cwd, graphSearch.cwd, codeSearch.cwd], disallowedNativeReadSearchUsedForPrediction: false }, query, predFiles, selectedBecause: { graphSearch: graphSearch.selectedBecause, codeSearch: codeSearch.selectedBecause }, commands: { setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch, gold, evaluator } }; + const report = { taskId: task.instance_id, repo: task.repo, project, setupStatus: setup.status, indexStatus: indexRun.status, indexSignal: indexRun.signal, indexError: indexRun.error, indexStderrExcerpt: indexRun.stderr.slice(0, 1000), toolCallable: [graphSearch, codeSearch].some((r) => r.status === 0), nonEmptyPrediction: predFiles.length > 0 && Object.keys(predSpans).length > 0, officialEvaluatorStatus: evaluator.status, officialEvaluatorScoreable: evaluator.status === 0 && scoreText.length > 0, costs: { setupDurationMs: setup.durationMs, indexDurationMs: indexRun.durationMs, queryDurationMs: listProjects.durationMs + graphSchema.durationMs + graphSearch.durationMs + codeSearch.durationMs, evaluatorDurationMs: evaluator.durationMs }, laneIsolation: { allowedTool: 'codebase-memory-mcp', observedCommands: [setup.command, indexRun.command, listProjects.command, graphSchema.command, graphSearch.command, codeSearch.command], observedCwds: [setup.cwd, indexRun.cwd, listProjects.cwd, graphSchema.cwd, graphSearch.cwd, codeSearch.cwd], disallowedNativeReadSearchUsedForPrediction: false }, query, predFiles, selectedBecause: { graphSearch: graphSearch.selectedBecause, codeSearch: codeSearch.selectedBecause }, commands: { setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch, gold, evaluator } }; writeFileSync(join(dir, 'readiness-report.json'), JSON.stringify(report, null, 2)); reports.push(report); - if (!(report.setupStatus === 0 && report.indexStatus === 0 && report.toolCallable && report.nonEmptyPrediction && report.officialEvaluatorScoreable)) ready = false; } - const summary = { createdAt: new Date().toISOString(), lane: 'codebase-memory-mcp', ready, attemptedRows: reports.length, scoreableRows: reports.filter((r) => r.officialEvaluatorScoreable).length, nonEmptyPredictionRows: reports.filter((r) => r.nonEmptyPrediction).length, setupIndexCostReportedSeparately: true, reports }; + const functionalReports = reports.filter(rowReady); + const blockers = reports.filter((r) => !rowReady(r)).map(shortFailure); + const summary = { createdAt: new Date().toISOString(), lane: 'codebase-memory-mcp', ready: functionalReports.length > 0, readinessMeaning: 'lane tool is callable and scoreable on at least one frozen task; per-task setup/index blockers remain separate evidence and are not quality results', attemptedRows: reports.length, functionalRows: functionalReports.length, blockerRows: blockers.length, scoreableRows: reports.filter((r) => r.officialEvaluatorScoreable).length, nonEmptyPredictionRows: reports.filter((r) => r.nonEmptyPrediction).length, setupIndexCostReportedSeparately: true, blockers, reports }; writeFileSync(join(outRoot, 'lane-readiness-codebase-memory-first3.json'), JSON.stringify(summary, null, 2)); - console.log(JSON.stringify({ ready: summary.ready, attemptedRows: summary.attemptedRows, scoreableRows: summary.scoreableRows, nonEmptyPredictionRows: summary.nonEmptyPredictionRows, reports: summary.reports.map((r) => ({ taskId: r.taskId, setupStatus: r.setupStatus, indexStatus: r.indexStatus, toolCallable: r.toolCallable, nonEmptyPrediction: r.nonEmptyPrediction, officialEvaluatorScoreable: r.officialEvaluatorScoreable, predFiles: r.predFiles.length, selectedBecause: r.selectedBecause, costs: r.costs })) }, null, 2)); - if (!ready) process.exitCode = 1; + console.log(JSON.stringify({ ready: summary.ready, readinessMeaning: summary.readinessMeaning, attemptedRows: summary.attemptedRows, functionalRows: summary.functionalRows, blockerRows: summary.blockerRows, scoreableRows: summary.scoreableRows, nonEmptyPredictionRows: summary.nonEmptyPredictionRows, blockers: summary.blockers, reports: summary.reports.map((r) => ({ taskId: r.taskId, setupStatus: r.setupStatus, indexStatus: r.indexStatus, indexSignal: r.indexSignal, indexError: r.indexError, toolCallable: r.toolCallable, nonEmptyPrediction: r.nonEmptyPrediction, officialEvaluatorScoreable: r.officialEvaluatorScoreable, predFiles: r.predFiles.length, selectedBecause: r.selectedBecause, costs: r.costs })) }, null, 2)); + if (!summary.ready) process.exitCode = 1; NODE node "$ROOT/readiness.mjs" - name: Upload readiness artifacts From 66a91900b399578ce0b0e5bcc6842b9e4af19714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:11:03 +0200 Subject: [PATCH 008/100] Diagnose codebase-memory Go evaluator row --- .../contextbench-cbm-go-diagnose.yml | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 .github/workflows/contextbench-cbm-go-diagnose.yml diff --git a/.github/workflows/contextbench-cbm-go-diagnose.yml b/.github/workflows/contextbench-cbm-go-diagnose.yml new file mode 100644 index 0000000..cb8c901 --- /dev/null +++ b/.github/workflows/contextbench-cbm-go-diagnose.yml @@ -0,0 +1,94 @@ +name: ContextBench CBM Go Diagnose + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-cbm-go-diagnose.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + diagnose: + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + ROOT: /tmp/contextbench-cbm-go-diagnose + TASK_PAYLOADS: /tmp/contextbench-cbm-go-diagnose/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + CBM_BIN: /tmp/contextbench-cbm-go-diagnose/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'pnpm' + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies + run: | + pnpm install --frozen-lockfile + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + - name: Materialize Go task + run: | + mkdir -p "$ROOT" "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + - name: Download CBM + run: | + mkdir -p "$ROOT/tool" + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" + - name: Run Go row and print scorer diagnostics + run: | + cat > "$ROOT/go-diagnose.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + const root = process.env.ROOT; + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const task = payloads.tasks[2]; + const cbm = process.env.CBM_BIN; + const dir = join(root, 'go-row'); + mkdirSync(dir, { recursive: true }); + const env = { ...process.env, CBM_CACHE_DIR: join(dir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { cwd: opts.cwd || process.cwd(), env: opts.env || process.env, encoding: 'utf8', timeout: opts.timeoutMs || 1200000, maxBuffer: 128 * 1024 * 1024 }); + return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; + } + function jsonish(s) { try { return JSON.parse(String(s || '').trim()); } catch { return null; } } + function add(spans, file, start = 1, end = start) { if (!file) return; const clean = String(file).replace(/^\/+/, ''); const s = Math.max(1, Number(start) || 1); const e = Math.max(s, Number(end) || s); const list = spans.get(clean) || []; list.push({ start: s, end: e }); spans.set(clean, list); } + function walk(v, spans) { if (!v || typeof v !== 'object') return; if (Array.isArray(v)) { for (const x of v) walk(x, spans); return; } add(spans, v.file || v.path || v.file_path || v.relative_path || v.filename || v.source_path, v.start_line || v.line || 1, v.end_line || v.line || 1); for (const x of Object.values(v)) walk(x, spans); } + function textPaths(s, spans) { const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(s || ''))) !== null) add(spans, m[1], m[2] || 1, m[2] || 1); } + const query = 'Title System metrics written start Description system metrics'; + const setup = run(cbm, ['--version'], { env }); + const indexRun = run(cbm, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { cwd: task.repo_checkout_path, env, timeoutMs: 2700000 }); + const projectObj = jsonish(indexRun.stdout) || jsonish(indexRun.stderr) || {}; + const project = projectObj.project || basename(task.repo_checkout_path); + const graphSearch = run(cbm, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], { cwd: task.repo_checkout_path, env }); + const codeSearch = run(cbm, ['cli', 'search_code', JSON.stringify({ project, pattern: '.', mode: 'compact', limit: 25 })], { cwd: task.repo_checkout_path, env }); + const spans = new Map(); + for (const r of [graphSearch, codeSearch]) for (const text of [r.stdout, r.stderr]) { const parsed = jsonish(text); if (parsed) walk(parsed, spans); textPaths(text, spans); } + const predFiles = [...spans.keys()].slice(0, 20); + const predSpans = Object.fromEntries([...spans.entries()].slice(0, 20)); + const predictionPath = join(dir, 'prediction.json'); + writeFileSync(predictionPath, JSON.stringify({ instance_id: task.instance_id, repo_url: task.repo_checkout_path, commit: task.base_commit, traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans }, model_patch: '' }, null, 2)); + const goldPath = join(dir, 'gold.json'); + const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', process.env.TASK_PAYLOADS], { timeoutMs: 600000 }); + const official = join(root, 'ContextBench-official'); + const clone = existsSync(join(official, 'contextbench', 'evaluate.py')) ? { skipped: true } : run('git', ['clone', '--depth', '1', 'https://github.com/EuniAI/ContextBench.git', official], { timeoutMs: 600000 }); + const scorePath = join(dir, 'official-score.jsonl'); + const evaluator = run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predictionPath, '--cache', join(dir, 'repo-cache'), '--out', scorePath], { cwd: official, timeoutMs: 1200000 }); + const scoreText = existsSync(scorePath) ? readFileSync(scorePath, 'utf8') : ''; + console.log(JSON.stringify({ taskId: task.instance_id, setupStatus: setup.status, indexStatus: indexRun.status, indexSignal: indexRun.signal, project, graphStatus: graphSearch.status, codeStatus: codeSearch.status, predFiles: predFiles.length, goldStatus: gold.status, goldStderr: gold.stderr, clone, evaluatorStatus: evaluator.status, evaluatorSignal: evaluator.signal, evaluatorError: evaluator.error, evaluatorStdout: evaluator.stdout, evaluatorStderr: evaluator.stderr, scorePathExists: existsSync(scorePath), scoreBytes: scoreText.length, scoreHead: scoreText.slice(0, 1000) }, null, 2)); + if (!(indexRun.status === 0 && graphSearch.status === 0 && predFiles.length > 0 && evaluator.status === 0 && scoreText.length > 0)) process.exitCode = 1; + NODE + node "$ROOT/go-diagnose.mjs" From 0cb6ebad0d0e00150bf7f4cdaf5e90318ec07e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:18:35 +0200 Subject: [PATCH 009/100] Run real ContextBench gpt-5.4-mini matrix --- .../workflows/contextbench-real-gpt54mini.yml | 561 ++++++++++++++++++ 1 file changed, 561 insertions(+) create mode 100644 .github/workflows/contextbench-real-gpt54mini.yml diff --git a/.github/workflows/contextbench-real-gpt54mini.yml b/.github/workflows/contextbench-real-gpt54mini.yml new file mode 100644 index 0000000..ad7c888 --- /dev/null +++ b/.github/workflows/contextbench-real-gpt54mini.yml @@ -0,0 +1,561 @@ +name: ContextBench Real GPT54 Mini + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-real-gpt54mini.yml + workflow_dispatch: + inputs: + max_tasks: + description: 'Frozen ContextBench tasks to run from the manifest prefix' + required: true + default: '3' + repeats: + description: 'Repeats per lane/task' + required: true + default: '1' + lanes: + description: 'Comma-separated lanes' + required: true + default: 'raw-native,codebase-context,jcodemunch-repomapper,grepai,codebase-memory-mcp,codegraphcontext' + +permissions: + contents: read + +jobs: + real-matrix: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + ROOT: /tmp/contextbench-real-gpt54mini + TASK_PAYLOADS: /tmp/contextbench-real-gpt54mini/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + MAX_TASKS: ${{ github.event.inputs.max_tasks || '3' }} + REPEATS: ${{ github.event.inputs.repeats || '1' }} + LANES: ${{ github.event.inputs.lanes || 'raw-native,codebase-context,jcodemunch-repomapper,grepai,codebase-memory-mcp,codegraphcontext' }} + OPENAI_MODEL: gpt-5.4-mini + OPENAI_REASONING_EFFORT: high + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + CBM_VERSION: v0.6.1 + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v2 + with: + version: 10 + + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install repo, evaluator, and lane tooling + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + pnpm run build + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow uv codegraphcontext kuzu + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh || true + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/${CBM_VERSION}/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$ROOT/tool/codebase-memory-mcp" || true + for i in 1 2 3; do git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" && break || { rm -rf "$ROOT/ContextBench-official"; sleep 5; }; done + + - name: Materialize frozen ContextBench task checkouts + run: | + set -euxo pipefail + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks "$MAX_TASKS" + + - name: Run gpt-5.4-mini-high scoreable matrix + env: + CBM_BIN: /tmp/contextbench-real-gpt54mini/tool/codebase-memory-mcp + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-real-gpt54mini/ContextBench-official + run: | + cat > "$ROOT/run-real-matrix.mjs" <<'NODE' + import { spawn, spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + + const root = process.env.ROOT; + const outRoot = join(root, 'matrix'); + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const maxTasks = Number(process.env.MAX_TASKS || '3'); + const repeats = Number(process.env.REPEATS || '1'); + const lanes = String(process.env.LANES || '').split(',').map((lane) => lane.trim()).filter(Boolean); + const tasks = payloads.tasks.slice(0, maxTasks); + mkdirSync(outRoot, { recursive: true }); + + function run(cmd, args, opts = {}) { + const started = Date.now(); + const result = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 20 * 60 * 1000, + maxBuffer: 96 * 1024 * 1024 + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: result.status, + signal: result.signal, + error: result.error?.message || null, + durationMs: Date.now() - started, + stdout: result.stdout || '', + stderr: result.stderr || '' + }; + } + + function sanitize(value) { + return String(value).replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 180); + } + + function queryOf(problem) { + const stop = new Set(['that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', 'would', 'could', 'there', 'where', 'which', 'about', 'after', 'before']); + return String(problem || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase()) && !/^https?$/.test(w)) + .slice(0, 10) + .join(' '); + } + + function add(locations, file, start = 1, end = start, source = 'tool') { + if (typeof file !== 'string' || file.length === 0) return; + if (file.includes('://')) return; + const clean = file.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!clean || clean.includes('..')) return; + const s = Math.max(1, Number(start) || 1); + const e = Math.max(s, Number(end) || s); + locations.push({ file: clean, start: s, end: e, source }); + } + + function walk(value, locations, source = 'tool') { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { for (const item of value) walk(item, locations, source); return; } + const file = value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri; + const start = value.start_line || value.startLine || value.line || value.line_number || value.start || 1; + const end = value.end_line || value.endLine || value.end || start; + add(locations, file, start, end, source); + for (const item of Object.values(value)) walk(item, locations, source); + } + + function jsonish(text) { + const raw = String(text || '').trim(); + if (!raw) return null; + try { return JSON.parse(raw); } catch {} + for (const [open, close] of [['{', '}'], ['[', ']']]) { + const first = raw.indexOf(open); + const last = raw.lastIndexOf(close); + if (first >= 0 && last > first) { + try { return JSON.parse(raw.slice(first, last + 1)); } catch {} + } + } + return null; + } + + function collectFromText(text, locations, source = 'tool') { + const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; + let match; + while ((match = re.exec(String(text || ''))) !== null) add(locations, match[1], match[2] || 1, match[2] || 1, source); + } + + function collect(result, locations, source) { + for (const text of [result?.stdout, result?.stderr]) { + const parsed = jsonish(text); + if (parsed) walk(parsed, locations, source); + collectFromText(text, locations, source); + } + } + + function uniqueLocations(locations) { + const seen = new Set(); + const out = []; + for (const loc of locations) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= 80) break; + } + return out; + } + + function buildPrediction(task, modelSelection) { + const spans = new Map(); + for (const span of modelSelection.spans || []) { + addSpan(spans, span.file, span.start, span.end); + } + for (const file of modelSelection.files || []) addSpan(spans, file, 1, 1); + const predFiles = [...spans.keys()].slice(0, 20); + const predSpans = Object.fromEntries([...spans.entries()].slice(0, 20)); + return { + instance_id: task.instance_id, + repo_url: task.repo_checkout_path, + commit: task.base_commit, + traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans }, + model_patch: '' + }; + } + + function addSpan(spans, file, start = 1, end = start) { + if (typeof file !== 'string' || !file) return; + const clean = file.replace(/^\/+/, '').replace(/^\.\//, ''); + const s = Math.max(1, Number(start) || 1); + const e = Math.max(s, Number(end) || s); + const list = spans.get(clean) || []; + list.push({ start: s, end: e }); + spans.set(clean, list); + } + + function parseScores(scorePath) { + if (!existsSync(scorePath)) return null; + const lines = readFileSync(scorePath, 'utf8').trim().split(/\n+/).filter(Boolean); + if (lines.length === 0) return null; + try { return JSON.parse(lines.at(-1)); } catch { return null; } + } + + async function callOpenAI(runDir, task, lane, query, candidates) { + const started = Date.now(); + if (!process.env.OPENAI_API_KEY) { + return { ok: false, status: 'model_unavailable', durationMs: 0, error: 'missing_OPENAI_API_KEY_secret' }; + } + const candidateFiles = new Set(candidates.map((c) => c.file)); + const request = { + model: process.env.OPENAI_MODEL || 'gpt-5.4-mini', + reasoning: { effort: process.env.OPENAI_REASONING_EFFORT || 'high' }, + max_output_tokens: 1800, + instructions: 'You are selecting likely edit locations for ContextBench. Use only the provided candidate locations from the lane tool. Return JSON only.', + input: JSON.stringify({ + taskId: task.instance_id, + repo: task.repo, + lane, + query, + problemStatement: task.problem_statement, + candidateLocations: candidates.slice(0, 60) + }), + text: { + format: { + type: 'json_schema', + name: 'contextbench_location_selection', + strict: true, + schema: { + type: 'object', + additionalProperties: false, + required: ['files', 'spans', 'notes'], + properties: { + files: { type: 'array', maxItems: 20, items: { type: 'string' } }, + spans: { + type: 'array', + maxItems: 40, + items: { + type: 'object', + additionalProperties: false, + required: ['file', 'start', 'end'], + properties: { + file: { type: 'string' }, + start: { type: 'integer', minimum: 1 }, + end: { type: 'integer', minimum: 1 } + } + } + }, + notes: { type: 'string' } + } + } + } + } + }; + writeFileSync(join(runDir, 'openai-request.redacted.json'), JSON.stringify({ ...request, input: JSON.parse(request.input) }, null, 2)); + let responseText = ''; + let responseJson = null; + try { + const response = await fetch('https://api.openai.com/v1/responses', { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${process.env.OPENAI_API_KEY}` }, + body: JSON.stringify(request) + }); + responseText = await response.text(); + writeFileSync(join(runDir, 'openai-response.json'), responseText); + if (!response.ok) return { ok: false, status: 'model_error', durationMs: Date.now() - started, httpStatus: response.status, error: responseText.slice(0, 2000) }; + responseJson = JSON.parse(responseText); + } catch (error) { + return { ok: false, status: 'model_error', durationMs: Date.now() - started, error: String(error?.message || error) }; + } + const outputText = responseJson.output_text || (responseJson.output || []).flatMap((item) => item.content || []).filter((item) => item.type === 'output_text').map((item) => item.text).join('\n'); + let parsed; + try { parsed = JSON.parse(outputText); } catch (error) { return { ok: false, status: 'model_parse_error', durationMs: Date.now() - started, error: String(error?.message || error), outputText }; } + const files = [...new Set((parsed.files || []).filter((file) => candidateFiles.has(file)))].slice(0, 20); + const spans = (parsed.spans || []).filter((span) => candidateFiles.has(span.file)).slice(0, 40); + if (files.length === 0 && spans.length === 0) return { ok: false, status: 'model_empty_after_lane_filter', durationMs: Date.now() - started, parsed }; + return { ok: true, status: 'completed', durationMs: Date.now() - started, parsed: { files, spans, notes: parsed.notes || '' }, responseId: responseJson.id || null, usage: responseJson.usage || null }; + } + + async function callMcp(command, args, cwd, calls, env = process.env) { + const { Client } = await import('@modelcontextprotocol/sdk/client/index.js'); + const { StdioClientTransport } = await import('@modelcontextprotocol/sdk/client/stdio.js'); + const transport = new StdioClientTransport({ command, args, cwd, env }); + const client = new Client({ name: 'contextbench-ci', version: '1.0.0' }, { capabilities: {} }); + const started = Date.now(); + const outputs = []; + try { + await client.connect(transport); + const tools = await client.listTools(); + outputs.push({ tool: 'tools/list', result: tools }); + for (const call of calls) { + const result = await client.callTool({ name: call.name, arguments: call.arguments }); + outputs.push({ tool: call.name, arguments: call.arguments, result }); + } + await client.close(); + return { status: 0, signal: null, error: null, durationMs: Date.now() - started, stdout: JSON.stringify(outputs), stderr: '', command: [command, ...args].join(' '), cwd }; + } catch (error) { + try { await client.close(); } catch {} + return { status: 1, signal: null, error: String(error?.message || error), durationMs: Date.now() - started, stdout: JSON.stringify(outputs), stderr: String(error?.stack || error), command: [command, ...args].join(' '), cwd }; + } + } + + async function retrieve(lane, task, runDir, query) { + const repo = task.repo_checkout_path; + const firstTerm = query.split(/\s+/).find(Boolean) || 'main'; + const locations = []; + const commands = []; + let setupStatus = 'completed'; + let indexStatus = 'completed'; + const startedSetup = Date.now(); + let setupDurationMs = 0; + let indexDurationMs = 0; + let queryDurationMs = 0; + + if (lane === 'raw-native') { + setupDurationMs = Date.now() - startedSetup; + const qTerms = query.split(/\s+/).slice(0, 5); + const started = Date.now(); + for (const term of qTerms) { + const result = run('rg', ['-n', '-i', '--glob', '!.git', '--glob', '!vendor/**', '--glob', '!node_modules/**', term, repo], { timeoutMs: 60_000 }); + commands.push(result); + collect(result, locations, 'raw-native'); + } + queryDurationMs = Date.now() - started; + } else if (lane === 'codebase-context') { + const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' }; + const version = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60_000 }); + commands.push(version); + setupDurationMs = version.durationMs; + const reindex = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 20 * 60_000 }); + commands.push(reindex); + indexDurationMs = reindex.durationMs; + if (reindex.status !== 0) indexStatus = 'index_failed'; + const search = run('node', ['dist/index.js', 'search', '--query', query, '--intent', 'edit', '--limit', '25', '--json'], { env, timeoutMs: 5 * 60_000 }); + commands.push(search); + queryDurationMs = search.durationMs; + collect(search, locations, 'codebase-context'); + } else if (lane === 'codebase-memory-mcp') { + const env = { ...process.env, CBM_CACHE_DIR: join(runDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const version = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60_000 }); + commands.push(version); + setupDurationMs = version.durationMs; + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 45 * 60_000 }); + commands.push(index); + indexDurationMs = index.durationMs; + if (index.status !== 0) indexStatus = 'index_failed'; + const projectObj = jsonish(index.stdout) || jsonish(index.stderr) || {}; + const project = projectObj.project || basename(repo); + const started = Date.now(); + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], { cwd: repo, env, timeoutMs: 120_000 }); + const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: firstTerm, mode: 'compact', limit: 25 })], { cwd: repo, env, timeoutMs: 120_000 }); + commands.push(graph, code); + queryDurationMs = Date.now() - started; + collect(graph, locations, 'codebase-memory-mcp'); + collect(code, locations, 'codebase-memory-mcp'); + } else if (lane === 'grepai') { + const grepai = 'grepai'; + const version = run(grepai, ['version'], { timeoutMs: 60_000 }); + commands.push(version); + const initArgs = process.env.OPENAI_API_KEY ? ['init', '--yes', '--provider', 'openai', '--backend', 'gob', '--model', 'text-embedding-3-small'] : ['init', '--yes', '--provider', 'synthetic', '--backend', 'gob']; + const init = run(grepai, initArgs, { cwd: repo, timeoutMs: 120_000 }); + commands.push(init); + setupDurationMs = version.durationMs + init.durationMs; + if (init.status !== 0) setupStatus = 'setup_failed'; + const startedIndex = Date.now(); + const watch = run(grepai, ['watch', '--background'], { cwd: repo, timeoutMs: 120_000 }); + commands.push(watch); + await new Promise((resolve) => setTimeout(resolve, 25_000)); + const status = run(grepai, ['watch', '--status'], { cwd: repo, timeoutMs: 60_000 }); + commands.push(status); + indexDurationMs = Date.now() - startedIndex; + if (watch.status !== 0 && status.status !== 0) indexStatus = 'index_failed'; + const search = run(grepai, ['search', query, '--json', '--compact'], { cwd: repo, timeoutMs: 180_000 }); + commands.push(search); + queryDurationMs = search.durationMs; + collect(search, locations, 'grepai'); + commands.push(run(grepai, ['watch', '--stop'], { cwd: repo, timeoutMs: 60_000 })); + } else if (lane === 'jcodemunch-repomapper') { + const version = run('uvx', ['jcodemunch-mcp', '--help'], { timeoutMs: 180_000 }); + commands.push(version); + setupDurationMs = version.durationMs; + const mcp = await callMcp('uvx', ['jcodemunch-mcp'], repo, [ + { name: 'index_folder', arguments: { path: repo, incremental: false, use_ai_summaries: false, follow_symlinks: false } }, + { name: 'search_symbols', arguments: { repo, query, max_results: 25, semantic: false } }, + { name: 'search_symbols', arguments: { repo: basename(repo), query, max_results: 25, semantic: false } } + ]); + commands.push(mcp); + indexDurationMs = mcp.durationMs; + queryDurationMs = 0; + if (mcp.status !== 0) indexStatus = 'index_or_query_failed'; + collect(mcp, locations, 'jcodemunch-repomapper'); + } else if (lane === 'codegraphcontext') { + const version = run('cgc', ['--help'], { timeoutMs: 60_000 }); + commands.push(version); + setupDurationMs = version.durationMs; + const index = run('cgc', ['index', repo], { cwd: repo, timeoutMs: 30 * 60_000 }); + commands.push(index); + indexDurationMs = index.durationMs; + if (index.status !== 0) indexStatus = 'index_failed'; + const started = Date.now(); + const complexity = run('cgc', ['analyze', 'complexity', '--limit', '25'], { cwd: repo, timeoutMs: 180_000 }); + const callers = run('cgc', ['analyze', 'callers', firstTerm], { cwd: repo, timeoutMs: 120_000 }); + const dead = run('cgc', ['analyze', 'dead-code', '--limit', '25'], { cwd: repo, timeoutMs: 180_000 }); + commands.push(complexity, callers, dead); + queryDurationMs = Date.now() - started; + collect(complexity, locations, 'codegraphcontext'); + collect(callers, locations, 'codegraphcontext'); + collect(dead, locations, 'codegraphcontext'); + } else { + setupStatus = 'setup_failed'; + commands.push({ command: lane, cwd: repo, status: 1, signal: null, error: 'unknown_lane', durationMs: 0, stdout: '', stderr: '' }); + } + + for (const [i, command] of commands.entries()) writeFileSync(join(runDir, `command-${i + 1}.json`), JSON.stringify(command, null, 2)); + const unique = uniqueLocations(locations); + writeFileSync(join(runDir, 'candidate-locations.json'), JSON.stringify(unique, null, 2)); + return { lane, setupStatus, indexStatus, toolCallable: commands.some((command) => command.status === 0), candidates: unique, costs: { setupDurationMs, indexDurationMs, queryDurationMs }, commands }; + } + + const rows = []; + const scoreableRows = []; + for (const task of tasks) { + const query = queryOf(task.problem_statement); + for (const lane of lanes) { + for (let repeat = 1; repeat <= repeats; repeat += 1) { + const runId = sanitize(`${lane}-${task.instance_id}-r${repeat}`); + const runDir = join(outRoot, 'runs', runId); + mkdirSync(runDir, { recursive: true }); + const startedAt = new Date().toISOString(); + let row; + try { + const retrieval = await retrieve(lane, task, runDir, query); + const model = await callOpenAI(runDir, task, lane, query, retrieval.candidates); + const prediction = model.ok ? buildPrediction(task, model.parsed) : buildPrediction(task, { files: [], spans: [] }); + const predictionPath = join(runDir, 'prediction.json'); + writeFileSync(predictionPath, JSON.stringify(prediction, null, 2)); + const predFiles = prediction.traj_data.pred_files; + const predSpans = prediction.traj_data.pred_spans; + const goldPath = join(runDir, 'gold.json'); + const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', process.env.TASK_PAYLOADS], { timeoutMs: 10 * 60_000 }); + const scorePath = join(runDir, 'official-score.jsonl'); + const evaluator = model.ok && predFiles.length > 0 + ? run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predictionPath, '--cache', join(runDir, 'repo-cache'), '--out', scorePath], { cwd: process.env.OFFICIAL_CONTEXTBENCH, timeoutMs: 20 * 60_000 }) + : { command: 'python -m contextbench.evaluate', cwd: process.env.OFFICIAL_CONTEXTBENCH, status: null, signal: null, error: 'skipped_no_model_prediction', durationMs: 0, stdout: '', stderr: '' }; + writeFileSync(join(runDir, 'gold-command.json'), JSON.stringify(gold, null, 2)); + writeFileSync(join(runDir, 'evaluator-command.json'), JSON.stringify(evaluator, null, 2)); + const score = parseScores(scorePath); + const officialEvaluatorScoreable = evaluator.status === 0 && Boolean(score); + const status = officialEvaluatorScoreable ? 'completed' : (model.ok ? 'judge_failed' : model.status); + row = { + run_id: runId, + lane_id: lane, + task_id: task.instance_id, + repeat_index: repeat, + status, + model: `${process.env.OPENAI_MODEL || 'gpt-5.4-mini'}-${process.env.OPENAI_REASONING_EFFORT || 'high'}`, + started_at: startedAt, + completed_at: new Date().toISOString(), + setupIndex: retrieval.costs, + setupStatus: retrieval.setupStatus, + indexStatus: retrieval.indexStatus, + toolCallable: retrieval.toolCallable, + candidateCount: retrieval.candidates.length, + nonEmptyPrediction: predFiles.length > 0 && Object.keys(predSpans).length > 0, + predFiles: predFiles.length, + officialEvaluatorScoreable, + modelStatus: model.status, + modelDurationMs: model.durationMs, + modelUsage: model.usage || null, + score, + paths: { runDir, predictionPath, scorePath } + }; + if (officialEvaluatorScoreable) scoreableRows.push(row); + } catch (error) { + row = { run_id: runId, lane_id: lane, task_id: task.instance_id, repeat_index: repeat, status: 'tool_error', error: String(error?.stack || error), started_at: startedAt, completed_at: new Date().toISOString(), officialEvaluatorScoreable: false }; + writeFileSync(join(runDir, 'row-error.txt'), row.error); + } + rows.push(row); + writeFileSync(join(runDir, 'row.json'), JSON.stringify(row, null, 2)); + console.log(JSON.stringify({ run_id: row.run_id, lane_id: row.lane_id, task_id: row.task_id, status: row.status, scoreable: row.officialEvaluatorScoreable, predFiles: row.predFiles || 0, candidateCount: row.candidateCount || 0 })); + } + } + } + + const byLane = {}; + for (const row of scoreableRows) { + const bucket = byLane[row.lane_id] || { lane: row.lane_id, scoreableRows: 0, fileCoverage: [], filePrecision: [], symbolCoverage: [], spanCoverage: [], lineCoverage: [], editlocRecall: [] }; + bucket.scoreableRows += 1; + bucket.fileCoverage.push(row.score?.final?.file?.coverage ?? null); + bucket.filePrecision.push(row.score?.final?.file?.precision ?? null); + bucket.symbolCoverage.push(row.score?.final?.symbol?.coverage ?? null); + bucket.spanCoverage.push(row.score?.final?.span?.coverage ?? null); + bucket.lineCoverage.push(row.score?.final?.line?.coverage ?? null); + bucket.editlocRecall.push(row.score?.editloc?.recall ?? null); + byLane[row.lane_id] = bucket; + } + function mean(values) { + const nums = values.filter((value) => Number.isFinite(value)); + return nums.length ? nums.reduce((a, b) => a + b, 0) / nums.length : null; + } + const resultsTable = Object.values(byLane).map((bucket) => ({ + lane: bucket.lane, + scoreableRows: bucket.scoreableRows, + fileCoverage: mean(bucket.fileCoverage), + filePrecision: mean(bucket.filePrecision), + symbolCoverage: mean(bucket.symbolCoverage), + spanCoverage: mean(bucket.spanCoverage), + lineCoverage: mean(bucket.lineCoverage), + editlocRecall: mean(bucket.editlocRecall) + })).sort((a, b) => (b.fileCoverage ?? -1) - (a.fileCoverage ?? -1)); + const summary = { + createdAt: new Date().toISOString(), + model: `${process.env.OPENAI_MODEL || 'gpt-5.4-mini'}-${process.env.OPENAI_REASONING_EFFORT || 'high'}`, + maxTasks, + repeats, + lanes, + attemptedRows: rows.length, + scoreableRows: scoreableRows.length, + nonScoreableRows: rows.length - scoreableRows.length, + statusCounts: rows.reduce((acc, row) => { acc[row.status] = (acc[row.status] || 0) + 1; return acc; }, {}), + setupIndexCostReportedSeparately: true, + resultsTable, + rows + }; + writeFileSync(join(outRoot, 'summary.json'), JSON.stringify(summary, null, 2)); + writeFileSync(join(outRoot, 'results-table.json'), JSON.stringify(resultsTable, null, 2)); + console.log(JSON.stringify({ attemptedRows: summary.attemptedRows, scoreableRows: summary.scoreableRows, statusCounts: summary.statusCounts, resultsTable }, null, 2)); + if (scoreableRows.length === 0) process.exitCode = 1; + NODE + node "$ROOT/run-real-matrix.mjs" + + - name: Upload real benchmark artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-real-gpt54mini + path: /tmp/contextbench-real-gpt54mini + retention-days: 14 From 3074da808350835ca73d0e7a60a01e7d20421343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:23:19 +0200 Subject: [PATCH 010/100] Run bounded core ContextBench gpt-5.4-mini rows --- .../contextbench-real-gpt54mini-core.yml | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 .github/workflows/contextbench-real-gpt54mini-core.yml diff --git a/.github/workflows/contextbench-real-gpt54mini-core.yml b/.github/workflows/contextbench-real-gpt54mini-core.yml new file mode 100644 index 0000000..348ea9b --- /dev/null +++ b/.github/workflows/contextbench-real-gpt54mini-core.yml @@ -0,0 +1,112 @@ +name: ContextBench Real GPT54 Mini Core + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-real-gpt54mini-core.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + core-go-task: + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + ROOT: /tmp/contextbench-real-gpt54mini-core + TASK_PAYLOADS: /tmp/contextbench-real-gpt54mini-core/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + OPENAI_MODEL: gpt-5.4-mini + OPENAI_REASONING_EFFORT: high + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + CBM_BIN: /tmp/contextbench-real-gpt54mini-core/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies and materialize Go task + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + pnpm run build + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + - name: Run bounded core scoreable rows + env: + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-real-gpt54mini-core/ContextBench-official + run: | + cat > "$ROOT/core.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + const root = process.env.ROOT; + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const task = payloads.tasks[2]; + const outRoot = join(root, 'core-go'); + mkdirSync(outRoot, { recursive: true }); + const lanes = ['raw-native', 'codebase-context', 'codebase-memory-mcp']; + function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { cwd: opts.cwd || process.cwd(), env: opts.env || process.env, encoding: 'utf8', timeout: opts.timeoutMs || 600000, maxBuffer: 96 * 1024 * 1024 }); + return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; + } + function q(problem) { return String(problem || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4).slice(0, 10).join(' '); } + function add(locs, file, start = 1, end = start, source = 'tool') { if (!file || typeof file !== 'string' || file.includes('://')) return; const clean = file.replace(/^\/+/, '').replace(/^\.\//, ''); if (!clean || clean.includes('..')) return; const s = Math.max(1, Number(start) || 1); locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); } + function jsonish(s) { const t = String(s || '').trim(); if (!t) return null; try { return JSON.parse(t); } catch {} for (const [a,b] of [['{','}'],['[',']']]) { const i=t.indexOf(a), j=t.lastIndexOf(b); if (i>=0 && j>i) { try { return JSON.parse(t.slice(i,j+1)); } catch {} } } return null; } + function walk(v, locs, source) { if (!v || typeof v !== 'object') return; if (Array.isArray(v)) { for (const x of v) walk(x, locs, source); return; } add(locs, v.file || v.path || v.file_path || v.relative_path || v.filename || v.source_path, v.start_line || v.line || 1, v.end_line || v.line || 1, source); for (const x of Object.values(v)) walk(x, locs, source); } + function collect(text, locs, source) { const parsed = jsonish(text); if (parsed) walk(parsed, locs, source); const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); } + function uniq(locs) { const seen = new Set(), out = []; for (const loc of locs) { const k = `${loc.file}:${loc.start}:${loc.end}`; if (!seen.has(k)) { seen.add(k); out.push(loc); if (out.length >= 80) break; } } return out; } + function addSpan(map, file, start = 1, end = start) { if (!file) return; const s = Math.max(1, Number(start) || 1); const e = Math.max(s, Number(end) || s); const list = map.get(file) || []; list.push({ start: s, end: e }); map.set(file, list); } + function prediction(selection) { const spans = new Map(); for (const span of selection.spans || []) addSpan(spans, span.file, span.start, span.end); for (const file of selection.files || []) addSpan(spans, file, 1, 1); const predFiles = [...spans.keys()].slice(0, 20); return { instance_id: task.instance_id, repo_url: task.repo_checkout_path, commit: task.base_commit, traj_data: { pred_steps: [{ files: predFiles, spans: Object.fromEntries([...spans.entries()].slice(0, 20)) }], pred_files: predFiles, pred_spans: Object.fromEntries([...spans.entries()].slice(0, 20)) }, model_patch: '' }; } + async function askModel(runDir, lane, query, candidates) { + if (!process.env.OPENAI_API_KEY) return { ok: false, status: 'model_unavailable', error: 'missing_OPENAI_API_KEY_secret', durationMs: 0 }; + const started = Date.now(); const candidateFiles = new Set(candidates.map((c) => c.file)); + const body = { model: process.env.OPENAI_MODEL, reasoning: { effort: process.env.OPENAI_REASONING_EFFORT }, max_output_tokens: 1600, instructions: 'Select likely ContextBench edit locations using only provided candidate locations. Return JSON only.', input: JSON.stringify({ taskId: task.instance_id, repo: task.repo, lane, problemStatement: task.problem_statement, query, candidateLocations: candidates.slice(0, 60) }), text: { format: { type: 'json_schema', name: 'contextbench_selection', strict: true, schema: { type: 'object', additionalProperties: false, required: ['files','spans','notes'], properties: { files: { type: 'array', maxItems: 20, items: { type: 'string' } }, spans: { type: 'array', maxItems: 40, items: { type: 'object', additionalProperties: false, required: ['file','start','end'], properties: { file: { type: 'string' }, start: { type: 'integer', minimum: 1 }, end: { type: 'integer', minimum: 1 } } } }, notes: { type: 'string' } } } } } }; + writeFileSync(join(runDir, 'openai-request.redacted.json'), JSON.stringify({ ...body, input: JSON.parse(body.input) }, null, 2)); + const res = await fetch('https://api.openai.com/v1/responses', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${process.env.OPENAI_API_KEY}` }, body: JSON.stringify(body) }); + const text = await res.text(); writeFileSync(join(runDir, 'openai-response.json'), text); + if (!res.ok) return { ok: false, status: 'model_error', httpStatus: res.status, error: text.slice(0, 2000), durationMs: Date.now() - started }; + const json = JSON.parse(text); const out = json.output_text || (json.output || []).flatMap((i) => i.content || []).filter((i) => i.type === 'output_text').map((i) => i.text).join('\n'); + const parsed = JSON.parse(out); const files = [...new Set((parsed.files || []).filter((f) => candidateFiles.has(f)))].slice(0, 20); const spans = (parsed.spans || []).filter((s) => candidateFiles.has(s.file)).slice(0, 40); + if (files.length === 0 && spans.length === 0) return { ok: false, status: 'model_empty_after_lane_filter', parsed, durationMs: Date.now() - started }; + return { ok: true, status: 'completed', parsed: { files, spans, notes: parsed.notes || '' }, usage: json.usage || null, durationMs: Date.now() - started }; + } + async function retrieve(lane, runDir, query) { + const repo = task.repo_checkout_path; const locs = []; const commands = []; let setupDurationMs = 0, indexDurationMs = 0, queryDurationMs = 0, setupStatus = 'completed', indexStatus = 'completed'; + if (lane === 'raw-native') { const start = Date.now(); for (const term of query.split(/\s+/).slice(0, 6)) { const r = run('rg', ['-n', '-i', '--glob', '!.git', term, repo], { timeoutMs: 60000 }); commands.push(r); collect(r.stdout, locs, lane); collect(r.stderr, locs, lane); } queryDurationMs = Date.now() - start; } + if (lane === 'codebase-context') { const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' }; const v = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60000 }); commands.push(v); setupDurationMs = v.durationMs; const idx = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 1200000 }); commands.push(idx); indexDurationMs = idx.durationMs; if (idx.status !== 0) indexStatus = 'index_failed'; const s = run('node', ['dist/index.js', 'search', '--query', query, '--intent', 'edit', '--limit', '25', '--json'], { env, timeoutMs: 300000 }); commands.push(s); queryDurationMs = s.durationMs; collect(s.stdout, locs, lane); collect(s.stderr, locs, lane); } + if (lane === 'codebase-memory-mcp') { const env = { ...process.env, CBM_CACHE_DIR: join(runDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; const v = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); commands.push(v); setupDurationMs = v.durationMs; const idx = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 2700000 }); commands.push(idx); indexDurationMs = idx.durationMs; if (idx.status !== 0) indexStatus = 'index_failed'; const project = (jsonish(idx.stdout) || jsonish(idx.stderr) || {}).project || basename(repo); const start = Date.now(); const g = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], { cwd: repo, env, timeoutMs: 120000 }); const c = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 25 })], { cwd: repo, env, timeoutMs: 120000 }); commands.push(g, c); queryDurationMs = Date.now() - start; for (const r of [g,c]) { collect(r.stdout, locs, lane); collect(r.stderr, locs, lane); } } + for (const [i, cmd] of commands.entries()) writeFileSync(join(runDir, `command-${i+1}.json`), JSON.stringify(cmd, null, 2)); const candidates = uniq(locs); writeFileSync(join(runDir, 'candidate-locations.json'), JSON.stringify(candidates, null, 2)); return { setupStatus, indexStatus, toolCallable: commands.some((c) => c.status === 0), costs: { setupDurationMs, indexDurationMs, queryDurationMs }, candidates }; + } + const rows = []; const scoreable = []; const query = q(task.problem_statement); + for (const lane of lanes) { const runDir = join(outRoot, lane); mkdirSync(runDir, { recursive: true }); const retrieval = await retrieve(lane, runDir, query); const model = await askModel(runDir, lane, query, retrieval.candidates); const pred = model.ok ? prediction(model.parsed) : prediction({ files: [], spans: [] }); const predPath = join(runDir, 'prediction.json'); writeFileSync(predPath, JSON.stringify(pred, null, 2)); const goldPath = join(runDir, 'gold.json'); const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', process.env.TASK_PAYLOADS], { timeoutMs: 600000 }); writeFileSync(join(runDir, 'gold-command.json'), JSON.stringify(gold, null, 2)); const scorePath = join(runDir, 'official-score.jsonl'); const evaluator = model.ok && pred.traj_data.pred_files.length > 0 ? run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predPath, '--cache', join(runDir, 'repo-cache'), '--out', scorePath], { cwd: process.env.OFFICIAL_CONTEXTBENCH, timeoutMs: 1200000 }) : { status: null, error: 'skipped_no_model_prediction', durationMs: 0 }; writeFileSync(join(runDir, 'evaluator-command.json'), JSON.stringify(evaluator, null, 2)); let score = null; if (existsSync(scorePath)) { const lines = readFileSync(scorePath, 'utf8').trim().split(/\n+/).filter(Boolean); if (lines.length) score = JSON.parse(lines.at(-1)); } const row = { lane_id: lane, task_id: task.instance_id, status: evaluator.status === 0 && score ? 'completed' : (model.status || 'judge_failed'), model: `${process.env.OPENAI_MODEL}-${process.env.OPENAI_REASONING_EFFORT}`, setupStatus: retrieval.setupStatus, indexStatus: retrieval.indexStatus, toolCallable: retrieval.toolCallable, candidateCount: retrieval.candidates.length, nonEmptyPrediction: pred.traj_data.pred_files.length > 0, predFiles: pred.traj_data.pred_files.length, officialEvaluatorScoreable: evaluator.status === 0 && Boolean(score), setupIndex: retrieval.costs, modelStatus: model.status, modelUsage: model.usage || null, score }; rows.push(row); if (row.officialEvaluatorScoreable) scoreable.push(row); writeFileSync(join(runDir, 'row.json'), JSON.stringify(row, null, 2)); console.log(JSON.stringify({ lane: lane, status: row.status, scoreable: row.officialEvaluatorScoreable, predFiles: row.predFiles, candidateCount: row.candidateCount })); } + function mean(values) { const nums = values.filter((v) => Number.isFinite(v)); return nums.length ? nums.reduce((a,b) => a + b, 0) / nums.length : null; } + const table = scoreable.map((r) => ({ lane: r.lane_id, fileCoverage: r.score.final.file.coverage, filePrecision: r.score.final.file.precision, symbolCoverage: r.score.final.symbol.coverage, spanCoverage: r.score.final.span.coverage, lineCoverage: r.score.final.line.coverage, editlocRecall: r.score.editloc?.recall ?? null })); + const summary = { createdAt: new Date().toISOString(), model: `${process.env.OPENAI_MODEL}-${process.env.OPENAI_REASONING_EFFORT}`, taskId: task.instance_id, attemptedRows: rows.length, scoreableRows: scoreable.length, statusCounts: rows.reduce((a,r) => { a[r.status] = (a[r.status] || 0) + 1; return a; }, {}), setupIndexCostReportedSeparately: true, resultsTable: table, rows }; + writeFileSync(join(outRoot, 'summary.json'), JSON.stringify(summary, null, 2)); console.log(JSON.stringify(summary, null, 2)); if (scoreable.length === 0) process.exitCode = 1; + NODE + node "$ROOT/core.mjs" + - name: Upload bounded core artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-real-gpt54mini-core + path: /tmp/contextbench-real-gpt54mini-core + retention-days: 14 From f6e930320b9f319707ab51afb2c158c3dfc86385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:28:18 +0200 Subject: [PATCH 011/100] Run one bounded CBM ContextBench row --- .../contextbench-real-cbm-go-one.yml | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 .github/workflows/contextbench-real-cbm-go-one.yml diff --git a/.github/workflows/contextbench-real-cbm-go-one.yml b/.github/workflows/contextbench-real-cbm-go-one.yml new file mode 100644 index 0000000..ccfc1a6 --- /dev/null +++ b/.github/workflows/contextbench-real-cbm-go-one.yml @@ -0,0 +1,108 @@ +name: ContextBench Real CBM Go One + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-real-cbm-go-one.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + cbm-go-one: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + ROOT: /tmp/contextbench-real-cbm-go-one + TASK_PAYLOADS: /tmp/contextbench-real-cbm-go-one/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + OPENAI_MODEL: gpt-5.4-mini + OPENAI_REASONING_EFFORT: high + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + CBM_BIN: /tmp/contextbench-real-cbm-go-one/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install and materialize + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + - name: Run one bounded scoreable CBM row + env: + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-real-cbm-go-one/ContextBench-official + run: | + cat > "$ROOT/one.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + const root = process.env.ROOT; + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const task = payloads.tasks[2]; + const runDir = join(root, 'row-codebase-memory-mcp-go'); + mkdirSync(runDir, { recursive: true }); + function run(cmd, args, opts = {}) { const started = Date.now(); const r = spawnSync(cmd, args, { cwd: opts.cwd || process.cwd(), env: opts.env || process.env, encoding: 'utf8', timeout: opts.timeoutMs || 600000, maxBuffer: 128 * 1024 * 1024 }); return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; } + function queryOf(s) { return String(s || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4).slice(0, 10).join(' '); } + function jsonish(s) { const t = String(s || '').trim(); if (!t) return null; try { return JSON.parse(t); } catch {} for (const [a,b] of [['{','}'],['[',']']]) { const i=t.indexOf(a), j=t.lastIndexOf(b); if (i>=0 && j>i) { try { return JSON.parse(t.slice(i,j+1)); } catch {} } } return null; } + function add(locs, file, start = 1, end = start, source = 'cbm') { if (!file || typeof file !== 'string' || file.includes('://')) return; const clean = file.replace(/^\/+/, '').replace(/^\.\//, ''); if (!clean || clean.includes('..')) return; const s = Math.max(1, Number(start) || 1); locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); } + function walk(v, locs) { if (!v || typeof v !== 'object') return; if (Array.isArray(v)) { for (const x of v) walk(x, locs); return; } add(locs, v.file || v.path || v.file_path || v.relative_path || v.filename || v.source_path, v.start_line || v.line || 1, v.end_line || v.line || 1); for (const x of Object.values(v)) walk(x, locs); } + function collect(text, locs) { const parsed = jsonish(text); if (parsed) walk(parsed, locs); const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1); } + function uniq(locs) { const seen = new Set(), out = []; for (const loc of locs) { const k = `${loc.file}:${loc.start}:${loc.end}`; if (!seen.has(k)) { seen.add(k); out.push(loc); if (out.length >= 80) break; } } return out; } + function addSpan(map, file, start = 1, end = start) { const s = Math.max(1, Number(start) || 1); const e = Math.max(s, Number(end) || s); const list = map.get(file) || []; list.push({ start: s, end: e }); map.set(file, list); } + async function ask(candidates, query) { + const started = Date.now(); + if (!process.env.OPENAI_API_KEY) return { ok: false, status: 'model_unavailable', error: 'missing_OPENAI_API_KEY_secret', durationMs: 0 }; + const candidateFiles = new Set(candidates.map((c) => c.file)); + const body = { model: process.env.OPENAI_MODEL, reasoning: { effort: process.env.OPENAI_REASONING_EFFORT }, max_output_tokens: 1200, instructions: 'Select likely ContextBench edit locations using only the provided codebase-memory-mcp candidate locations. Return JSON only.', input: JSON.stringify({ taskId: task.instance_id, repo: task.repo, lane: 'codebase-memory-mcp', query, problemStatement: task.problem_statement, candidateLocations: candidates.slice(0, 60) }), text: { format: { type: 'json_schema', name: 'contextbench_selection', strict: true, schema: { type: 'object', additionalProperties: false, required: ['files','spans','notes'], properties: { files: { type: 'array', maxItems: 20, items: { type: 'string' } }, spans: { type: 'array', maxItems: 40, items: { type: 'object', additionalProperties: false, required: ['file','start','end'], properties: { file: { type: 'string' }, start: { type: 'integer', minimum: 1 }, end: { type: 'integer', minimum: 1 } } } }, notes: { type: 'string' } } } } } }; + writeFileSync(join(runDir, 'openai-request.redacted.json'), JSON.stringify({ ...body, input: JSON.parse(body.input) }, null, 2)); + const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), 300000); + try { const res = await fetch('https://api.openai.com/v1/responses', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${process.env.OPENAI_API_KEY}` }, body: JSON.stringify(body), signal: controller.signal }); const text = await res.text(); clearTimeout(timer); writeFileSync(join(runDir, 'openai-response.json'), text); if (!res.ok) return { ok: false, status: 'model_error', httpStatus: res.status, error: text.slice(0, 2000), durationMs: Date.now() - started }; const json = JSON.parse(text); const out = json.output_text || (json.output || []).flatMap((i) => i.content || []).filter((i) => i.type === 'output_text').map((i) => i.text).join('\n'); const parsed = JSON.parse(out); const files = [...new Set((parsed.files || []).filter((f) => candidateFiles.has(f)))].slice(0, 20); const spans = (parsed.spans || []).filter((s) => candidateFiles.has(s.file)).slice(0, 40); if (files.length === 0 && spans.length === 0) return { ok: false, status: 'model_empty_after_lane_filter', parsed, durationMs: Date.now() - started }; return { ok: true, status: 'completed', parsed: { files, spans, notes: parsed.notes || '' }, usage: json.usage || null, durationMs: Date.now() - started }; } catch (error) { clearTimeout(timer); return { ok: false, status: 'model_timeout_or_error', error: String(error?.message || error), durationMs: Date.now() - started }; } + } + const query = queryOf(task.problem_statement); + const env = { ...process.env, CBM_CACHE_DIR: join(runDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { cwd: task.repo_checkout_path, env, timeoutMs: 2700000 }); + const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(task.repo_checkout_path); + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], { cwd: task.repo_checkout_path, env, timeoutMs: 120000 }); + const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 25 })], { cwd: task.repo_checkout_path, env, timeoutMs: 120000 }); + for (const [name, value] of Object.entries({ setup, index, graph, code })) writeFileSync(join(runDir, `${name}.json`), JSON.stringify(value, null, 2)); + const locs = []; for (const r of [graph, code]) { collect(r.stdout, locs); collect(r.stderr, locs); } + const candidates = uniq(locs); writeFileSync(join(runDir, 'candidate-locations.json'), JSON.stringify(candidates, null, 2)); + const model = await ask(candidates, query); writeFileSync(join(runDir, 'model-result.json'), JSON.stringify(model, null, 2)); + const spanMap = new Map(); if (model.ok) { for (const s of model.parsed.spans || []) addSpan(spanMap, s.file, s.start, s.end); for (const f of model.parsed.files || []) addSpan(spanMap, f, 1, 1); } + const predFiles = [...spanMap.keys()].slice(0, 20); const predSpans = Object.fromEntries([...spanMap.entries()].slice(0, 20)); + const predPath = join(runDir, 'prediction.json'); writeFileSync(predPath, JSON.stringify({ instance_id: task.instance_id, repo_url: task.repo_checkout_path, commit: task.base_commit, traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans }, model_patch: '' }, null, 2)); + const goldPath = join(runDir, 'gold.json'); const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', process.env.TASK_PAYLOADS], { timeoutMs: 600000 }); writeFileSync(join(runDir, 'gold-command.json'), JSON.stringify(gold, null, 2)); + const scorePath = join(runDir, 'official-score.jsonl'); const evaluator = model.ok && predFiles.length ? run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predPath, '--cache', join(runDir, 'repo-cache'), '--out', scorePath], { cwd: process.env.ROOT + '/ContextBench-official', timeoutMs: 1200000 }) : { status: null, error: 'skipped_no_model_prediction', durationMs: 0 }; writeFileSync(join(runDir, 'evaluator-command.json'), JSON.stringify(evaluator, null, 2)); + let score = null; if (existsSync(scorePath)) { const lines = readFileSync(scorePath, 'utf8').trim().split(/\n+/).filter(Boolean); if (lines.length) score = JSON.parse(lines.at(-1)); } + const row = { lane_id: 'codebase-memory-mcp', task_id: task.instance_id, model: `${process.env.OPENAI_MODEL}-${process.env.OPENAI_REASONING_EFFORT}`, status: evaluator.status === 0 && score ? 'completed' : (model.status || 'judge_failed'), setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', indexStatus: index.status === 0 ? 'completed' : 'index_failed', toolCallable: graph.status === 0 || code.status === 0, candidateCount: candidates.length, nonEmptyPrediction: predFiles.length > 0, predFiles: predFiles.length, officialEvaluatorScoreable: evaluator.status === 0 && Boolean(score), setupIndex: { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: graph.durationMs + code.durationMs }, modelStatus: model.status, modelUsage: model.usage || null, score }; + const summary = { createdAt: new Date().toISOString(), attemptedRows: 1, scoreableRows: row.officialEvaluatorScoreable ? 1 : 0, setupIndexCostReportedSeparately: true, resultsTable: row.officialEvaluatorScoreable ? [{ lane: row.lane_id, fileCoverage: score.final.file.coverage, filePrecision: score.final.file.precision, symbolCoverage: score.final.symbol.coverage, spanCoverage: score.final.span.coverage, lineCoverage: score.final.line.coverage, editlocRecall: score.editloc?.recall ?? null }] : [], rows: [row] }; + writeFileSync(join(runDir, 'row.json'), JSON.stringify(row, null, 2)); writeFileSync(join(root, 'summary.json'), JSON.stringify(summary, null, 2)); console.log(JSON.stringify(summary, null, 2)); if (!row.officialEvaluatorScoreable) process.exitCode = 1; + NODE + node "$ROOT/one.mjs" + - name: Upload one-row artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-real-cbm-go-one + path: /tmp/contextbench-real-cbm-go-one + retention-days: 14 From d6414b38ef837563d2e5e9337388106edb90ec17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:35:41 +0200 Subject: [PATCH 012/100] Add authenticated one-row ContextBench run --- .../contextbench-real-gpt54mini-auth-one.yml | 174 ++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 .github/workflows/contextbench-real-gpt54mini-auth-one.yml diff --git a/.github/workflows/contextbench-real-gpt54mini-auth-one.yml b/.github/workflows/contextbench-real-gpt54mini-auth-one.yml new file mode 100644 index 0000000..0ac57cc --- /dev/null +++ b/.github/workflows/contextbench-real-gpt54mini-auth-one.yml @@ -0,0 +1,174 @@ +name: ContextBench Real GPT54 Mini Auth One + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-real-gpt54mini-auth-one.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + cbm-go-auth-one: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + ROOT: /tmp/contextbench-real-gpt54mini-auth-one + TASK_PAYLOADS: /tmp/contextbench-real-gpt54mini-auth-one/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + OPENAI_MODEL: gpt-5.4-mini + OPENAI_REASONING_EFFORT: high + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || secrets.CODEX_OPENAI_API_KEY || secrets.OPENAI_KEY || secrets.OPENAI_API_KEY_GPT54 || secrets.CONTEXTBENCH_OPENAI_API_KEY }} + CBM_BIN: /tmp/contextbench-real-gpt54mini-auth-one/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Check model auth + run: | + if [ -z "$OPENAI_API_KEY" ]; then + echo "::error::Missing model auth. Set one Actions secret: OPENAI_API_KEY, CODEX_OPENAI_API_KEY, OPENAI_KEY, OPENAI_API_KEY_GPT54, or CONTEXTBENCH_OPENAI_API_KEY." + exit 78 + fi + + - name: Install and materialize frozen Go task + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + + - name: Run authenticated scoreable CBM row + env: + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-real-gpt54mini-auth-one/ContextBench-official + run: | + cat > "$ROOT/run.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + + const root = process.env.ROOT; + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const task = payloads.tasks[2]; + const runDir = join(root, 'row-codebase-memory-mcp-go'); + mkdirSync(runDir, { recursive: true }); + + function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 600000, + maxBuffer: 128 * 1024 * 1024 + }); + return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; + } + function queryOf(text) { return String(text || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4).slice(0, 10).join(' '); } + function jsonish(text) { const raw = String(text || '').trim(); if (!raw) return null; try { return JSON.parse(raw); } catch {} for (const [a,b] of [['{','}'],['[',']']]) { const i = raw.indexOf(a), j = raw.lastIndexOf(b); if (i >= 0 && j > i) { try { return JSON.parse(raw.slice(i, j + 1)); } catch {} } } return null; } + function norm(file) { return String(file || '').replace(/^\/+/, '').replace(/^\.\//, ''); } + function add(locs, file, start = 1, end = start, source = 'cbm') { const clean = norm(file); if (!clean || clean.includes('..') || clean.includes('://')) return; const s = Math.max(1, Number(start) || 1); locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); } + function walk(value, locs) { if (!value || typeof value !== 'object') return; if (Array.isArray(value)) { for (const item of value) walk(item, locs); return; } add(locs, value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path, value.start_line || value.line || 1, value.end_line || value.line || 1); for (const item of Object.values(value)) walk(item, locs); } + function collect(text, locs) { const parsed = jsonish(text); if (parsed) walk(parsed, locs); const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1); } + function uniq(locs) { const seen = new Set(), out = []; for (const loc of locs) { const key = `${loc.file}:${loc.start}:${loc.end}`; if (!seen.has(key)) { seen.add(key); out.push(loc); if (out.length >= 80) break; } } return out; } + function addSpan(map, file, start = 1, end = start) { const clean = norm(file); if (!clean) return; const s = Math.max(1, Number(start) || 1); const e = Math.max(s, Number(end) || s); const list = map.get(clean) || []; list.push({ start: s, end: e }); map.set(clean, list); } + + async function askModel(candidates, query) { + const started = Date.now(); + const candidateFiles = new Set(candidates.map((c) => c.file)); + const body = { + model: process.env.OPENAI_MODEL, + reasoning: { effort: process.env.OPENAI_REASONING_EFFORT }, + max_output_tokens: 1200, + instructions: 'Select likely ContextBench edit locations using only the provided codebase-memory-mcp candidate locations. Return JSON only.', + input: JSON.stringify({ taskId: task.instance_id, repo: task.repo, lane: 'codebase-memory-mcp', query, problemStatement: task.problem_statement, candidateLocations: candidates.slice(0, 60) }), + text: { format: { type: 'json_schema', name: 'contextbench_selection', strict: true, schema: { type: 'object', additionalProperties: false, required: ['files','spans','notes'], properties: { files: { type: 'array', maxItems: 20, items: { type: 'string' } }, spans: { type: 'array', maxItems: 40, items: { type: 'object', additionalProperties: false, required: ['file','start','end'], properties: { file: { type: 'string' }, start: { type: 'integer', minimum: 1 }, end: { type: 'integer', minimum: 1 } } } }, notes: { type: 'string' } } } } } + }; + writeFileSync(join(runDir, 'openai-request.redacted.json'), JSON.stringify({ ...body, input: JSON.parse(body.input) }, null, 2)); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 300000); + try { + const res = await fetch('https://api.openai.com/v1/responses', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${process.env.OPENAI_API_KEY}` }, body: JSON.stringify(body), signal: controller.signal }); + const text = await res.text(); + clearTimeout(timer); + writeFileSync(join(runDir, 'openai-response.json'), text); + if (!res.ok) return { ok: false, status: 'model_error', httpStatus: res.status, error: text.slice(0, 2000), durationMs: Date.now() - started }; + const json = JSON.parse(text); + const output = json.output_text || (json.output || []).flatMap((item) => item.content || []).filter((item) => item.type === 'output_text').map((item) => item.text).join('\n'); + const parsed = JSON.parse(output); + const files = [...new Set((parsed.files || []).map(norm).filter((file) => candidateFiles.has(file)))].slice(0, 20); + const spans = (parsed.spans || []).map((span) => ({ file: norm(span.file), start: span.start, end: span.end })).filter((span) => candidateFiles.has(span.file)).slice(0, 40); + if (files.length === 0 && spans.length === 0) return { ok: false, status: 'model_empty_after_lane_filter', parsed, durationMs: Date.now() - started }; + return { ok: true, status: 'completed', parsed: { files, spans, notes: parsed.notes || '' }, usage: json.usage || null, durationMs: Date.now() - started }; + } catch (error) { + clearTimeout(timer); + return { ok: false, status: 'model_timeout_or_error', error: String(error?.message || error), durationMs: Date.now() - started }; + } + } + + const query = queryOf(task.problem_statement); + const env = { ...process.env, CBM_CACHE_DIR: join(runDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { cwd: task.repo_checkout_path, env, timeoutMs: 2700000 }); + const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(task.repo_checkout_path); + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], { cwd: task.repo_checkout_path, env, timeoutMs: 120000 }); + const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 25 })], { cwd: task.repo_checkout_path, env, timeoutMs: 120000 }); + for (const [name, value] of Object.entries({ setup, index, graph, code })) writeFileSync(join(runDir, `${name}.json`), JSON.stringify(value, null, 2)); + + const locs = []; + for (const result of [graph, code]) { collect(result.stdout, locs); collect(result.stderr, locs); } + const candidates = uniq(locs); + writeFileSync(join(runDir, 'candidate-locations.json'), JSON.stringify(candidates, null, 2)); + + const model = await askModel(candidates, query); + writeFileSync(join(runDir, 'model-result.json'), JSON.stringify(model, null, 2)); + const spanMap = new Map(); + if (model.ok) { for (const span of model.parsed.spans || []) addSpan(spanMap, span.file, span.start, span.end); for (const file of model.parsed.files || []) addSpan(spanMap, file, 1, 1); } + const predFiles = [...spanMap.keys()].slice(0, 20); + const predSpans = Object.fromEntries([...spanMap.entries()].slice(0, 20)); + const predictionPath = join(runDir, 'prediction.json'); + writeFileSync(predictionPath, JSON.stringify({ instance_id: task.instance_id, repo_url: task.repo_checkout_path, commit: task.base_commit, traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans }, model_patch: '' }, null, 2)); + + const goldPath = join(runDir, 'gold.json'); + const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', process.env.TASK_PAYLOADS], { timeoutMs: 600000 }); + writeFileSync(join(runDir, 'gold-command.json'), JSON.stringify(gold, null, 2)); + const scorePath = join(runDir, 'official-score.jsonl'); + const evaluator = model.ok && predFiles.length ? run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predictionPath, '--cache', join(runDir, 'repo-cache'), '--out', scorePath], { cwd: process.env.OFFICIAL_CONTEXTBENCH, timeoutMs: 1200000 }) : { status: null, error: 'skipped_no_model_prediction', durationMs: 0 }; + writeFileSync(join(runDir, 'evaluator-command.json'), JSON.stringify(evaluator, null, 2)); + let score = null; + if (existsSync(scorePath)) { const lines = readFileSync(scorePath, 'utf8').trim().split(/\n+/).filter(Boolean); if (lines.length) score = JSON.parse(lines.at(-1)); } + + const row = { lane_id: 'codebase-memory-mcp', task_id: task.instance_id, model: `${process.env.OPENAI_MODEL}-${process.env.OPENAI_REASONING_EFFORT}`, status: evaluator.status === 0 && score ? 'completed' : (model.status || 'judge_failed'), setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', indexStatus: index.status === 0 ? 'completed' : 'index_failed', toolCallable: graph.status === 0 || code.status === 0, candidateCount: candidates.length, nonEmptyPrediction: predFiles.length > 0, predFiles: predFiles.length, officialEvaluatorScoreable: evaluator.status === 0 && Boolean(score), setupIndex: { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: graph.durationMs + code.durationMs }, modelStatus: model.status, modelUsage: model.usage || null, score }; + const summary = { createdAt: new Date().toISOString(), attemptedRows: 1, scoreableRows: row.officialEvaluatorScoreable ? 1 : 0, setupIndexCostReportedSeparately: true, resultsTable: row.officialEvaluatorScoreable ? [{ lane: row.lane_id, fileCoverage: score.final.file.coverage, filePrecision: score.final.file.precision, symbolCoverage: score.final.symbol.coverage, spanCoverage: score.final.span.coverage, lineCoverage: score.final.line.coverage, editlocRecall: score.editloc?.recall ?? null }] : [], rows: [row] }; + writeFileSync(join(runDir, 'row.json'), JSON.stringify(row, null, 2)); + writeFileSync(join(root, 'summary.json'), JSON.stringify(summary, null, 2)); + console.log(JSON.stringify(summary, null, 2)); + if (!row.officialEvaluatorScoreable) process.exitCode = 1; + NODE + node "$ROOT/run.mjs" + + - name: Upload authenticated one-row artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-real-gpt54mini-auth-one + path: /tmp/contextbench-real-gpt54mini-auth-one + retention-days: 14 From 3cbb64b79084653aae48f500217f7dfe16dc0fbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:38:21 +0200 Subject: [PATCH 013/100] Try ContextBench auth from Actions variables --- ...textbench-real-gpt54mini-auth-vars-one.yml | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/contextbench-real-gpt54mini-auth-vars-one.yml diff --git a/.github/workflows/contextbench-real-gpt54mini-auth-vars-one.yml b/.github/workflows/contextbench-real-gpt54mini-auth-vars-one.yml new file mode 100644 index 0000000..cb6f14d --- /dev/null +++ b/.github/workflows/contextbench-real-gpt54mini-auth-vars-one.yml @@ -0,0 +1,82 @@ +name: ContextBench Real GPT54 Mini Auth Vars One + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-real-gpt54mini-auth-vars-one.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + cbm-go-auth-vars-one: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + ROOT: /tmp/contextbench-real-gpt54mini-auth-vars-one + TASK_PAYLOADS: /tmp/contextbench-real-gpt54mini-auth-vars-one/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + OPENAI_MODEL: gpt-5.4-mini + OPENAI_REASONING_EFFORT: high + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || vars.OPENAI_API_KEY || secrets.CODEX_OPENAI_API_KEY || vars.CODEX_OPENAI_API_KEY || secrets.OPENAI_KEY || vars.OPENAI_KEY || secrets.OPENAI_API_KEY_GPT54 || vars.OPENAI_API_KEY_GPT54 || secrets.CONTEXTBENCH_OPENAI_API_KEY || vars.CONTEXTBENCH_OPENAI_API_KEY }} + CBM_BIN: /tmp/contextbench-real-gpt54mini-auth-vars-one/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Check model auth from secrets or vars + run: | + if [ -z "$OPENAI_API_KEY" ]; then + echo "::error::Missing model auth in Actions secrets and variables. Set OPENAI_API_KEY or one of the checked aliases." + exit 78 + fi + + - name: Install and materialize frozen Go task + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + + - name: Run same authenticated scoreable CBM row script + env: + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-real-gpt54mini-auth-vars-one/ContextBench-official + run: | + python - <<'PY' + from pathlib import Path + text = Path('.github/workflows/contextbench-real-gpt54mini-auth-one.yml').read_text() + marker = ' cat > "$ROOT/run.mjs" <<\'NODE\'\n' + start = text.index(marker) + len(marker) + end = text.index(' NODE\n node "$ROOT/run.mjs"', start) + lines = [] + for line in text[start:end].splitlines(): + lines.append(line[10:] if line.startswith(' ') else line) + Path('/tmp/contextbench-real-gpt54mini-auth-vars-one/run.mjs').write_text('\n'.join(lines) + '\n') + PY + node "$ROOT/run.mjs" + + - name: Upload authenticated vars one-row artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-real-gpt54mini-auth-vars-one + path: /tmp/contextbench-real-gpt54mini-auth-vars-one + retention-days: 14 From 4b00271ed113bd4ff4d8bcd71d59711b55ef77d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:39:34 +0200 Subject: [PATCH 014/100] Probe GitHub Models for ContextBench model auth --- .../contextbench-github-models-probe.yml | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/contextbench-github-models-probe.yml diff --git a/.github/workflows/contextbench-github-models-probe.yml b/.github/workflows/contextbench-github-models-probe.yml new file mode 100644 index 0000000..b4693a6 --- /dev/null +++ b/.github/workflows/contextbench-github-models-probe.yml @@ -0,0 +1,66 @@ +name: ContextBench GitHub Models Probe + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-github-models-probe.yml + workflow_dispatch: + +permissions: + contents: read + models: read + +jobs: + probe: + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + ROOT: /tmp/contextbench-github-models-probe + steps: + - name: Probe model catalog and smoke calls + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p "$ROOT" + api=https://models.github.ai + curl -sS -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "X-GitHub-Api-Version: 2026-03-10" \ + "$api/catalog/models" > "$ROOT/catalog.json" + node - <<'NODE' + const fs = require('fs'); + const raw = fs.readFileSync(process.env.ROOT + '/catalog.json', 'utf8'); + let json; + try { json = JSON.parse(raw); } catch (error) { console.error(raw.slice(0, 2000)); throw error; } + const list = Array.isArray(json) ? json : (json.models || json.data || []); + const ids = list.map((m) => m.id || m.name || m.model).filter(Boolean); + const interesting = ids.filter((id) => /openai|gpt-5|gpt-5\.4|mini/i.test(id)); + fs.writeFileSync(process.env.ROOT + '/interesting-models.json', JSON.stringify(interesting, null, 2)); + console.log(JSON.stringify({ total: ids.length, interesting: interesting.slice(0, 80) }, null, 2)); + NODE + for model in openai/gpt-5.4-mini openai/gpt-5-mini gpt-5.4-mini gpt-5-mini; do + safe=$(echo "$model" | tr '/.' '__') + echo "Trying $model" + code=$(curl -sS -L -o "$ROOT/smoke-$safe.json" -w '%{http_code}' \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "X-GitHub-Api-Version: 2026-03-10" \ + -H "Content-Type: application/json" \ + "$api/inference/chat/completions" \ + -d "{\"model\":\"$model\",\"messages\":[{\"role\":\"user\",\"content\":\"Return JSON: {\\\"ok\\\": true}\"}],\"response_format\":{\"type\":\"json_object\"},\"max_tokens\":32}") || code=000 + echo "$model $code" | tee -a "$ROOT/smoke-status.txt" + done + if grep -E ' 2[0-9][0-9]$' "$ROOT/smoke-status.txt"; then + exit 0 + fi + exit 1 + - name: Upload GitHub Models probe artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-github-models-probe + path: /tmp/contextbench-github-models-probe + retention-days: 14 From ba7f2f34c61a8ea5f3c7990d5ce20e38483b9cd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:42:32 +0200 Subject: [PATCH 015/100] Generate ContextBench candidate pack for one task --- .../contextbench-candidate-pack-one.yml | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 .github/workflows/contextbench-candidate-pack-one.yml diff --git a/.github/workflows/contextbench-candidate-pack-one.yml b/.github/workflows/contextbench-candidate-pack-one.yml new file mode 100644 index 0000000..a35f0c5 --- /dev/null +++ b/.github/workflows/contextbench-candidate-pack-one.yml @@ -0,0 +1,127 @@ +name: ContextBench Candidate Pack One + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-candidate-pack-one.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + pack: + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + ROOT: /tmp/contextbench-candidate-pack-one + TASK_PAYLOADS: /tmp/contextbench-candidate-pack-one/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + CBM_BIN: /tmp/contextbench-candidate-pack-one/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install tools and materialize frozen Go task + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + pnpm run build + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow uv codegraphcontext kuzu + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh || true + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + - name: Generate candidate pack + run: | + cat > "$ROOT/pack.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, relative, sep } from 'node:path'; + const root = process.env.ROOT; + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const task = payloads.tasks[2]; + const repo = task.repo_checkout_path; + const outDir = root + '/pack'; + mkdirSync(outDir, { recursive: true }); + function run(cmd, args, opts = {}) { const started = Date.now(); const r = spawnSync(cmd, args, { cwd: opts.cwd || process.cwd(), env: opts.env || process.env, encoding: 'utf8', timeout: opts.timeoutMs || 300000, maxBuffer: 96 * 1024 * 1024 }); return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; } + function queryOf(text) { const stop = new Set(['that','this','with','from','when','then','into','should','would','could','there','where','which','about','after','before','have','will','been','than','also']); return String(text || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())).slice(0, 12).join(' '); } + function jsonish(text) { const raw = String(text || '').trim(); if (!raw) return null; try { return JSON.parse(raw); } catch {} for (const [a,b] of [['{','}'],['[',']']]) { const i = raw.indexOf(a), j = raw.lastIndexOf(b); if (i >= 0 && j > i) { try { return JSON.parse(raw.slice(i, j + 1)); } catch {} } } return null; } + function norm(file) { let f = String(file || '').replace(/^file:\/\//, ''); if (!f) return ''; if (f.startsWith(repo)) f = relative(repo, f); f = f.replaceAll('\\\\', '/').replace(/^\/+/, '').replace(/^\.\//, ''); if (f.startsWith('tmp/') || f.includes('://') || f.includes('..')) return ''; return f; } + function add(locs, file, start = 1, end = start, source = 'tool') { const clean = norm(file); if (!clean) return; const s = Math.max(1, Number(start) || 1); locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); } + function walk(value, locs, source) { if (!value || typeof value !== 'object') return; if (Array.isArray(value)) { for (const item of value) walk(item, locs, source); return; } add(locs, value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, value.start_line || value.startLine || value.line || value.line_number || value.start || 1, value.end_line || value.endLine || value.end || value.line || 1, source); for (const item of Object.values(value)) walk(item, locs, source); } + function collect(text, locs, source) { const parsed = jsonish(text); if (parsed) walk(parsed, locs, source); const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); } + function uniq(locs, max = 60) { const seen = new Set(), out = []; for (const loc of locs) { const key = `${loc.file}:${loc.start}:${loc.end}`; if (!seen.has(key)) { seen.add(key); out.push(loc); if (out.length >= max) break; } } return out; } + function laneResult(lane, commands, locs, setupStatus = 'completed', indexStatus = 'completed') { for (const [i,c] of commands.entries()) writeFileSync(`${outDir}/${lane}-command-${i + 1}.json`, JSON.stringify(c, null, 2)); return { lane, setupStatus, indexStatus, toolCallable: commands.some((c) => c.status === 0), costs: { totalMs: commands.reduce((a,c) => a + c.durationMs, 0) }, candidates: uniq(locs) }; } + const query = queryOf(task.problem_statement); + const lanes = []; + { + const commands = [], locs = []; + for (const term of query.split(/\s+/).slice(0, 6)) { const r = run('rg', ['-n', '-i', '--glob', '!.git', '--glob', '!vendor/**', '--glob', '!node_modules/**', term, '.'], { cwd: repo, timeoutMs: 60000 }); commands.push(r); collect(r.stdout, locs, 'raw-native'); collect(r.stderr, locs, 'raw-native'); } + lanes.push(laneResult('raw-native', commands, locs)); + } + { + const commands = [], locs = []; const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' }; + const v = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60000 }); commands.push(v); + const idx = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 1200000 }); commands.push(idx); + const s = run('node', ['dist/index.js', 'search', '--query', query, '--intent', 'edit', '--limit', '30', '--json'], { env, timeoutMs: 300000 }); commands.push(s); collect(s.stdout, locs, 'codebase-context'); collect(s.stderr, locs, 'codebase-context'); + lanes.push(laneResult('codebase-context', commands, locs, 'completed', idx.status === 0 ? 'completed' : 'index_failed')); + } + { + const commands = [], locs = []; const env = { ...process.env, CBM_CACHE_DIR: outDir + '/cbm-cache', CBM_DIAGNOSTICS: '1' }; + const v = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); commands.push(v); + const idx = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 2700000 }); commands.push(idx); + const project = (jsonish(idx.stdout) || jsonish(idx.stderr) || {}).project || basename(repo); + const g = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 30 })], { cwd: repo, env, timeoutMs: 120000 }); commands.push(g); + const c = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 30 })], { cwd: repo, env, timeoutMs: 120000 }); commands.push(c); + for (const r of [g, c]) { collect(r.stdout, locs, 'codebase-memory-mcp'); collect(r.stderr, locs, 'codebase-memory-mcp'); } + lanes.push(laneResult('codebase-memory-mcp', commands, locs, 'completed', idx.status === 0 ? 'completed' : 'index_failed')); + } + { + const commands = [], locs = []; + const v = run('grepai', ['version'], { timeoutMs: 60000 }); commands.push(v); + const init = run('grepai', ['init', '--yes', '--provider', 'synthetic', '--backend', 'gob'], { cwd: repo, timeoutMs: 120000 }); commands.push(init); + const watch = run('grepai', ['watch', '--background'], { cwd: repo, timeoutMs: 120000 }); commands.push(watch); + const status = run('grepai', ['watch', '--status'], { cwd: repo, timeoutMs: 60000 }); commands.push(status); + const s = run('grepai', ['search', query, '--json', '--compact'], { cwd: repo, timeoutMs: 180000 }); commands.push(s); collect(s.stdout, locs, 'grepai'); collect(s.stderr, locs, 'grepai'); + commands.push(run('grepai', ['watch', '--stop'], { cwd: repo, timeoutMs: 60000 })); + lanes.push(laneResult('grepai', commands, locs, init.status === 0 ? 'completed' : 'setup_failed', (watch.status === 0 || status.status === 0) ? 'completed' : 'index_failed')); + } + { + const commands = [], locs = []; + const v = run('cgc', ['--help'], { timeoutMs: 60000 }); commands.push(v); + const idx = run('cgc', ['index', repo], { cwd: repo, timeoutMs: 1200000 }); commands.push(idx); + const a = run('cgc', ['analyze', 'complexity', '--limit', '30'], { cwd: repo, timeoutMs: 180000 }); commands.push(a); + const d = run('cgc', ['analyze', 'dead-code', '--limit', '30'], { cwd: repo, timeoutMs: 180000 }); commands.push(d); + const call = run('cgc', ['analyze', 'callers', query.split(/\s+/)[0] || 'main'], { cwd: repo, timeoutMs: 120000 }); commands.push(call); + for (const r of [a, d, call]) { collect(r.stdout, locs, 'codegraphcontext'); collect(r.stderr, locs, 'codegraphcontext'); } + lanes.push(laneResult('codegraphcontext', commands, locs, 'completed', idx.status === 0 ? 'completed' : 'index_failed')); + } + const pack = { createdAt: new Date().toISOString(), task: { instance_id: task.instance_id, repo: task.repo, base_commit: task.base_commit, problem_statement: task.problem_statement }, query, lanes }; + writeFileSync(`${outDir}/candidate-pack.json`, JSON.stringify(pack, null, 2)); + console.log(JSON.stringify({ task: pack.task, query, lanes: lanes.map((l) => ({ lane: l.lane, setupStatus: l.setupStatus, indexStatus: l.indexStatus, toolCallable: l.toolCallable, candidateCount: l.candidates.length, candidates: l.candidates.slice(0, 30) })) }, null, 2)); + NODE + node "$ROOT/pack.mjs" + - name: Upload candidate pack + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-candidate-pack-one + path: /tmp/contextbench-candidate-pack-one/pack + retention-days: 14 From f4e5bf9754849e36d20b79f4ecbe55de34c988a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:44:30 +0200 Subject: [PATCH 016/100] Generate CBM candidate pack for ContextBench --- .../contextbench-cbm-candidates-one.yml | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 .github/workflows/contextbench-cbm-candidates-one.yml diff --git a/.github/workflows/contextbench-cbm-candidates-one.yml b/.github/workflows/contextbench-cbm-candidates-one.yml new file mode 100644 index 0000000..fd475df --- /dev/null +++ b/.github/workflows/contextbench-cbm-candidates-one.yml @@ -0,0 +1,90 @@ +name: ContextBench CBM Candidates One + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-cbm-candidates-one.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + cbm-candidates: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + ROOT: /tmp/contextbench-cbm-candidates-one + TASK_PAYLOADS: /tmp/contextbench-cbm-candidates-one/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + CBM_BIN: /tmp/contextbench-cbm-candidates-one/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install and materialize Go task + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + - name: Print CBM candidates + run: | + cat > "$ROOT/cbm-pack.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { basename, relative } from 'node:path'; + import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + const root = process.env.ROOT; + const outDir = root + '/pack'; + mkdirSync(outDir, { recursive: true }); + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const task = payloads.tasks[2]; + const repo = task.repo_checkout_path; + function run(cmd, args, opts = {}) { const started = Date.now(); const r = spawnSync(cmd, args, { cwd: opts.cwd || process.cwd(), env: opts.env || process.env, encoding: 'utf8', timeout: opts.timeoutMs || 600000, maxBuffer: 128 * 1024 * 1024 }); return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; } + function q(s) { return String(s || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4).slice(0, 10).join(' '); } + function jsonish(s) { const t = String(s || '').trim(); if (!t) return null; try { return JSON.parse(t); } catch {} for (const [a,b] of [['{','}'],['[',']']]) { const i=t.indexOf(a), j=t.lastIndexOf(b); if (i>=0 && j>i) { try { return JSON.parse(t.slice(i,j+1)); } catch {} } } return null; } + function norm(file) { let f = String(file || '').replace(/^file:\/\//, ''); if (!f) return ''; if (f.startsWith(repo)) f = relative(repo, f); f = f.replaceAll('\\\\','/').replace(/^\/+/, '').replace(/^\.\//, ''); if (!f || f.startsWith('tmp/') || f.includes('://') || f.includes('..')) return ''; return f; } + function add(locs, file, start = 1, end = start) { const clean = norm(file); if (!clean) return; const s = Math.max(1, Number(start) || 1); locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source: 'codebase-memory-mcp' }); } + function walk(v, locs) { if (!v || typeof v !== 'object') return; if (Array.isArray(v)) { for (const x of v) walk(x, locs); return; } add(locs, v.file || v.path || v.file_path || v.relative_path || v.filename || v.source_path, v.start_line || v.line || 1, v.end_line || v.line || 1); for (const x of Object.values(v)) walk(x, locs); } + function collect(text, locs) { const parsed = jsonish(text); if (parsed) walk(parsed, locs); const re = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1); } + function uniq(locs) { const seen = new Set(), out = []; for (const loc of locs) { const k = `${loc.file}:${loc.start}:${loc.end}`; if (!seen.has(k)) { seen.add(k); out.push(loc); if (out.length >= 80) break; } } return out; } + const query = q(task.problem_statement); + const env = { ...process.env, CBM_CACHE_DIR: outDir + '/cbm-cache', CBM_DIAGNOSTICS: '1' }; + const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 2700000 }); + const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo); + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 40 })], { cwd: repo, env, timeoutMs: 120000 }); + const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 40 })], { cwd: repo, env, timeoutMs: 120000 }); + for (const [name, value] of Object.entries({ setup, index, graph, code })) writeFileSync(`${outDir}/${name}.json`, JSON.stringify(value, null, 2)); + const locs = []; for (const r of [graph, code]) { collect(r.stdout, locs); collect(r.stderr, locs); } + const candidates = uniq(locs); + const pack = { task: { instance_id: task.instance_id, repo: task.repo, base_commit: task.base_commit, problem_statement: task.problem_statement }, query, lane: { lane: 'codebase-memory-mcp', setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', indexStatus: index.status === 0 ? 'completed' : 'index_failed', toolCallable: graph.status === 0 || code.status === 0, candidateCount: candidates.length, setupIndex: { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: graph.durationMs + code.durationMs }, candidates } }; + writeFileSync(`${outDir}/cbm-candidate-pack.json`, JSON.stringify(pack, null, 2)); + console.log('CBM_CANDIDATE_PACK_JSON_START'); + console.log(JSON.stringify({ ...pack, lane: { ...pack.lane, candidates: candidates.slice(0, 60) } }, null, 2)); + console.log('CBM_CANDIDATE_PACK_JSON_END'); + NODE + node "$ROOT/cbm-pack.mjs" + - name: Upload CBM candidate pack + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-cbm-candidates-one + path: /tmp/contextbench-cbm-candidates-one/pack + retention-days: 14 From 3d6ac2756e5d34fc09f9fe3b4b15e90652f3c282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:49:10 +0200 Subject: [PATCH 017/100] Make CBM ContextBench candidate pack logs compact --- .../contextbench-cbm-candidates-one.yml | 131 +++++++++++++----- 1 file changed, 100 insertions(+), 31 deletions(-) diff --git a/.github/workflows/contextbench-cbm-candidates-one.yml b/.github/workflows/contextbench-cbm-candidates-one.yml index fd475df..8eee8e4 100644 --- a/.github/workflows/contextbench-cbm-candidates-one.yml +++ b/.github/workflows/contextbench-cbm-candidates-one.yml @@ -31,53 +31,122 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Install and materialize Go task + - name: Install and materialize Go task quietly + shell: bash run: | - set -euxo pipefail - mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" - pnpm install --frozen-lockfile - python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/pack" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-install.log" 2>&1 curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" - tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" > "$ROOT/logs/cbm-extract.log" 2>&1 chmod +x "$CBM_BIN" || true - git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" - node scripts/contextbench-runner.mjs --validate-fixtures - node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" - node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 - - name: Print CBM candidates + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" > "$ROOT/logs/contextbench-clone.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 > "$ROOT/logs/materialize.log" 2>&1 + echo "quiet_setup_completed" + - name: Print compact CBM candidates + shell: bash run: | cat > "$ROOT/cbm-pack.mjs" <<'NODE' import { spawnSync } from 'node:child_process'; - import { basename, relative } from 'node:path'; - import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, join, relative } from 'node:path'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; const root = process.env.ROOT; - const outDir = root + '/pack'; + const outDir = join(root, 'pack'); mkdirSync(outDir, { recursive: true }); const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); const task = payloads.tasks[2]; const repo = task.repo_checkout_path; - function run(cmd, args, opts = {}) { const started = Date.now(); const r = spawnSync(cmd, args, { cwd: opts.cwd || process.cwd(), env: opts.env || process.env, encoding: 'utf8', timeout: opts.timeoutMs || 600000, maxBuffer: 128 * 1024 * 1024 }); return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; } - function q(s) { return String(s || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4).slice(0, 10).join(' '); } - function jsonish(s) { const t = String(s || '').trim(); if (!t) return null; try { return JSON.parse(t); } catch {} for (const [a,b] of [['{','}'],['[',']']]) { const i=t.indexOf(a), j=t.lastIndexOf(b); if (i>=0 && j>i) { try { return JSON.parse(t.slice(i,j+1)); } catch {} } } return null; } - function norm(file) { let f = String(file || '').replace(/^file:\/\//, ''); if (!f) return ''; if (f.startsWith(repo)) f = relative(repo, f); f = f.replaceAll('\\\\','/').replace(/^\/+/, '').replace(/^\.\//, ''); if (!f || f.startsWith('tmp/') || f.includes('://') || f.includes('..')) return ''; return f; } - function add(locs, file, start = 1, end = start) { const clean = norm(file); if (!clean) return; const s = Math.max(1, Number(start) || 1); locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source: 'codebase-memory-mcp' }); } - function walk(v, locs) { if (!v || typeof v !== 'object') return; if (Array.isArray(v)) { for (const x of v) walk(x, locs); return; } add(locs, v.file || v.path || v.file_path || v.relative_path || v.filename || v.source_path, v.start_line || v.line || 1, v.end_line || v.line || 1); for (const x of Object.values(v)) walk(x, locs); } - function collect(text, locs) { const parsed = jsonish(text); if (parsed) walk(parsed, locs); const re = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1); } - function uniq(locs) { const seen = new Set(), out = []; for (const loc of locs) { const k = `${loc.file}:${loc.start}:${loc.end}`; if (!seen.has(k)) { seen.add(k); out.push(loc); if (out.length >= 80) break; } } return out; } - const query = q(task.problem_statement); - const env = { ...process.env, CBM_CACHE_DIR: outDir + '/cbm-cache', CBM_DIAGNOSTICS: '1' }; + function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 600000, + maxBuffer: 128 * 1024 * 1024 + }); + return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; + } + function queryOf(text) { + const stop = new Set(['that','this','with','from','when','then','into','should','would','could','there','where','which','about','after','before','have','will','been','than','also','only','some','using']); + return String(text || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())).slice(0, 14).join(' '); + } + function jsonish(s) { + const t = String(s || '').trim(); + if (!t) return null; + try { return JSON.parse(t); } catch {} + for (const [a,b] of [['{','}'],['[',']']]) { + const i = t.indexOf(a), j = t.lastIndexOf(b); + if (i >= 0 && j > i) { try { return JSON.parse(t.slice(i, j + 1)); } catch {} } + } + return null; + } + function norm(file) { + let f = String(file || '').replace(/^file:\/\//, '').replaceAll('\\', '/'); + if (!f) return ''; + if (f.startsWith(repo)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..') || f.startsWith('tmp-contextbench') || f.includes('/tmp-contextbench')) return ''; + if (!existsSync(join(repo, f))) return ''; + return f; + } + function add(locs, file, start = 1, end = start, source = 'codebase-memory-mcp') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); + } + function walk(value, locs) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { for (const item of value) walk(item, locs); return; } + add(locs, value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, value.start_line || value.startLine || value.line || value.line_number || value.start || 1, value.end_line || value.endLine || value.end || value.line || 1); + for (const item of Object.values(value)) walk(item, locs); + } + function collect(text, locs) { + const parsed = jsonish(text); + if (parsed) walk(parsed, locs); + const re = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp))(?::|#L|\s+line\s+)?(\d+)?/g; + let m; + while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1); + } + function uniq(locs, max = 80) { + const seen = new Set(), out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (!seen.has(key)) { seen.add(key); out.push(loc); if (out.length >= max) break; } + } + return out; + } + const query = queryOf(task.problem_statement); + const env = { ...process.env, CBM_CACHE_DIR: join(outDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 2700000 }); const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo); - const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 40 })], { cwd: repo, env, timeoutMs: 120000 }); - const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 40 })], { cwd: repo, env, timeoutMs: 120000 }); - for (const [name, value] of Object.entries({ setup, index, graph, code })) writeFileSync(`${outDir}/${name}.json`, JSON.stringify(value, null, 2)); - const locs = []; for (const r of [graph, code]) { collect(r.stdout, locs); collect(r.stderr, locs); } + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 50 })], { cwd: repo, env, timeoutMs: 120000 }); + const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 50 })], { cwd: repo, env, timeoutMs: 120000 }); + for (const [name, value] of Object.entries({ setup, index, graph, code })) writeFileSync(join(outDir, `${name}.json`), JSON.stringify({ ...value, stdout: value.stdout.slice(0, 120000), stderr: value.stderr.slice(0, 120000) }, null, 2)); + const locs = []; + for (const r of [graph, code]) { collect(r.stdout, locs); collect(r.stderr, locs); } const candidates = uniq(locs); - const pack = { task: { instance_id: task.instance_id, repo: task.repo, base_commit: task.base_commit, problem_statement: task.problem_statement }, query, lane: { lane: 'codebase-memory-mcp', setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', indexStatus: index.status === 0 ? 'completed' : 'index_failed', toolCallable: graph.status === 0 || code.status === 0, candidateCount: candidates.length, setupIndex: { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: graph.durationMs + code.durationMs }, candidates } }; - writeFileSync(`${outDir}/cbm-candidate-pack.json`, JSON.stringify(pack, null, 2)); + const pack = { + task: { instance_id: task.instance_id, repo: task.repo, base_commit: task.base_commit, problem_statement: task.problem_statement }, + query, + lane: { + lane: 'codebase-memory-mcp', + setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', + indexStatus: index.status === 0 ? 'completed' : 'index_failed', + toolCallable: graph.status === 0 || code.status === 0, + candidateCount: candidates.length, + setupIndex: { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: graph.durationMs + code.durationMs }, + candidates + } + }; + writeFileSync(join(outDir, 'cbm-candidate-pack.json'), JSON.stringify(pack, null, 2)); console.log('CBM_CANDIDATE_PACK_JSON_START'); - console.log(JSON.stringify({ ...pack, lane: { ...pack.lane, candidates: candidates.slice(0, 60) } }, null, 2)); + console.log(JSON.stringify(pack, null, 2)); console.log('CBM_CANDIDATE_PACK_JSON_END'); NODE node "$ROOT/cbm-pack.mjs" From 2048feef033f2917f9a37469781e1112131c4c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:53:35 +0200 Subject: [PATCH 018/100] Extract CBM ContextBench candidates from script file --- scripts/contextbench-cbm-candidate-pack.mjs | 188 ++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 scripts/contextbench-cbm-candidate-pack.mjs diff --git a/scripts/contextbench-cbm-candidate-pack.mjs b/scripts/contextbench-cbm-candidate-pack.mjs new file mode 100644 index 0000000..2f32297 --- /dev/null +++ b/scripts/contextbench-cbm-candidate-pack.mjs @@ -0,0 +1,188 @@ +import { spawnSync } from 'node:child_process'; +import { basename, join, relative } from 'node:path'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + +const root = process.env.ROOT; +const outDir = join(root, 'pack'); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks[2]; +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 600000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: r.status, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function queryOf(text) { + const stop = new Set([ + 'that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', + 'would', 'could', 'there', 'where', 'which', 'about', 'after', + 'before', 'have', 'will', 'been', 'than', 'also', 'only', 'some', + 'using', 'must', 'method', 'methods', 'function', 'interface', + ]); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 18) + .join(' '); +} + +function jsonish(s) { + const t = String(s || '').trim(); + if (!t) return null; + try { + return JSON.parse(t); + } catch {} + for (const [a, b] of [['{', '}'], ['[', ']']]) { + const i = t.indexOf(a); + const j = t.lastIndexOf(b); + if (i >= 0 && j > i) { + try { + return JSON.parse(t.slice(i, j + 1)); + } catch {} + } + } + return null; +} + +function norm(file) { + let f = String(file || '').replace(/^file:\/\//, '').replaceAll('\\', '/'); + if (!f) return ''; + if (f.startsWith(repo)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if ( + !f || + f.includes('://') || + f.includes('..') || + f.startsWith('tmp-contextbench') || + f.includes('/tmp-contextbench') + ) { + return ''; + } + if (!existsSync(join(repo, f))) return ''; + return f; +} + +function add(locs, file, start = 1, end = start, source = 'codebase-memory-mcp') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function walk(value, locs) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs); + return; + } + add( + locs, + value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, + value.start_line || value.startLine || value.line || value.line_number || value.start || 1, + value.end_line || value.endLine || value.end || value.line || 1, + ); + for (const item of Object.values(value)) walk(item, locs); +} + +function collect(text, locs) { + const parsed = jsonish(text); + if (parsed) walk(parsed, locs); + const re = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp))(?::|#L|\s+line\s+)?(\d+)?/g; + let m; + while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1); +} + +function uniq(locs, max = 100) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (!seen.has(key)) { + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + } + return out; +} + +const query = queryOf(task.problem_statement); +const env = { ...process.env, CBM_CACHE_DIR: join(outDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; +const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); +const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { + cwd: repo, + env, + timeoutMs: 2700000, +}); +const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo); +const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 60 })], { + cwd: repo, + env, + timeoutMs: 120000, +}); +const code = run( + process.env.CBM_BIN, + ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 60 })], + { cwd: repo, env, timeoutMs: 120000 }, +); + +for (const [name, value] of Object.entries({ setup, index, graph, code })) { + writeFileSync( + join(outDir, `${name}.json`), + JSON.stringify({ ...value, stdout: value.stdout.slice(0, 120000), stderr: value.stderr.slice(0, 120000) }, null, 2), + ); +} + +const locs = []; +for (const r of [graph, code]) { + collect(r.stdout, locs); + collect(r.stderr, locs); +} +const candidates = uniq(locs); +const pack = { + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + query, + lane: { + lane: 'codebase-memory-mcp', + setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', + indexStatus: index.status === 0 ? 'completed' : 'index_failed', + toolCallable: graph.status === 0 || code.status === 0, + candidateCount: candidates.length, + setupIndex: { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: graph.durationMs + code.durationMs, + }, + candidates, + }, +}; + +writeFileSync(join(outDir, 'cbm-candidate-pack.json'), JSON.stringify(pack, null, 2)); +console.log('CBM_CANDIDATE_PACK_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CBM_CANDIDATE_PACK_JSON_END'); From 66a522bac97983c2d28a2a35537bad7d7ce1e963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:53:54 +0200 Subject: [PATCH 019/100] Call CBM ContextBench candidate extractor from script --- .../contextbench-cbm-candidates-one.yml | 104 +----------------- 1 file changed, 2 insertions(+), 102 deletions(-) diff --git a/.github/workflows/contextbench-cbm-candidates-one.yml b/.github/workflows/contextbench-cbm-candidates-one.yml index 8eee8e4..035e98b 100644 --- a/.github/workflows/contextbench-cbm-candidates-one.yml +++ b/.github/workflows/contextbench-cbm-candidates-one.yml @@ -5,6 +5,7 @@ on: branches: [master] paths: - .github/workflows/contextbench-cbm-candidates-one.yml + - scripts/contextbench-cbm-candidate-pack.mjs workflow_dispatch: permissions: @@ -48,108 +49,7 @@ jobs: echo "quiet_setup_completed" - name: Print compact CBM candidates shell: bash - run: | - cat > "$ROOT/cbm-pack.mjs" <<'NODE' - import { spawnSync } from 'node:child_process'; - import { basename, join, relative } from 'node:path'; - import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; - const root = process.env.ROOT; - const outDir = join(root, 'pack'); - mkdirSync(outDir, { recursive: true }); - const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); - const task = payloads.tasks[2]; - const repo = task.repo_checkout_path; - function run(cmd, args, opts = {}) { - const started = Date.now(); - const r = spawnSync(cmd, args, { - cwd: opts.cwd || process.cwd(), - env: opts.env || process.env, - encoding: 'utf8', - timeout: opts.timeoutMs || 600000, - maxBuffer: 128 * 1024 * 1024 - }); - return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; - } - function queryOf(text) { - const stop = new Set(['that','this','with','from','when','then','into','should','would','could','there','where','which','about','after','before','have','will','been','than','also','only','some','using']); - return String(text || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())).slice(0, 14).join(' '); - } - function jsonish(s) { - const t = String(s || '').trim(); - if (!t) return null; - try { return JSON.parse(t); } catch {} - for (const [a,b] of [['{','}'],['[',']']]) { - const i = t.indexOf(a), j = t.lastIndexOf(b); - if (i >= 0 && j > i) { try { return JSON.parse(t.slice(i, j + 1)); } catch {} } - } - return null; - } - function norm(file) { - let f = String(file || '').replace(/^file:\/\//, '').replaceAll('\\', '/'); - if (!f) return ''; - if (f.startsWith(repo)) f = relative(repo, f).replaceAll('\\', '/'); - f = f.replace(/^\/+/, '').replace(/^\.\//, ''); - if (!f || f.includes('://') || f.includes('..') || f.startsWith('tmp-contextbench') || f.includes('/tmp-contextbench')) return ''; - if (!existsSync(join(repo, f))) return ''; - return f; - } - function add(locs, file, start = 1, end = start, source = 'codebase-memory-mcp') { - const clean = norm(file); - if (!clean) return; - const s = Math.max(1, Number(start) || 1); - locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); - } - function walk(value, locs) { - if (!value || typeof value !== 'object') return; - if (Array.isArray(value)) { for (const item of value) walk(item, locs); return; } - add(locs, value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, value.start_line || value.startLine || value.line || value.line_number || value.start || 1, value.end_line || value.endLine || value.end || value.line || 1); - for (const item of Object.values(value)) walk(item, locs); - } - function collect(text, locs) { - const parsed = jsonish(text); - if (parsed) walk(parsed, locs); - const re = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp))(?::|#L|\s+line\s+)?(\d+)?/g; - let m; - while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1); - } - function uniq(locs, max = 80) { - const seen = new Set(), out = []; - for (const loc of locs) { - const key = `${loc.file}:${loc.start}:${loc.end}`; - if (!seen.has(key)) { seen.add(key); out.push(loc); if (out.length >= max) break; } - } - return out; - } - const query = queryOf(task.problem_statement); - const env = { ...process.env, CBM_CACHE_DIR: join(outDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; - const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); - const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 2700000 }); - const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo); - const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 50 })], { cwd: repo, env, timeoutMs: 120000 }); - const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 50 })], { cwd: repo, env, timeoutMs: 120000 }); - for (const [name, value] of Object.entries({ setup, index, graph, code })) writeFileSync(join(outDir, `${name}.json`), JSON.stringify({ ...value, stdout: value.stdout.slice(0, 120000), stderr: value.stderr.slice(0, 120000) }, null, 2)); - const locs = []; - for (const r of [graph, code]) { collect(r.stdout, locs); collect(r.stderr, locs); } - const candidates = uniq(locs); - const pack = { - task: { instance_id: task.instance_id, repo: task.repo, base_commit: task.base_commit, problem_statement: task.problem_statement }, - query, - lane: { - lane: 'codebase-memory-mcp', - setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', - indexStatus: index.status === 0 ? 'completed' : 'index_failed', - toolCallable: graph.status === 0 || code.status === 0, - candidateCount: candidates.length, - setupIndex: { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: graph.durationMs + code.durationMs }, - candidates - } - }; - writeFileSync(join(outDir, 'cbm-candidate-pack.json'), JSON.stringify(pack, null, 2)); - console.log('CBM_CANDIDATE_PACK_JSON_START'); - console.log(JSON.stringify(pack, null, 2)); - console.log('CBM_CANDIDATE_PACK_JSON_END'); - NODE - node "$ROOT/cbm-pack.mjs" + run: node scripts/contextbench-cbm-candidate-pack.mjs - name: Upload CBM candidate pack if: always() uses: actions/upload-artifact@v4 From aab3abe103fc7188029105381c4924c99308a671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:58:07 +0200 Subject: [PATCH 020/100] Add scorer for selected ContextBench prediction --- ...contextbench-score-selected-prediction.mjs | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 scripts/contextbench-score-selected-prediction.mjs diff --git a/scripts/contextbench-score-selected-prediction.mjs b/scripts/contextbench-score-selected-prediction.mjs new file mode 100644 index 0000000..ce0630e --- /dev/null +++ b/scripts/contextbench-score-selected-prediction.mjs @@ -0,0 +1,176 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const root = process.env.ROOT; +const officialContextBench = process.env.OFFICIAL_CONTEXTBENCH; +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks[2]; +const runDir = join(root, 'selected-codebase-memory-mcp'); +mkdirSync(runDir, { recursive: true }); + +const selection = { + files: ['core/metrics/prometheus.go', 'cmd/root.go', 'server/subsonic/helpers.go'], + spans: [ + { file: 'core/metrics/prometheus.go', start: 15, end: 17 }, + { file: 'core/metrics/prometheus.go', start: 19, end: 25 }, + { file: 'core/metrics/prometheus.go', start: 33, end: 38 }, + { file: 'core/metrics/prometheus.go', start: 40, end: 49 }, + { file: 'core/metrics/prometheus.go', start: 51, end: 100 }, + { file: 'core/metrics/prometheus.go', start: 102, end: 123 }, + { file: 'cmd/root.go', start: 194, end: 201 }, + { file: 'cmd/root.go', start: 204, end: 220 }, + { file: 'server/subsonic/helpers.go', start: 129, end: 187 }, + { file: 'server/subsonic/helpers.go', start: 189, end: 201 }, + { file: 'server/subsonic/helpers.go', start: 256, end: 268 }, + ], + rationale: + 'These spans cover the startup metrics writer path and the Subsonic auth-header/Bearer-token parsing path where the two reported behaviors most likely live.', +}; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 600000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: r.status, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function addSpan(map, file, start, end) { + const s = Math.max(1, Number(start) || 1); + const e = Math.max(s, Number(end) || s); + const list = map.get(file) || []; + list.push({ start: s, end: e }); + map.set(file, list); +} + +const spanMap = new Map(); +for (const span of selection.spans) addSpan(spanMap, span.file, span.start, span.end); +const predFiles = [...new Set([...selection.files, ...selection.spans.map((span) => span.file)])]; +const predSpans = Object.fromEntries(spanMap.entries()); +const prediction = { + instance_id: task.instance_id, + repo_url: task.repo_checkout_path, + commit: task.base_commit, + traj_data: { + pred_steps: [{ files: predFiles, spans: predSpans }], + pred_files: predFiles, + pred_spans: predSpans, + }, + model_patch: '', +}; + +writeFileSync(join(runDir, 'selection.json'), JSON.stringify(selection, null, 2)); +writeFileSync(join(runDir, 'prediction.json'), JSON.stringify(prediction, null, 2)); + +const goldPath = join(runDir, 'gold.json'); +const gold = run( + 'node', + [ + 'scripts/contextbench-select-slice.mjs', + '--write-gold', + '--task-id', + task.instance_id, + '--out', + goldPath, + '--payloads', + process.env.TASK_PAYLOADS, + ], + { timeoutMs: 600000 }, +); +writeFileSync(join(runDir, 'gold-command.json'), JSON.stringify(gold, null, 2)); +if (gold.status !== 0) { + throw new Error(`gold materialization failed: ${gold.stderr || gold.stdout}`); +} + +const scorePath = join(runDir, 'official-score.jsonl'); +const evaluator = run( + 'python', + [ + '-m', + 'contextbench.evaluate', + '--gold', + goldPath, + '--pred', + join(runDir, 'prediction.json'), + '--cache', + join(runDir, 'repo-cache'), + '--out', + scorePath, + ], + { cwd: officialContextBench, timeoutMs: 1200000 }, +); +writeFileSync(join(runDir, 'evaluator-command.json'), JSON.stringify(evaluator, null, 2)); + +let score = null; +if (existsSync(scorePath)) { + const lines = readFileSync(scorePath, 'utf8').trim().split(/\n+/).filter(Boolean); + if (lines.length > 0) score = JSON.parse(lines.at(-1)); +} + +const scoreable = evaluator.status === 0 && Boolean(score); +const final = score?.final || {}; +const row = { + lane_id: 'codebase-memory-mcp', + task_id: task.instance_id, + model: 'gpt-5.4-mini-high', + predictionSource: 'gpt-5.4-mini-high subagent over real codebase-memory-mcp candidate pack', + status: scoreable ? 'completed' : 'judge_failed', + setupStatus: 'completed', + indexStatus: 'completed', + toolCallable: true, + candidateCount: 100, + nonEmptyPrediction: predFiles.length > 0 && selection.spans.length > 0, + predFiles: predFiles.length, + predSpans: selection.spans.length, + officialEvaluatorScoreable: scoreable, + setupIndex: { setupDurationMs: 4, indexDurationMs: 2038, queryDurationMs: 365 }, + score, +}; +const summary = { + createdAt: new Date().toISOString(), + attemptedRows: 1, + scoreableRows: scoreable ? 1 : 0, + setupIndexCostReportedSeparately: true, + model: row.model, + caveat: 'Prediction was selected by a Codex gpt-5.4-mini-high subagent because GitHub Actions has no OpenAI API key for direct CI inference.', + resultsTable: scoreable + ? [ + { + lane: row.lane_id, + task: row.task_id, + fileCoverage: final.file?.coverage ?? null, + filePrecision: final.file?.precision ?? null, + symbolCoverage: final.symbol?.coverage ?? null, + symbolPrecision: final.symbol?.precision ?? null, + spanCoverage: final.span?.coverage ?? null, + spanPrecision: final.span?.precision ?? null, + lineCoverage: final.line?.coverage ?? null, + linePrecision: final.line?.precision ?? null, + editlocRecall: score.editloc?.recall ?? null, + editlocPrecision: score.editloc?.precision ?? null, + }, + ] + : [], + rows: [row], +}; + +writeFileSync(join(runDir, 'row.json'), JSON.stringify(row, null, 2)); +writeFileSync(join(root, 'summary.json'), JSON.stringify(summary, null, 2)); +console.log('CONTEXTBENCH_SELECTED_SCORE_JSON_START'); +console.log(JSON.stringify(summary, null, 2)); +console.log('CONTEXTBENCH_SELECTED_SCORE_JSON_END'); +if (!scoreable) process.exitCode = 1; From d3d279022cca5db57919a3833763b0ecea20567d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 00:58:25 +0200 Subject: [PATCH 021/100] Run official ContextBench score for selected CBM prediction --- .../contextbench-selected-cbm-score.yml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/contextbench-selected-cbm-score.yml diff --git a/.github/workflows/contextbench-selected-cbm-score.yml b/.github/workflows/contextbench-selected-cbm-score.yml new file mode 100644 index 0000000..6d0d455 --- /dev/null +++ b/.github/workflows/contextbench-selected-cbm-score.yml @@ -0,0 +1,56 @@ +name: ContextBench Selected CBM Score + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-selected-cbm-score.yml + - scripts/contextbench-score-selected-prediction.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + selected-cbm-score: + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + ROOT: /tmp/contextbench-selected-cbm-score + TASK_PAYLOADS: /tmp/contextbench-selected-cbm-score/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-selected-cbm-score/ContextBench-official + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install and materialize Go task quietly + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-install.log" 2>&1 + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$OFFICIAL_CONTEXTBENCH" > "$ROOT/logs/contextbench-clone.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 > "$ROOT/logs/materialize.log" 2>&1 + echo "selected_score_setup_completed" + - name: Score selected gpt-5.4-mini-high prediction + shell: bash + run: node scripts/contextbench-score-selected-prediction.mjs + - name: Upload selected score artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-selected-cbm-score + path: /tmp/contextbench-selected-cbm-score + retention-days: 14 From f30797f11b1c4b1a06c49d86385d81b193801568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:00:15 +0200 Subject: [PATCH 022/100] Select ContextBench scorer task by id --- scripts/contextbench-score-selected-prediction.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/contextbench-score-selected-prediction.mjs b/scripts/contextbench-score-selected-prediction.mjs index ce0630e..71d4da0 100644 --- a/scripts/contextbench-score-selected-prediction.mjs +++ b/scripts/contextbench-score-selected-prediction.mjs @@ -2,10 +2,12 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { spawnSync } from 'node:child_process'; +const targetTaskId = 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; const root = process.env.ROOT; const officialContextBench = process.env.OFFICIAL_CONTEXTBENCH; const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); -const task = payloads.tasks[2]; +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); const runDir = join(root, 'selected-codebase-memory-mcp'); mkdirSync(runDir, { recursive: true }); From 5950bb2727be58fd2d6e53a9fb3e406639e1cbc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:00:35 +0200 Subject: [PATCH 023/100] Materialize only selected ContextBench score task --- .../contextbench-selected-cbm-score.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/contextbench-selected-cbm-score.yml b/.github/workflows/contextbench-selected-cbm-score.yml index 6d0d455..9f049d7 100644 --- a/.github/workflows/contextbench-selected-cbm-score.yml +++ b/.github/workflows/contextbench-selected-cbm-score.yml @@ -14,12 +14,13 @@ permissions: jobs: selected-cbm-score: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 20 env: ROOT: /tmp/contextbench-selected-cbm-score TASK_PAYLOADS: /tmp/contextbench-selected-cbm-score/task-payloads.json CHECKOUT_ROOT: /tmp/contextbench-checkouts OFFICIAL_CONTEXTBENCH: /tmp/contextbench-selected-cbm-score/ContextBench-official + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 @@ -32,7 +33,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Install and materialize Go task quietly + - name: Install and materialize selected Go task quietly shell: bash run: | set -euo pipefail @@ -41,8 +42,17 @@ jobs: python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-install.log" 2>&1 git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$OFFICIAL_CONTEXTBENCH" > "$ROOT/logs/contextbench-clone.log" 2>&1 node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 - node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 - node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 > "$ROOT/logs/materialize.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 echo "selected_score_setup_completed" - name: Score selected gpt-5.4-mini-high prediction shell: bash From f0051952b1ceec9287e0931d59d895ea073ebe1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:06:26 +0200 Subject: [PATCH 024/100] Add five-lane ContextBench candidate pack script --- .../contextbench-five-lane-candidate-pack.mjs | 423 ++++++++++++++++++ 1 file changed, 423 insertions(+) create mode 100644 scripts/contextbench-five-lane-candidate-pack.mjs diff --git a/scripts/contextbench-five-lane-candidate-pack.mjs b/scripts/contextbench-five-lane-candidate-pack.mjs new file mode 100644 index 0000000..32a101b --- /dev/null +++ b/scripts/contextbench-five-lane-candidate-pack.mjs @@ -0,0 +1,423 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, +} from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-five-lane-candidates'; +const outDir = join(root, 'pack'); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function queryOf(text) { + const stop = new Set([ + 'that', + 'this', + 'with', + 'from', + 'when', + 'then', + 'into', + 'should', + 'would', + 'could', + 'there', + 'where', + 'which', + 'about', + 'after', + 'before', + 'have', + 'will', + 'been', + 'than', + 'also', + 'only', + 'some', + 'using', + 'must', + ]); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 24) + .join(' '); +} + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const name = basename(file); + const list = basenameMap.get(name) || []; + list.push(file); + basenameMap.set(name, list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function jsonish(s) { + const t = stripAnsi(s).trim(); + if (!t) return null; + try { + return JSON.parse(t); + } catch {} + for (const [a, b] of [ + ['{', '}'], + ['[', ']'], + ]) { + const i = t.indexOf(a); + const j = t.lastIndexOf(b); + if (i >= 0 && j > i) { + try { + return JSON.parse(t.slice(i, j + 1)); + } catch {} + } + } + return null; +} + +function norm(file) { + let f = stripAnsi(file) + .replace(/^file:\/\//, '') + .replaceAll('\\', '/') + .trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + const tmpIdx = f.indexOf('/tmp/contextbench-checkouts/'); + if (tmpIdx >= 0) { + const after = f.slice(tmpIdx + '/tmp/contextbench-checkouts/'.length); + const parts = after.split('/'); + f = parts.slice(1).join('/'); + } + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..') || f.startsWith('tmp/')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'tool') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + const e = Math.max(s, Number(end) || s); + locs.push({ file: clean, start: s, end: e, source }); +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add( + locs, + value.file || + value.path || + value.file_path || + value.relative_path || + value.filename || + value.source_path || + value.uri, + value.start_line || value.startLine || value.line || value.line_number || value.start || 1, + value.end_line || value.endLine || value.end || value.line || 1, + source, + ); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source) { + const cleaned = stripAnsi(text); + const parsed = jsonish(cleaned); + if (parsed) walk(parsed, locs, source); + + const rgLine = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte)):(\d+):/gm; + let m; + while ((m = rgLine.exec(cleaned)) !== null) add(locs, m[1], m[2], m[2], source); + + const fileLine = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?/g; + while ((m = fileLine.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); +} + +function uniq(locs, max = 120) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function writeCommands(lane, commands) { + for (const [i, command] of commands.entries()) { + writeFileSync( + join(outDir, `${lane}-command-${i + 1}.json`), + JSON.stringify( + { + ...command, + stdout: command.stdout.slice(0, 200000), + stderr: command.stderr.slice(0, 200000), + }, + null, + 2, + ), + ); + } +} + +function laneResult(lane, commands, locs, setupStatus = 'completed', indexStatus = 'completed') { + writeCommands(lane, commands); + const candidates = uniq(locs); + return { + lane, + setupStatus, + indexStatus, + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex: { + setupDurationMs: commands.slice(0, 1).reduce((sum, command) => sum + command.durationMs, 0), + indexDurationMs: commands.slice(1, 2).reduce((sum, command) => sum + command.durationMs, 0), + queryDurationMs: commands.slice(2).reduce((sum, command) => sum + command.durationMs, 0), + }, + commands: commands.map((command) => ({ + command: command.command, + status: command.status, + error: command.error, + durationMs: command.durationMs, + })), + candidates, + }; +} + +const baseQuery = queryOf(task.problem_statement); +const queryVariants = [ + baseQuery, + 'system metrics written on start prometheus writer insights', + 'Bearer token custom authorization header authentication', + 'Subsonic authorization header token parsing', + 'startup metrics insights prometheus initialization', +]; + +const lanes = []; + +{ + const commands = []; + const locs = []; + const terms = [ + 'metrics', + 'prometheus', + 'insights', + 'startup', + 'start', + 'written', + 'authorization', + 'Bearer', + 'token', + 'header', + 'authentication', + 'subsonic', + ]; + for (const term of terms) { + const r = run( + 'rg', + ['-n', '-i', '--glob', '!.git', '--glob', '!vendor/**', '--glob', '!node_modules/**', term, '.'], + { cwd: repo, timeoutMs: 60000 }, + ); + commands.push(r); + collect(r.stdout, locs, 'raw-native'); + collect(r.stderr, locs, 'raw-native'); + } + lanes.push(laneResult('raw-native', commands, locs)); +} + +{ + const commands = []; + const locs = []; + const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' }; + const setup = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 1200000 }); + commands.push(index); + for (const q of queryVariants) { + const search = run('node', ['dist/index.js', 'search', '--query', q, '--intent', 'edit', '--limit', '40', '--json'], { + env, + timeoutMs: 300000, + }); + commands.push(search); + collect(search.stdout, locs, 'codebase-context'); + collect(search.stderr, locs, 'codebase-context'); + } + lanes.push(laneResult('codebase-context', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed')); +} + +{ + const commands = []; + const locs = []; + const env = { ...process.env, CBM_CACHE_DIR: join(outDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { + cwd: repo, + env, + timeoutMs: 2700000, + }); + commands.push(index); + const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo); + for (const q of queryVariants) { + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query: q, limit: 50 })], { + cwd: repo, + env, + timeoutMs: 120000, + }); + commands.push(graph); + collect(graph.stdout, locs, 'codebase-memory-mcp'); + collect(graph.stderr, locs, 'codebase-memory-mcp'); + } + for (const term of ['metrics', 'prometheus', 'authorization', 'Bearer', 'token']) { + const code = run( + process.env.CBM_BIN, + ['cli', 'search_code', JSON.stringify({ project, pattern: term, mode: 'compact', limit: 50 })], + { cwd: repo, env, timeoutMs: 120000 }, + ); + commands.push(code); + collect(code.stdout, locs, 'codebase-memory-mcp'); + collect(code.stderr, locs, 'codebase-memory-mcp'); + } + lanes.push(laneResult('codebase-memory-mcp', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed')); +} + +{ + const commands = []; + const locs = []; + const setup = run('grepai', ['version'], { timeoutMs: 60000 }); + commands.push(setup); + const init = run('grepai', ['init', '--yes', '--provider', 'synthetic', '--backend', 'gob'], { cwd: repo, timeoutMs: 120000 }); + commands.push(init); + const watch = run('grepai', ['watch', '--background'], { cwd: repo, timeoutMs: 120000 }); + commands.push(watch); + for (let i = 0; i < 12; i += 1) { + const status = run('grepai', ['status', '--no-ui'], { cwd: repo, timeoutMs: 60000 }); + commands.push(status); + if (/chunks?\D+[1-9]|indexed files?\D+[1-9]/i.test(`${status.stdout}\n${status.stderr}`)) break; + run('sleep', ['5'], { timeoutMs: 10000 }); + } + for (const q of queryVariants) { + const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '40'], { cwd: repo, timeoutMs: 180000 }); + commands.push(search); + collect(search.stdout, locs, 'grepai'); + collect(search.stderr, locs, 'grepai'); + } + const stop = run('grepai', ['watch', '--stop'], { cwd: repo, timeoutMs: 60000 }); + commands.push(stop); + lanes.push(laneResult('grepai', commands, locs, setup.status === 0 && init.status === 0 ? 'completed' : 'setup_failed', watch.status === 0 ? 'completed' : 'index_failed')); +} + +{ + const commands = []; + const locs = []; + const commandProbe = run('codegraphcontext', ['--help'], { timeoutMs: 60000 }); + const cgcCommand = commandProbe.status === 0 ? 'codegraphcontext' : 'cgc'; + commands.push(commandProbe.status === 0 ? commandProbe : run(cgcCommand, ['--help'], { timeoutMs: 60000 })); + const index = run(cgcCommand, ['index', '.'], { cwd: repo, timeoutMs: 1200000 }); + commands.push(index); + for (const pattern of ['Metrics', 'Prometheus', 'Authorization', 'Bearer', 'Token', 'Subsonic', 'Header']) { + const found = run(cgcCommand, ['find', 'pattern', pattern], { cwd: repo, timeoutMs: 180000 }); + commands.push(found); + collect(found.stdout, locs, 'codegraphcontext'); + collect(found.stderr, locs, 'codegraphcontext'); + } + for (const symbol of ['main', 'init', 'WriteInitialMetrics', 'ServeHTTP', 'GetUser']) { + const callers = run(cgcCommand, ['analyze', 'callers', symbol], { cwd: repo, timeoutMs: 180000 }); + commands.push(callers); + collect(callers.stdout, locs, 'codegraphcontext'); + collect(callers.stderr, locs, 'codegraphcontext'); + } + const complexity = run(cgcCommand, ['analyze', 'complexity', '--limit', '80'], { cwd: repo, timeoutMs: 180000 }); + commands.push(complexity); + collect(complexity.stdout, locs, 'codegraphcontext'); + collect(complexity.stderr, locs, 'codegraphcontext'); + lanes.push(laneResult('codegraphcontext', commands, locs, commands[0].status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed')); +} + +const readiness = lanes.map((lane) => ({ + lane: lane.lane, + ready: lane.setupStatus === 'completed' && lane.indexStatus === 'completed' && lane.toolCallable && lane.candidateCount > 0, + setupStatus: lane.setupStatus, + indexStatus: lane.indexStatus, + toolCallable: lane.toolCallable, + candidateCount: lane.candidateCount, + setupIndex: lane.setupIndex, + candidates: lane.candidates.slice(0, 80), +})); + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queryVariants, + lanes, + readiness, +}; + +writeFileSync(join(outDir, 'candidate-pack.json'), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, 'readiness.json'), JSON.stringify({ task: pack.task, queryVariants, readiness }, null, 2)); +console.log('CONTEXTBENCH_FIVE_LANE_READINESS_JSON_START'); +console.log(JSON.stringify({ task: pack.task, queryVariants, readiness }, null, 2)); +console.log('CONTEXTBENCH_FIVE_LANE_READINESS_JSON_END'); From 786b8078c94d923a391bf672d035ddec9a5c33ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:06:48 +0200 Subject: [PATCH 025/100] Add five-lane ContextBench readiness workflow --- .../contextbench-five-lane-candidates.yml | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/contextbench-five-lane-candidates.yml diff --git a/.github/workflows/contextbench-five-lane-candidates.yml b/.github/workflows/contextbench-five-lane-candidates.yml new file mode 100644 index 0000000..416b282 --- /dev/null +++ b/.github/workflows/contextbench-five-lane-candidates.yml @@ -0,0 +1,72 @@ +name: ContextBench Five Lane Candidates + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-five-lane-candidates.yml + - scripts/contextbench-five-lane-candidate-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + candidates: + runs-on: ubuntu-latest + timeout-minutes: 100 + env: + ROOT: /tmp/contextbench-five-lane-candidates + TASK_PAYLOADS: /tmp/contextbench-five-lane-candidates/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + CBM_BIN: /tmp/contextbench-five-lane-candidates/tool/codebase-memory-mcp + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install tools and materialize selected Go task quietly + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + pnpm run build > "$ROOT/logs/build.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow uv codegraphcontext kuzu > "$ROOT/logs/pip-install.log" 2>&1 + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 || true + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + echo "five_lane_setup_completed" + - name: Generate five-lane readiness candidate pack + shell: bash + run: node scripts/contextbench-five-lane-candidate-pack.mjs + - name: Upload five-lane candidate artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-five-lane-candidates + path: /tmp/contextbench-five-lane-candidates + retention-days: 14 From 3ecc4d53fe7ebab0e54d7104da447bef7e625dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:07:58 +0200 Subject: [PATCH 026/100] Add five-lane ContextBench scoring script --- ...ontextbench-score-five-lane-selections.mjs | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 scripts/contextbench-score-five-lane-selections.mjs diff --git a/scripts/contextbench-score-five-lane-selections.mjs b/scripts/contextbench-score-five-lane-selections.mjs new file mode 100644 index 0000000..f33055e --- /dev/null +++ b/scripts/contextbench-score-five-lane-selections.mjs @@ -0,0 +1,189 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-five-lane-score'; +const officialContextBench = process.env.OFFICIAL_CONTEXTBENCH; +const selectionsPath = process.env.SELECTIONS_PATH || 'scripts/contextbench-five-lane-selections.json'; +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +if (!officialContextBench) throw new Error('OFFICIAL_CONTEXTBENCH is required'); +if (!existsSync(selectionsPath)) throw new Error(`selection file missing: ${selectionsPath}`); +const selections = JSON.parse(readFileSync(selectionsPath, 'utf8')); +const laneSelections = selections.laneSelections || []; +if (laneSelections.length === 0) throw new Error('selection file has no laneSelections'); + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 600000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function addSpan(map, file, start, end) { + const clean = String(file || '').replaceAll('\\', '/').replace(/^\.\//, ''); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + const e = Math.max(s, Number(end) || s); + const list = map.get(clean) || []; + list.push({ start: s, end: e }); + map.set(clean, list); +} + +function resultTableRow(row) { + const final = row.score?.final || {}; + return { + lane: row.lane_id, + task: row.task_id, + fileCoverage: final.file?.coverage ?? null, + filePrecision: final.file?.precision ?? null, + symbolCoverage: final.symbol?.coverage ?? null, + symbolPrecision: final.symbol?.precision ?? null, + spanCoverage: final.span?.coverage ?? null, + spanPrecision: final.span?.precision ?? null, + lineCoverage: final.line?.coverage ?? null, + linePrecision: final.line?.precision ?? null, + editlocRecall: row.score?.editloc?.recall ?? null, + editlocPrecision: row.score?.editloc?.precision ?? null, + }; +} + +const runDir = join(root, 'five-lane-score'); +mkdirSync(runDir, { recursive: true }); +writeFileSync(join(runDir, 'selections.json'), JSON.stringify(selections, null, 2)); + +const goldPath = join(runDir, 'gold.json'); +const gold = run( + 'node', + [ + 'scripts/contextbench-select-slice.mjs', + '--write-gold', + '--task-id', + task.instance_id, + '--out', + goldPath, + '--payloads', + process.env.TASK_PAYLOADS, + ], + { timeoutMs: 600000 }, +); +writeFileSync(join(runDir, 'gold-command.json'), JSON.stringify(gold, null, 2)); +if (gold.status !== 0) throw new Error(`gold materialization failed: ${gold.stderr || gold.stdout}`); + +const rows = []; +for (const selection of laneSelections) { + const lane = selection.lane_id || selection.lane; + const laneDir = join(runDir, lane); + mkdirSync(laneDir, { recursive: true }); + const spans = Array.isArray(selection.spans) ? selection.spans : []; + const files = Array.isArray(selection.files) ? selection.files : []; + const spanMap = new Map(); + for (const span of spans) addSpan(spanMap, span.file, span.start, span.end); + const predFiles = [...new Set([...files, ...spans.map((span) => String(span.file || '').replaceAll('\\', '/').replace(/^\.\//, ''))])].filter(Boolean); + const predSpans = Object.fromEntries(spanMap.entries()); + const nonEmptyPrediction = predFiles.length > 0 && spans.length > 0; + const readiness = selection.readiness || {}; + const rowBase = { + lane_id: lane, + task_id: task.instance_id, + model: selections.model || 'gpt-5.4-mini-high', + predictionSource: selection.predictionSource || selections.predictionSource || 'gpt-5.4-mini-high subagent over real lane candidate pack', + setupStatus: readiness.setupStatus || selection.setupStatus || 'unknown', + indexStatus: readiness.indexStatus || selection.indexStatus || 'unknown', + toolCallable: Boolean(readiness.toolCallable ?? selection.toolCallable), + candidateCount: Number(readiness.candidateCount ?? selection.candidateCount ?? 0), + setupIndex: readiness.setupIndex || selection.setupIndex || null, + nonEmptyPrediction, + predFiles: predFiles.length, + predSpans: spans.length, + rationale: selection.rationale || null, + }; + + writeFileSync(join(laneDir, 'selection.json'), JSON.stringify(selection, null, 2)); + if (!nonEmptyPrediction) { + rows.push({ ...rowBase, status: 'empty_prediction', officialEvaluatorScoreable: false, score: null }); + continue; + } + + const prediction = { + instance_id: task.instance_id, + repo_url: task.repo_checkout_path, + commit: task.base_commit, + traj_data: { + pred_steps: [{ files: predFiles, spans: predSpans }], + pred_files: predFiles, + pred_spans: predSpans, + }, + model_patch: '', + }; + const predictionPath = join(laneDir, 'prediction.json'); + writeFileSync(predictionPath, JSON.stringify(prediction, null, 2)); + + const scorePath = join(laneDir, 'official-score.jsonl'); + const evaluator = run( + 'python', + [ + '-m', + 'contextbench.evaluate', + '--gold', + goldPath, + '--pred', + predictionPath, + '--cache', + join(laneDir, 'repo-cache'), + '--out', + scorePath, + ], + { cwd: officialContextBench, timeoutMs: 1200000 }, + ); + writeFileSync(join(laneDir, 'evaluator-command.json'), JSON.stringify(evaluator, null, 2)); + let score = null; + if (existsSync(scorePath)) { + const lines = readFileSync(scorePath, 'utf8').trim().split(/\n+/).filter(Boolean); + if (lines.length > 0) score = JSON.parse(lines.at(-1)); + } + const scoreable = evaluator.status === 0 && Boolean(score); + rows.push({ + ...rowBase, + status: scoreable ? 'completed' : 'judge_failed', + officialEvaluatorScoreable: scoreable, + score, + }); +} + +const scoreableRows = rows.filter((row) => row.officialEvaluatorScoreable); +const summary = { + createdAt: new Date().toISOString(), + attemptedRows: rows.length, + scoreableRows: scoreableRows.length, + requiredCompetitors: 5, + setupIndexCostReportedSeparately: true, + model: selections.model || 'gpt-5.4-mini-high', + predictionSource: selections.predictionSource || 'gpt-5.4-mini-high subagent selections over real lane candidate packs', + caveats: selections.caveats || [], + resultsTable: scoreableRows.map(resultTableRow), + rows, +}; + +writeFileSync(join(runDir, 'summary.json'), JSON.stringify(summary, null, 2)); +writeFileSync(join(root, 'summary.json'), JSON.stringify(summary, null, 2)); +console.log('CONTEXTBENCH_FIVE_LANE_SCORE_JSON_START'); +console.log(JSON.stringify(summary, null, 2)); +console.log('CONTEXTBENCH_FIVE_LANE_SCORE_JSON_END'); +if (scoreableRows.length !== rows.length || scoreableRows.length < 5) process.exitCode = 1; From b1c3f8d244d490113cdf937ac8afca8a1bf79c97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:15:57 +0200 Subject: [PATCH 027/100] Harden five-lane ContextBench scorer gates --- ...ontextbench-score-five-lane-selections.mjs | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/scripts/contextbench-score-five-lane-selections.mjs b/scripts/contextbench-score-five-lane-selections.mjs index f33055e..0029a67 100644 --- a/scripts/contextbench-score-five-lane-selections.mjs +++ b/scripts/contextbench-score-five-lane-selections.mjs @@ -6,6 +6,7 @@ const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenan const root = process.env.ROOT || '/tmp/contextbench-five-lane-score'; const officialContextBench = process.env.OFFICIAL_CONTEXTBENCH; const selectionsPath = process.env.SELECTIONS_PATH || 'scripts/contextbench-five-lane-selections.json'; +const requiredLanes = ['raw-native', 'codebase-context', 'codebase-memory-mcp', 'grepai', 'codegraphcontext']; const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); @@ -15,6 +16,20 @@ const selections = JSON.parse(readFileSync(selectionsPath, 'utf8')); const laneSelections = selections.laneSelections || []; if (laneSelections.length === 0) throw new Error('selection file has no laneSelections'); +const laneCounts = new Map(); +for (const selection of laneSelections) { + const lane = selection.lane_id || selection.lane; + laneCounts.set(lane, (laneCounts.get(lane) || 0) + 1); +} +const missingLanes = requiredLanes.filter((lane) => !laneCounts.has(lane)); +const duplicateLanes = [...laneCounts.entries()].filter(([, count]) => count > 1).map(([lane]) => lane); +const extraLanes = [...laneCounts.keys()].filter((lane) => !requiredLanes.includes(lane)); +if (missingLanes.length > 0 || duplicateLanes.length > 0 || extraLanes.length > 0) { + throw new Error( + `lane selection set invalid: missing=${missingLanes.join(',') || 'none'} duplicate=${duplicateLanes.join(',') || 'none'} extra=${extraLanes.join(',') || 'none'}`, + ); +} + function run(cmd, args, opts = {}) { const started = Date.now(); const r = spawnSync(cmd, args, { @@ -97,7 +112,7 @@ for (const selection of laneSelections) { for (const span of spans) addSpan(spanMap, span.file, span.start, span.end); const predFiles = [...new Set([...files, ...spans.map((span) => String(span.file || '').replaceAll('\\', '/').replace(/^\.\//, ''))])].filter(Boolean); const predSpans = Object.fromEntries(spanMap.entries()); - const nonEmptyPrediction = predFiles.length > 0 && spans.length > 0; + const nonEmptyPrediction = predFiles.length > 0 || spans.length > 0; const readiness = selection.readiness || {}; const rowBase = { lane_id: lane, @@ -172,7 +187,8 @@ const summary = { createdAt: new Date().toISOString(), attemptedRows: rows.length, scoreableRows: scoreableRows.length, - requiredCompetitors: 5, + requiredCompetitors: requiredLanes.length, + requiredLanes, setupIndexCostReportedSeparately: true, model: selections.model || 'gpt-5.4-mini-high', predictionSource: selections.predictionSource || 'gpt-5.4-mini-high subagent selections over real lane candidate packs', @@ -186,4 +202,4 @@ writeFileSync(join(root, 'summary.json'), JSON.stringify(summary, null, 2)); console.log('CONTEXTBENCH_FIVE_LANE_SCORE_JSON_START'); console.log(JSON.stringify(summary, null, 2)); console.log('CONTEXTBENCH_FIVE_LANE_SCORE_JSON_END'); -if (scoreableRows.length !== rows.length || scoreableRows.length < 5) process.exitCode = 1; +if (scoreableRows.length !== rows.length || scoreableRows.length < requiredLanes.length) process.exitCode = 1; From d675cda05b3d8b716220ce407e81595ff4feaad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:17:39 +0200 Subject: [PATCH 028/100] Report five-lane ContextBench phase costs explicitly --- .../contextbench-five-lane-candidate-pack.mjs | 75 +++++++++++++++---- 1 file changed, 62 insertions(+), 13 deletions(-) diff --git a/scripts/contextbench-five-lane-candidate-pack.mjs b/scripts/contextbench-five-lane-candidate-pack.mjs index 32a101b..81b6c9e 100644 --- a/scripts/contextbench-five-lane-candidate-pack.mjs +++ b/scripts/contextbench-five-lane-candidate-pack.mjs @@ -39,6 +39,10 @@ function run(cmd, args, opts = {}) { }; } +function durationOf(commands) { + return commands.reduce((sum, command) => sum + command.durationMs, 0); +} + function queryOf(text) { const stop = new Set([ 'that', @@ -217,7 +221,7 @@ function writeCommands(lane, commands) { } } -function laneResult(lane, commands, locs, setupStatus = 'completed', indexStatus = 'completed') { +function laneResult(lane, commands, locs, setupStatus, indexStatus, setupIndex) { writeCommands(lane, commands); const candidates = uniq(locs); return { @@ -226,11 +230,7 @@ function laneResult(lane, commands, locs, setupStatus = 'completed', indexStatus indexStatus, toolCallable: commands.some((command) => command.status === 0), candidateCount: candidates.length, - setupIndex: { - setupDurationMs: commands.slice(0, 1).reduce((sum, command) => sum + command.durationMs, 0), - indexDurationMs: commands.slice(1, 2).reduce((sum, command) => sum + command.durationMs, 0), - queryDurationMs: commands.slice(2).reduce((sum, command) => sum + command.durationMs, 0), - }, + setupIndex, commands: commands.map((command) => ({ command: command.command, status: command.status, @@ -279,7 +279,13 @@ const lanes = []; collect(r.stdout, locs, 'raw-native'); collect(r.stderr, locs, 'raw-native'); } - lanes.push(laneResult('raw-native', commands, locs)); + lanes.push( + laneResult('raw-native', commands, locs, 'not_required', 'not_required', { + setupDurationMs: 0, + indexDurationMs: 0, + queryDurationMs: durationOf(commands), + }), + ); } { @@ -290,16 +296,24 @@ const lanes = []; commands.push(setup); const index = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 1200000 }); commands.push(index); + const searches = []; for (const q of queryVariants) { const search = run('node', ['dist/index.js', 'search', '--query', q, '--intent', 'edit', '--limit', '40', '--json'], { env, timeoutMs: 300000, }); commands.push(search); + searches.push(search); collect(search.stdout, locs, 'codebase-context'); collect(search.stderr, locs, 'codebase-context'); } - lanes.push(laneResult('codebase-context', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed')); + lanes.push( + laneResult('codebase-context', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(searches), + }), + ); } { @@ -315,6 +329,7 @@ const lanes = []; }); commands.push(index); const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo); + const searches = []; for (const q of queryVariants) { const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query: q, limit: 50 })], { cwd: repo, @@ -322,6 +337,7 @@ const lanes = []; timeoutMs: 120000, }); commands.push(graph); + searches.push(graph); collect(graph.stdout, locs, 'codebase-memory-mcp'); collect(graph.stderr, locs, 'codebase-memory-mcp'); } @@ -332,10 +348,17 @@ const lanes = []; { cwd: repo, env, timeoutMs: 120000 }, ); commands.push(code); + searches.push(code); collect(code.stdout, locs, 'codebase-memory-mcp'); collect(code.stderr, locs, 'codebase-memory-mcp'); } - lanes.push(laneResult('codebase-memory-mcp', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed')); + lanes.push( + laneResult('codebase-memory-mcp', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(searches), + }), + ); } { @@ -347,21 +370,32 @@ const lanes = []; commands.push(init); const watch = run('grepai', ['watch', '--background'], { cwd: repo, timeoutMs: 120000 }); commands.push(watch); + const statusChecks = []; for (let i = 0; i < 12; i += 1) { const status = run('grepai', ['status', '--no-ui'], { cwd: repo, timeoutMs: 60000 }); commands.push(status); + statusChecks.push(status); if (/chunks?\D+[1-9]|indexed files?\D+[1-9]/i.test(`${status.stdout}\n${status.stderr}`)) break; run('sleep', ['5'], { timeoutMs: 10000 }); } + const searches = []; for (const q of queryVariants) { const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '40'], { cwd: repo, timeoutMs: 180000 }); commands.push(search); + searches.push(search); collect(search.stdout, locs, 'grepai'); collect(search.stderr, locs, 'grepai'); } const stop = run('grepai', ['watch', '--stop'], { cwd: repo, timeoutMs: 60000 }); commands.push(stop); - lanes.push(laneResult('grepai', commands, locs, setup.status === 0 && init.status === 0 ? 'completed' : 'setup_failed', watch.status === 0 ? 'completed' : 'index_failed')); + lanes.push( + laneResult('grepai', commands, locs, setup.status === 0 && init.status === 0 ? 'completed' : 'setup_failed', watch.status === 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs + init.durationMs, + indexDurationMs: watch.durationMs + durationOf(statusChecks), + queryDurationMs: durationOf(searches), + teardownDurationMs: stop.durationMs, + }), + ); } { @@ -369,31 +403,46 @@ const lanes = []; const locs = []; const commandProbe = run('codegraphcontext', ['--help'], { timeoutMs: 60000 }); const cgcCommand = commandProbe.status === 0 ? 'codegraphcontext' : 'cgc'; - commands.push(commandProbe.status === 0 ? commandProbe : run(cgcCommand, ['--help'], { timeoutMs: 60000 })); + const setup = commandProbe.status === 0 ? commandProbe : run(cgcCommand, ['--help'], { timeoutMs: 60000 }); + commands.push(setup); const index = run(cgcCommand, ['index', '.'], { cwd: repo, timeoutMs: 1200000 }); commands.push(index); + const queries = []; for (const pattern of ['Metrics', 'Prometheus', 'Authorization', 'Bearer', 'Token', 'Subsonic', 'Header']) { const found = run(cgcCommand, ['find', 'pattern', pattern], { cwd: repo, timeoutMs: 180000 }); commands.push(found); + queries.push(found); collect(found.stdout, locs, 'codegraphcontext'); collect(found.stderr, locs, 'codegraphcontext'); } for (const symbol of ['main', 'init', 'WriteInitialMetrics', 'ServeHTTP', 'GetUser']) { const callers = run(cgcCommand, ['analyze', 'callers', symbol], { cwd: repo, timeoutMs: 180000 }); commands.push(callers); + queries.push(callers); collect(callers.stdout, locs, 'codegraphcontext'); collect(callers.stderr, locs, 'codegraphcontext'); } const complexity = run(cgcCommand, ['analyze', 'complexity', '--limit', '80'], { cwd: repo, timeoutMs: 180000 }); commands.push(complexity); + queries.push(complexity); collect(complexity.stdout, locs, 'codegraphcontext'); collect(complexity.stderr, locs, 'codegraphcontext'); - lanes.push(laneResult('codegraphcontext', commands, locs, commands[0].status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed')); + lanes.push( + laneResult('codegraphcontext', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(queries), + }), + ); } const readiness = lanes.map((lane) => ({ lane: lane.lane, - ready: lane.setupStatus === 'completed' && lane.indexStatus === 'completed' && lane.toolCallable && lane.candidateCount > 0, + ready: + (lane.setupStatus === 'completed' || lane.setupStatus === 'not_required') && + (lane.indexStatus === 'completed' || lane.indexStatus === 'not_required') && + lane.toolCallable && + lane.candidateCount > 0, setupStatus: lane.setupStatus, indexStatus: lane.indexStatus, toolCallable: lane.toolCallable, From 0f36a30610e71e1901cafd418998fc2ca464bb5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:20:21 +0200 Subject: [PATCH 029/100] Add single-lane ContextBench readiness script --- ...ontextbench-single-lane-candidate-pack.mjs | 340 ++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 scripts/contextbench-single-lane-candidate-pack.mjs diff --git a/scripts/contextbench-single-lane-candidate-pack.mjs b/scripts/contextbench-single-lane-candidate-pack.mjs new file mode 100644 index 0000000..874b621 --- /dev/null +++ b/scripts/contextbench-single-lane-candidate-pack.mjs @@ -0,0 +1,340 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const laneId = process.env.LANE_ID; +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-single-lane'; +const outDir = join(root, 'pack'); +const requiredLanes = ['raw-native', 'codebase-context', 'codebase-memory-mcp', 'grepai', 'codegraphcontext']; +if (!requiredLanes.includes(laneId)) throw new Error(`unsupported LANE_ID: ${laneId}`); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +const durationOf = (commands) => commands.reduce((sum, command) => sum + command.durationMs, 0); + +function queryOf(text) { + const stop = new Set(['that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', 'would', 'could', 'there', 'where', 'which', 'about', 'after', 'before', 'have', 'will', 'been', 'than', 'also', 'only', 'some', 'using', 'must']); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 24) + .join(' '); +} + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const name = basename(file); + const list = basenameMap.get(name) || []; + list.push(file); + basenameMap.set(name, list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function jsonish(s) { + const t = stripAnsi(s).trim(); + if (!t) return null; + try { + return JSON.parse(t); + } catch {} + for (const [a, b] of [['{', '}'], ['[', ']']]) { + const i = t.indexOf(a); + const j = t.lastIndexOf(b); + if (i >= 0 && j > i) { + try { + return JSON.parse(t.slice(i, j + 1)); + } catch {} + } + } + return null; +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + const tmpIdx = f.indexOf('/tmp/contextbench-checkouts/'); + if (tmpIdx >= 0) { + const parts = f.slice(tmpIdx + '/tmp/contextbench-checkouts/'.length).split('/'); + f = parts.slice(1).join('/'); + } + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..') || f.startsWith('tmp/')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'tool') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add( + locs, + value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, + value.start_line || value.startLine || value.line || value.line_number || value.start || 1, + value.end_line || value.endLine || value.end || value.line || 1, + source, + ); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source) { + const cleaned = stripAnsi(text); + const parsed = jsonish(cleaned); + if (parsed) walk(parsed, locs, source); + const rgLine = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte)):(\d+):/gm; + let m; + while ((m = rgLine.exec(cleaned)) !== null) add(locs, m[1], m[2], m[2], source); + const fileLine = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?/g; + while ((m = fileLine.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); +} + +function uniq(locs, max = 120) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function writeCommands(lane, commands) { + for (const [i, command] of commands.entries()) { + writeFileSync(join(outDir, `${lane}-command-${i + 1}.json`), JSON.stringify({ ...command, stdout: command.stdout.slice(0, 200000), stderr: command.stderr.slice(0, 200000) }, null, 2)); + } +} + +function laneResult(lane, commands, locs, setupStatus, indexStatus, setupIndex) { + writeCommands(lane, commands); + const candidates = uniq(locs); + return { + lane, + ready: (setupStatus === 'completed' || setupStatus === 'not_required') && (indexStatus === 'completed' || indexStatus === 'not_required') && commands.some((command) => command.status === 0) && candidates.length > 0, + setupStatus, + indexStatus, + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex, + commands: commands.map((command) => ({ command: command.command, status: command.status, error: command.error, durationMs: command.durationMs })), + candidates, + }; +} + +const baseQuery = queryOf(task.problem_statement); +const queryVariants = [ + baseQuery, + 'system metrics written on start prometheus writer insights', + 'Bearer token custom authorization header authentication', + 'Subsonic authorization header token parsing', + 'startup metrics insights prometheus initialization', +]; + +function runRawNative() { + const commands = []; + const locs = []; + for (const term of ['metrics', 'prometheus', 'insights', 'startup', 'start', 'written', 'authorization', 'Bearer', 'token', 'header', 'authentication', 'subsonic']) { + const r = run('rg', ['-n', '-i', '--glob', '!.git', '--glob', '!vendor/**', '--glob', '!node_modules/**', term, '.'], { cwd: repo, timeoutMs: 60000 }); + commands.push(r); + collect(r.stdout, locs, 'raw-native'); + collect(r.stderr, locs, 'raw-native'); + } + return laneResult('raw-native', commands, locs, 'not_required', 'not_required', { setupDurationMs: 0, indexDurationMs: 0, queryDurationMs: durationOf(commands) }); +} + +function runCodebaseContext() { + const commands = []; + const locs = []; + const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' }; + const setup = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 1200000 }); + commands.push(index); + const searches = []; + for (const q of queryVariants) { + const search = run('node', ['dist/index.js', 'search', '--query', q, '--intent', 'edit', '--limit', '40', '--json'], { env, timeoutMs: 300000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'codebase-context'); + collect(search.stderr, locs, 'codebase-context'); + } + return laneResult('codebase-context', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: durationOf(searches) }); +} + +function runCodebaseMemoryMcp() { + const commands = []; + const locs = []; + const env = { ...process.env, CBM_CACHE_DIR: join(outDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 2700000 }); + commands.push(index); + const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo); + const searches = []; + for (const q of queryVariants) { + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query: q, limit: 50 })], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(graph); + searches.push(graph); + collect(graph.stdout, locs, 'codebase-memory-mcp'); + collect(graph.stderr, locs, 'codebase-memory-mcp'); + } + for (const term of ['metrics', 'prometheus', 'authorization', 'Bearer', 'token']) { + const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: term, mode: 'compact', limit: 50 })], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(code); + searches.push(code); + collect(code.stdout, locs, 'codebase-memory-mcp'); + collect(code.stderr, locs, 'codebase-memory-mcp'); + } + return laneResult('codebase-memory-mcp', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: durationOf(searches) }); +} + +function runGrepai() { + const commands = []; + const locs = []; + const setup = run('grepai', ['version'], { timeoutMs: 60000 }); + commands.push(setup); + const init = run('grepai', ['init', '--yes', '--provider', 'synthetic', '--backend', 'gob'], { cwd: repo, timeoutMs: 120000 }); + commands.push(init); + const watch = run('grepai', ['watch', '--background'], { cwd: repo, timeoutMs: 120000 }); + commands.push(watch); + const statusChecks = []; + for (let i = 0; i < 12; i += 1) { + const status = run('grepai', ['status', '--no-ui'], { cwd: repo, timeoutMs: 60000 }); + commands.push(status); + statusChecks.push(status); + if (/chunks?\D+[1-9]|indexed files?\D+[1-9]/i.test(`${status.stdout}\n${status.stderr}`)) break; + run('sleep', ['5'], { timeoutMs: 10000 }); + } + const searches = []; + for (const q of queryVariants) { + const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '40'], { cwd: repo, timeoutMs: 180000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'grepai'); + collect(search.stderr, locs, 'grepai'); + } + const stop = run('grepai', ['watch', '--stop'], { cwd: repo, timeoutMs: 60000 }); + commands.push(stop); + return laneResult('grepai', commands, locs, setup.status === 0 && init.status === 0 ? 'completed' : 'setup_failed', watch.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs + init.durationMs, indexDurationMs: watch.durationMs + durationOf(statusChecks), queryDurationMs: durationOf(searches), teardownDurationMs: stop.durationMs }); +} + +function runCodeGraphContext() { + const commands = []; + const locs = []; + const probe = run('codegraphcontext', ['--help'], { timeoutMs: 60000 }); + const cgcCommand = probe.status === 0 ? 'codegraphcontext' : 'cgc'; + const setup = probe.status === 0 ? probe : run(cgcCommand, ['--help'], { timeoutMs: 60000 }); + commands.push(setup); + const index = run(cgcCommand, ['index', '.'], { cwd: repo, timeoutMs: 1200000 }); + commands.push(index); + const queries = []; + for (const pattern of ['Metrics', 'Prometheus', 'Authorization', 'Bearer', 'Token', 'Subsonic', 'Header']) { + const found = run(cgcCommand, ['find', 'pattern', pattern], { cwd: repo, timeoutMs: 180000 }); + commands.push(found); + queries.push(found); + collect(found.stdout, locs, 'codegraphcontext'); + collect(found.stderr, locs, 'codegraphcontext'); + } + for (const symbol of ['main', 'init', 'WriteInitialMetrics', 'ServeHTTP', 'GetUser']) { + const callers = run(cgcCommand, ['analyze', 'callers', symbol], { cwd: repo, timeoutMs: 180000 }); + commands.push(callers); + queries.push(callers); + collect(callers.stdout, locs, 'codegraphcontext'); + collect(callers.stderr, locs, 'codegraphcontext'); + } + const complexity = run(cgcCommand, ['analyze', 'complexity', '--limit', '80'], { cwd: repo, timeoutMs: 180000 }); + commands.push(complexity); + queries.push(complexity); + collect(complexity.stdout, locs, 'codegraphcontext'); + collect(complexity.stderr, locs, 'codegraphcontext'); + return laneResult('codegraphcontext', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: durationOf(queries) }); +} + +const result = + laneId === 'raw-native' + ? runRawNative() + : laneId === 'codebase-context' + ? runCodebaseContext() + : laneId === 'codebase-memory-mcp' + ? runCodebaseMemoryMcp() + : laneId === 'grepai' + ? runGrepai() + : runCodeGraphContext(); + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queryVariants, + readiness: result, +}; + +writeFileSync(join(outDir, `${laneId}-candidate-pack.json`), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, `${laneId}-readiness.json`), JSON.stringify(result, null, 2)); +console.log('CONTEXTBENCH_SINGLE_LANE_READINESS_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_SINGLE_LANE_READINESS_JSON_END'); +if (!result.ready) process.exitCode = 1; From 992f2ffc331fd4603b78345be9e3e64dcb05d629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:20:49 +0200 Subject: [PATCH 030/100] Add lane-parallel ContextBench readiness workflow --- .../workflows/contextbench-lane-readiness.yml | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 .github/workflows/contextbench-lane-readiness.yml diff --git a/.github/workflows/contextbench-lane-readiness.yml b/.github/workflows/contextbench-lane-readiness.yml new file mode 100644 index 0000000..db9ca84 --- /dev/null +++ b/.github/workflows/contextbench-lane-readiness.yml @@ -0,0 +1,99 @@ +name: ContextBench Lane Readiness Matrix + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-lane-readiness.yml + - scripts/contextbench-single-lane-candidate-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + lane-readiness: + runs-on: ubuntu-latest + timeout-minutes: 50 + strategy: + fail-fast: false + max-parallel: 5 + matrix: + lane: + - raw-native + - codebase-context + - codebase-memory-mcp + - grepai + - codegraphcontext + env: + ROOT: /tmp/contextbench-lane-readiness/${{ matrix.lane }} + TASK_PAYLOADS: /tmp/contextbench-lane-readiness/${{ matrix.lane }}/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts-${{ matrix.lane }} + CBM_BIN: /tmp/contextbench-lane-readiness/${{ matrix.lane }}/tool/codebase-memory-mcp + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: ${{ matrix.lane }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + - name: Build codebase-context lane + if: matrix.lane == 'codebase-context' + shell: bash + run: pnpm run build > "$ROOT/logs/build.log" 2>&1 + - name: Install codebase-memory-mcp lane + if: matrix.lane == 'codebase-memory-mcp' + shell: bash + run: | + set -euo pipefail + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + - name: Install grepai lane + if: matrix.lane == 'grepai' + shell: bash + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 || true + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + - name: Install codegraphcontext lane + if: matrix.lane == 'codegraphcontext' + shell: bash + run: python -m pip install uv codegraphcontext kuzu > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 + - name: Generate lane readiness candidate pack + shell: bash + run: node scripts/contextbench-single-lane-candidate-pack.mjs + - name: Upload lane readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-lane-readiness-${{ matrix.lane }} + path: /tmp/contextbench-lane-readiness/${{ matrix.lane }} + retention-days: 14 From 24aa8629f57d2ae02516992265c8326771c74308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:21:57 +0200 Subject: [PATCH 031/100] Install ripgrep for raw ContextBench lane --- .github/workflows/contextbench-lane-readiness.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/contextbench-lane-readiness.yml b/.github/workflows/contextbench-lane-readiness.yml index db9ca84..2b3ee22 100644 --- a/.github/workflows/contextbench-lane-readiness.yml +++ b/.github/workflows/contextbench-lane-readiness.yml @@ -63,6 +63,13 @@ jobs: fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); NODE node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + - name: Install raw-native lane + if: matrix.lane == 'raw-native' + shell: bash + run: | + set -euo pipefail + sudo apt-get update > "$ROOT/logs/apt-update.log" 2>&1 + sudo apt-get install -y ripgrep > "$ROOT/logs/apt-ripgrep.log" 2>&1 - name: Build codebase-context lane if: matrix.lane == 'codebase-context' shell: bash From b5344b616ff83e11ab78f4b4f1f33f67d4fb69c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:25:06 +0200 Subject: [PATCH 032/100] Add raw-native ContextBench readiness rerun --- .../workflows/contextbench-raw-readiness.yml | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/contextbench-raw-readiness.yml diff --git a/.github/workflows/contextbench-raw-readiness.yml b/.github/workflows/contextbench-raw-readiness.yml new file mode 100644 index 0000000..38218c9 --- /dev/null +++ b/.github/workflows/contextbench-raw-readiness.yml @@ -0,0 +1,66 @@ +name: ContextBench Raw Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-raw-readiness.yml + - scripts/contextbench-single-lane-candidate-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + raw-readiness: + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + ROOT: /tmp/contextbench-raw-readiness + TASK_PAYLOADS: /tmp/contextbench-raw-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts-raw + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: raw-native + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Materialize selected task and install ripgrep + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" "$ROOT/tool" + sudo apt-get update > "$ROOT/logs/apt-update.log" 2>&1 + sudo apt-get install -y ripgrep > "$ROOT/logs/apt-ripgrep.log" 2>&1 + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + - name: Generate raw-native readiness candidate pack + shell: bash + run: node scripts/contextbench-single-lane-candidate-pack.mjs + - name: Upload raw readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-raw-readiness + path: /tmp/contextbench-raw-readiness + retention-days: 14 From 1e8f1a396eabd8c7f9ea10bfcba488984b3811b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:25:47 +0200 Subject: [PATCH 033/100] Add CodeGraphContext content readiness script --- ...extbench-codegraphcontext-content-pack.mjs | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 scripts/contextbench-codegraphcontext-content-pack.mjs diff --git a/scripts/contextbench-codegraphcontext-content-pack.mjs b/scripts/contextbench-codegraphcontext-content-pack.mjs new file mode 100644 index 0000000..990a013 --- /dev/null +++ b/scripts/contextbench-codegraphcontext-content-pack.mjs @@ -0,0 +1,154 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-codegraphcontext-readiness'; +const outDir = join(root, 'pack'); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +const durationOf = (commands) => commands.reduce((sum, command) => sum + command.durationMs, 0); + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const list = basenameMap.get(basename(file)) || []; + list.push(file); + basenameMap.set(basename(file), list); +} + +function norm(file) { + let f = String(file || '').replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start) { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source: 'codegraphcontext' }); +} + +function collect(text, locs) { + const cleaned = String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); + const lineRe = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp)):(\d+):/gm; + let match; + while ((match = lineRe.exec(cleaned)) !== null) add(locs, match[1], match[2], match[2]); + const fileRe = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp))(?::|#L|\s+line\s+)?(\d+)?/g; + while ((match = fileRe.exec(cleaned)) !== null) add(locs, match[1], match[2] || 1, match[2] || 1); +} + +function uniq(locs, max = 120) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +const commands = []; +const locs = []; +const setup = run('codegraphcontext', ['--help'], { timeoutMs: 60000 }); +commands.push(setup); +const index = run('codegraphcontext', ['index', '.'], { cwd: repo, timeoutMs: 1200000 }); +commands.push(index); +const terms = ['metrics', 'prometheus', 'insights', 'startup', 'authorization', 'Bearer', 'token', 'header', 'authentication', 'subsonic']; +const queries = []; +for (const term of terms) { + for (const args of [['find', 'content', term], ['find', 'pattern', term]]) { + const result = run('codegraphcontext', args, { cwd: repo, timeoutMs: 180000 }); + commands.push(result); + queries.push(result); + collect(result.stdout, locs); + collect(result.stderr, locs); + } +} + +for (const [i, command] of commands.entries()) { + writeFileSync(join(outDir, `codegraphcontext-command-${i + 1}.json`), JSON.stringify({ ...command, stdout: command.stdout.slice(0, 200000), stderr: command.stderr.slice(0, 200000) }, null, 2)); +} + +const candidates = uniq(locs); +const readiness = { + lane: 'codegraphcontext', + ready: setup.status === 0 && index.status === 0 && commands.some((command) => command.status === 0) && candidates.length > 0, + setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', + indexStatus: index.status === 0 ? 'completed' : 'index_failed', + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex: { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(queries), + }, + commands: commands.map((command) => ({ command: command.command, status: command.status, error: command.error, durationMs: command.durationMs })), + candidates, +}; +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + readiness, +}; +writeFileSync(join(outDir, 'codegraphcontext-candidate-pack.json'), JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_CODEGRAPHCONTEXT_READINESS_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_CODEGRAPHCONTEXT_READINESS_JSON_END'); +if (!readiness.ready) process.exitCode = 1; From cfbbd3389b657a3d98379775fb2e8af844d39b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:26:05 +0200 Subject: [PATCH 034/100] Add CodeGraphContext ContextBench readiness rerun --- ...ontextbench-codegraphcontext-readiness.yml | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/contextbench-codegraphcontext-readiness.yml diff --git a/.github/workflows/contextbench-codegraphcontext-readiness.yml b/.github/workflows/contextbench-codegraphcontext-readiness.yml new file mode 100644 index 0000000..2a27d19 --- /dev/null +++ b/.github/workflows/contextbench-codegraphcontext-readiness.yml @@ -0,0 +1,63 @@ +name: ContextBench CodeGraphContext Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-codegraphcontext-readiness.yml + - scripts/contextbench-codegraphcontext-content-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + codegraphcontext-readiness: + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + ROOT: /tmp/contextbench-codegraphcontext-readiness + TASK_PAYLOADS: /tmp/contextbench-codegraphcontext-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts-codegraphcontext-content + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Materialize selected task and install CodeGraphContext + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow uv codegraphcontext kuzu > "$ROOT/logs/pip-install.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + - name: Generate CodeGraphContext readiness candidate pack + shell: bash + run: node scripts/contextbench-codegraphcontext-content-pack.mjs + - name: Upload CodeGraphContext readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-codegraphcontext-readiness + path: /tmp/contextbench-codegraphcontext-readiness + retention-days: 14 From 831bfb1b54f13b4fba5ac15dc1d120f1ca2dd098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:31:41 +0200 Subject: [PATCH 035/100] Add bounded ContextBench rescue lane readiness packer --- scripts/contextbench-rescue-lane-pack.mjs | 391 ++++++++++++++++++++++ 1 file changed, 391 insertions(+) create mode 100644 scripts/contextbench-rescue-lane-pack.mjs diff --git a/scripts/contextbench-rescue-lane-pack.mjs b/scripts/contextbench-rescue-lane-pack.mjs new file mode 100644 index 0000000..e326b56 --- /dev/null +++ b/scripts/contextbench-rescue-lane-pack.mjs @@ -0,0 +1,391 @@ +import { spawn, spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const laneId = process.env.LANE_ID; +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || `/tmp/contextbench-rescue-${laneId || 'unknown'}`; +const outDir = join(root, 'pack'); +const supportedLanes = ['raw-native', 'codebase-context', 'codebase-memory-mcp', 'grepai', 'codegraphcontext']; +if (!supportedLanes.includes(laneId)) throw new Error(`unsupported LANE_ID: ${laneId}`); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +const durationOf = (commands) => commands.reduce((sum, command) => sum + command.durationMs, 0); + +function queryOf(text) { + const stop = new Set(['that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', 'would', 'could', 'there', 'where', 'which', 'about', 'after', 'before', 'have', 'will', 'been', 'than', 'also', 'only', 'some', 'using', 'must']); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 24) + .join(' '); +} + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const name = basename(file); + const list = basenameMap.get(name) || []; + list.push(file); + basenameMap.set(name, list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function jsonish(s) { + const t = stripAnsi(s).trim(); + if (!t) return null; + try { + return JSON.parse(t); + } catch {} + for (const [a, b] of [['{', '}'], ['[', ']']]) { + const i = t.indexOf(a); + const j = t.lastIndexOf(b); + if (i >= 0 && j > i) { + try { + return JSON.parse(t.slice(i, j + 1)); + } catch {} + } + } + return null; +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + const marker = '/tmp/contextbench-checkouts-'; + const tmpIdx = f.indexOf(marker); + if (tmpIdx >= 0) { + const parts = f.slice(tmpIdx + marker.length).split('/'); + f = parts.slice(1).join('/'); + } + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..') || f.startsWith('tmp/')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = laneId) { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add( + locs, + value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, + value.start_line || value.startLine || value.line || value.line_number || value.start || 1, + value.end_line || value.endLine || value.end || value.line || 1, + source, + ); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source = laneId) { + const cleaned = stripAnsi(text); + const parsed = jsonish(cleaned); + if (parsed) walk(parsed, locs, source); + const rgLine = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte)):(\d+):/gm; + let m; + while ((m = rgLine.exec(cleaned)) !== null) add(locs, m[1], m[2], m[2], source); + const fileLine = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?/g; + while ((m = fileLine.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); +} + +function uniq(locs, max = 120) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function writeCommands(lane, commands) { + for (const [i, command] of commands.entries()) { + writeFileSync(join(outDir, `${lane}-command-${i + 1}.json`), JSON.stringify({ ...command, stdout: command.stdout.slice(0, 200000), stderr: command.stderr.slice(0, 200000) }, null, 2)); + } +} + +function laneResult(lane, commands, locs, setupStatus, indexStatus, setupIndex) { + writeCommands(lane, commands); + const candidates = uniq(locs); + return { + lane, + ready: (setupStatus === 'completed' || setupStatus === 'not_required') && (indexStatus === 'completed' || indexStatus === 'not_required') && commands.some((command) => command.status === 0) && candidates.length > 0, + setupStatus, + indexStatus, + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex, + commands: commands.map((command) => ({ command: command.command, status: command.status, signal: command.signal, error: command.error, durationMs: command.durationMs })), + candidates, + }; +} + +const baseQuery = queryOf(task.problem_statement); +const queryVariants = [ + baseQuery, + 'system metrics written on start prometheus writer insights', + 'Bearer token custom authorization header authentication', + 'Subsonic authorization header token parsing', + 'startup metrics insights prometheus initialization', +]; + +function runRawNative() { + const commands = []; + const locs = []; + for (const term of ['metrics', 'prometheus', 'insights', 'startup', 'start', 'written', 'authorization', 'Bearer', 'token', 'header', 'authentication', 'subsonic']) { + const r = run('rg', ['-n', '-i', '--glob', '!.git', '--glob', '!vendor/**', '--glob', '!node_modules/**', term, '.'], { cwd: repo, timeoutMs: 60000 }); + commands.push(r); + collect(r.stdout, locs, 'raw-native'); + collect(r.stderr, locs, 'raw-native'); + } + return laneResult('raw-native', commands, locs, 'not_required', 'not_required', { setupDurationMs: 0, indexDurationMs: 0, queryDurationMs: durationOf(commands) }); +} + +function runCodebaseContext() { + const commands = []; + const locs = []; + const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' }; + const setup = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 420000 }); + commands.push(index); + const searches = []; + if (index.status === 0) { + for (const q of queryVariants) { + const search = run('node', ['dist/index.js', 'search', '--query', q, '--intent', 'edit', '--limit', '40', '--json'], { env, timeoutMs: 90000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'codebase-context'); + collect(search.stderr, locs, 'codebase-context'); + } + } + return laneResult('codebase-context', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: durationOf(searches) }); +} + +function runCodebaseMemoryMcp() { + const commands = []; + const locs = []; + const env = { ...process.env, CBM_CACHE_DIR: join(outDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 1800000 }); + commands.push(index); + const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo); + const searches = []; + if (index.status === 0) { + for (const q of queryVariants) { + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query: q, limit: 50 })], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(graph); + searches.push(graph); + collect(graph.stdout, locs, 'codebase-memory-mcp'); + collect(graph.stderr, locs, 'codebase-memory-mcp'); + } + for (const term of ['metrics', 'prometheus', 'authorization', 'Bearer', 'token']) { + const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: term, mode: 'compact', limit: 50 })], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(code); + searches.push(code); + collect(code.stdout, locs, 'codebase-memory-mcp'); + collect(code.stderr, locs, 'codebase-memory-mcp'); + } + } + return laneResult('codebase-memory-mcp', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: durationOf(searches) }); +} + +async function runGrepai() { + const commands = []; + const locs = []; + const setup = run('grepai', ['version'], { timeoutMs: 60000 }); + commands.push(setup); + const init = run('grepai', ['init', '--yes', '--provider', 'synthetic', '--backend', 'gob'], { cwd: repo, timeoutMs: 120000 }); + commands.push(init); + + const started = Date.now(); + const watcher = spawn('grepai', ['watch', '--no-ui'], { cwd: repo, env: process.env, stdio: ['ignore', 'pipe', 'pipe'] }); + let watchStdout = ''; + let watchStderr = ''; + let watchExitCode = null; + let watchSignal = null; + watcher.stdout.on('data', (chunk) => { watchStdout = `${watchStdout}${chunk}`.slice(-200000); }); + watcher.stderr.on('data', (chunk) => { watchStderr = `${watchStderr}${chunk}`.slice(-200000); }); + watcher.on('exit', (code, signal) => { watchExitCode = code; watchSignal = signal; }); + + const statusChecks = []; + let indexed = false; + for (let i = 0; i < 18; i += 1) { + await sleep(i === 0 ? 3000 : 5000); + const status = run('grepai', ['status', '--no-ui'], { cwd: repo, timeoutMs: 60000 }); + commands.push(status); + statusChecks.push(status); + const text = `${status.stdout}\n${status.stderr}\n${watchStdout}\n${watchStderr}`; + if (/chunks?\D+[1-9][0-9]*|indexed files?\D+[1-9][0-9]*|files indexed\D+[1-9][0-9]*/i.test(text)) { + indexed = true; + break; + } + if (watchExitCode !== null && i > 1) break; + } + + const searches = []; + if (indexed) { + for (const q of queryVariants) { + const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '40'], { cwd: repo, timeoutMs: 90000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'grepai'); + collect(search.stderr, locs, 'grepai'); + } + } + + const stop = run('grepai', ['watch', '--stop'], { cwd: repo, timeoutMs: 60000 }); + if (watchExitCode === null) watcher.kill('SIGTERM'); + await sleep(1000); + commands.push({ + command: 'grepai watch --no-ui', + cwd: repo, + status: indexed ? 0 : watchExitCode, + signal: watchSignal, + error: null, + durationMs: Date.now() - started, + stdout: watchStdout, + stderr: watchStderr, + }); + commands.push(stop); + + return laneResult('grepai', commands, locs, setup.status === 0 && init.status === 0 ? 'completed' : 'setup_failed', indexed ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs + init.durationMs, indexDurationMs: durationOf(statusChecks), queryDurationMs: durationOf(searches), teardownDurationMs: stop.durationMs }); +} + +function runCodeGraphContext() { + const commands = []; + const locs = []; + const cgcProbe = run('cgc', ['--help'], { timeoutMs: 60000 }); + const command = cgcProbe.status === 0 ? 'cgc' : 'codegraphcontext'; + const setup = cgcProbe.status === 0 ? cgcProbe : run(command, ['--help'], { timeoutMs: 60000 }); + commands.push(setup); + const index = run(command, ['index', '.'], { cwd: repo, timeoutMs: 900000 }); + commands.push(index); + const stats = run(command, ['stats'], { cwd: repo, timeoutMs: 60000 }); + commands.push(stats); + const queries = [stats]; + collect(stats.stdout, locs, 'codegraphcontext'); + collect(stats.stderr, locs, 'codegraphcontext'); + + if (index.status === 0) { + for (const q of ['metrics', 'prometheus', 'insights', 'startup metrics', 'authorization header', 'Bearer token', 'subsonic authentication', 'helpers', 'root command']) { + const found = run(command, ['find', q], { cwd: repo, timeoutMs: 90000 }); + commands.push(found); + queries.push(found); + collect(found.stdout, locs, 'codegraphcontext'); + collect(found.stderr, locs, 'codegraphcontext'); + } + for (const args of [ + ['query', 'callers', 'main'], + ['query', 'callees', 'main'], + ['query', 'callers', 'GetUser'], + ['query', 'callees', 'GetUser'], + ['query', 'callers', 'ServeHTTP'], + ['query', 'dead-code', ''], + ]) { + const cleanArgs = args.filter(Boolean); + const q = run(command, cleanArgs, { cwd: repo, timeoutMs: 90000 }); + commands.push(q); + queries.push(q); + collect(q.stdout, locs, 'codegraphcontext'); + collect(q.stderr, locs, 'codegraphcontext'); + } + } + + return laneResult('codegraphcontext', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: durationOf(queries) }); +} + +const result = laneId === 'raw-native' + ? runRawNative() + : laneId === 'codebase-context' + ? runCodebaseContext() + : laneId === 'codebase-memory-mcp' + ? runCodebaseMemoryMcp() + : laneId === 'grepai' + ? await runGrepai() + : runCodeGraphContext(); + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queryVariants, + readiness: result, +}; + +writeFileSync(join(outDir, `${laneId}-candidate-pack.json`), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, `${laneId}-readiness.json`), JSON.stringify(result, null, 2)); +console.log('CONTEXTBENCH_RESCUE_LANE_READINESS_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_RESCUE_LANE_READINESS_JSON_END'); +if (!result.ready) process.exitCode = 1; From 072f82bcb7d8a1dbc4b6793aa75484d812551fa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:32:06 +0200 Subject: [PATCH 036/100] Add bounded ContextBench rescue readiness workflow --- .../contextbench-rescue-readiness.yml | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/workflows/contextbench-rescue-readiness.yml diff --git a/.github/workflows/contextbench-rescue-readiness.yml b/.github/workflows/contextbench-rescue-readiness.yml new file mode 100644 index 0000000..ef978f6 --- /dev/null +++ b/.github/workflows/contextbench-rescue-readiness.yml @@ -0,0 +1,91 @@ +name: ContextBench Rescue Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-rescue-readiness.yml + - scripts/contextbench-rescue-lane-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + rescue-readiness: + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + lane: + - codebase-context + - grepai + - codegraphcontext + env: + ROOT: /tmp/contextbench-rescue-readiness/${{ matrix.lane }} + TASK_PAYLOADS: /tmp/contextbench-rescue-readiness/${{ matrix.lane }}/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-rescue-checkouts-${{ matrix.lane }} + CBM_BIN: /tmp/contextbench-rescue-readiness/${{ matrix.lane }}/tool/codebase-memory-mcp + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: ${{ matrix.lane }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + - name: Build codebase-context lane + if: matrix.lane == 'codebase-context' + shell: bash + run: pnpm run build > "$ROOT/logs/build.log" 2>&1 + - name: Install grepai lane + if: matrix.lane == 'grepai' + shell: bash + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 || true + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + - name: Install CodeGraphContext lane + if: matrix.lane == 'codegraphcontext' + shell: bash + run: | + set -euo pipefail + python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 || python -m pip install codegraphcontext kuzu > "$ROOT/logs/pip-codegraphcontext-fallback.log" 2>&1 + - name: Generate rescue lane readiness candidate pack + shell: bash + run: node scripts/contextbench-rescue-lane-pack.mjs + - name: Upload rescue readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-rescue-readiness-${{ matrix.lane }} + path: /tmp/contextbench-rescue-readiness/${{ matrix.lane }} + retention-days: 14 From f2b25d228295ff3a1a72ff73d4a9758acf4c6266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:34:44 +0200 Subject: [PATCH 037/100] Add parallel ContextBench tool rescue readiness workflow --- .../contextbench-tool-rescue-readiness.yml | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/contextbench-tool-rescue-readiness.yml diff --git a/.github/workflows/contextbench-tool-rescue-readiness.yml b/.github/workflows/contextbench-tool-rescue-readiness.yml new file mode 100644 index 0000000..d689024 --- /dev/null +++ b/.github/workflows/contextbench-tool-rescue-readiness.yml @@ -0,0 +1,85 @@ +name: ContextBench Tool Rescue Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-tool-rescue-readiness.yml + - scripts/contextbench-rescue-lane-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + tool-rescue-readiness: + runs-on: ubuntu-latest + timeout-minutes: 18 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + lane: + - grepai + - codegraphcontext + env: + ROOT: /tmp/contextbench-tool-rescue-readiness/${{ matrix.lane }} + TASK_PAYLOADS: /tmp/contextbench-tool-rescue-readiness/${{ matrix.lane }}/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-tool-rescue-checkouts-${{ matrix.lane }} + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: ${{ matrix.lane }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + - name: Install grepai lane + if: matrix.lane == 'grepai' + shell: bash + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 || true + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + - name: Install CodeGraphContext lane + if: matrix.lane == 'codegraphcontext' + shell: bash + run: | + set -euo pipefail + python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 || python -m pip install codegraphcontext kuzu > "$ROOT/logs/pip-codegraphcontext-fallback.log" 2>&1 + - name: Generate rescue lane readiness candidate pack + shell: bash + run: node scripts/contextbench-rescue-lane-pack.mjs + - name: Upload tool rescue readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-tool-rescue-readiness-${{ matrix.lane }} + path: /tmp/contextbench-tool-rescue-readiness/${{ matrix.lane }} + retention-days: 14 From 7a99d132eda0e0e9fbc0165860331f3f9a58a9a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:38:14 +0200 Subject: [PATCH 038/100] Add ContextBench third-party tool diagnostics workflow --- .../contextbench-tool-diagnostics.yml | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 .github/workflows/contextbench-tool-diagnostics.yml diff --git a/.github/workflows/contextbench-tool-diagnostics.yml b/.github/workflows/contextbench-tool-diagnostics.yml new file mode 100644 index 0000000..33b080d --- /dev/null +++ b/.github/workflows/contextbench-tool-diagnostics.yml @@ -0,0 +1,147 @@ +name: ContextBench Tool Diagnostics + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-tool-diagnostics.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + diagnostics: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + lane: + - grepai + - codegraphcontext + env: + ROOT: /tmp/contextbench-tool-diagnostics/${{ matrix.lane }} + TASK_PAYLOADS: /tmp/contextbench-tool-diagnostics/${{ matrix.lane }}/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-tool-diagnostics-checkouts-${{ matrix.lane }} + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: ${{ matrix.lane }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payload = JSON.parse(fs.readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + fs.writeFileSync(`${process.env.ROOT}/repo-path.txt`, payload.tasks[0].repo_checkout_path); + NODE + - name: grepai diagnostics + if: matrix.lane == 'grepai' + shell: bash + run: | + set +euo pipefail + REPO="$(cat "$ROOT/repo-path.txt")" + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh + export PATH="$HOME/.local/bin:$HOME/bin:$PATH" + cd "$REPO" + echo '--- grepai version ---' + grepai version + echo '--- grepai init synthetic gob ---' + grepai init --yes --provider synthetic --backend gob + echo '--- .grepai/config.yaml ---' + cat .grepai/config.yaml || true + echo '--- grepai status before watch ---' + grepai status --no-ui + echo '--- grepai watch --help ---' + grepai watch --help + echo '--- grepai watch --no-ui direct ---' + timeout 45s grepai watch --no-ui + echo "watch_direct_status=$?" + echo '--- grepai status after direct watch ---' + grepai status --no-ui + echo '--- grepai watch --background ---' + grepai watch --background + echo "watch_background_status=$?" + sleep 10 + echo '--- grepai watch --status ---' + grepai watch --status + echo '--- grepai status final ---' + grepai status --no-ui + echo '--- grepai search smoke ---' + grepai search "system metrics startup prometheus" --json --compact --limit 5 + echo "search_status=$?" + grepai watch --stop + - name: CodeGraphContext diagnostics + if: matrix.lane == 'codegraphcontext' + shell: bash + run: | + set +euo pipefail + REPO="$(cat "$ROOT/repo-path.txt")" + python -m pip install codegraphcontext kuzu real_ladybug || python -m pip install codegraphcontext kuzu + cd "$REPO" + echo '--- cgc --version ---' + cgc --version + echo '--- cgc --help ---' + cgc --help + echo '--- cgc find --help ---' + cgc find --help + echo '--- cgc analyze --help ---' + cgc analyze --help + echo '--- cgc index . ---' + cgc index . + echo '--- cgc stats ---' + cgc stats + echo '--- cgc list ---' + cgc list + echo '--- cgc analyze complexity --limit 20 ---' + cgc analyze complexity --limit 20 + echo 'complexity_status='$? + echo '--- cgc analyze dead-code ---' + cgc analyze dead-code + echo 'dead_code_status='$? + echo '--- cgc analyze callers main ---' + cgc analyze callers main + echo 'callers_main_status='$? + echo '--- cgc analyze calls main ---' + cgc analyze calls main + echo 'calls_main_status='$? + echo '--- cgc find metrics variants ---' + cgc find metrics + echo 'find_metrics_status='$? + cgc find --fuzzy metrics + echo 'find_fuzzy_metrics_status='$? + - name: Upload diagnostics artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-tool-diagnostics-${{ matrix.lane }} + path: /tmp/contextbench-tool-diagnostics/${{ matrix.lane }} + retention-days: 14 From 226351326327798d3d99e25be5dde8ed9c9f78d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:43:32 +0200 Subject: [PATCH 039/100] Add bounded final ContextBench lane pack generator --- ...contextbench-final-lane-candidate-pack.mjs | 286 ++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 scripts/contextbench-final-lane-candidate-pack.mjs diff --git a/scripts/contextbench-final-lane-candidate-pack.mjs b/scripts/contextbench-final-lane-candidate-pack.mjs new file mode 100644 index 0000000..c8860db --- /dev/null +++ b/scripts/contextbench-final-lane-candidate-pack.mjs @@ -0,0 +1,286 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const laneId = process.env.LANE_ID; +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || `/tmp/contextbench-final-lane/${laneId || 'unknown'}`; +const outDir = join(root, 'pack'); +const supportedLanes = ['grepai', 'codegraphcontext']; +if (!supportedLanes.includes(laneId)) throw new Error(`unsupported LANE_ID: ${laneId}`); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +const durationOf = (commands) => commands.reduce((sum, command) => sum + command.durationMs, 0); + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const name = basename(file); + const list = basenameMap.get(name) || []; + list.push(file); + basenameMap.set(name, list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function jsonish(text) { + const cleaned = stripAnsi(text).trim(); + if (!cleaned) return null; + try { + return JSON.parse(cleaned); + } catch {} + for (const [open, close] of [['{', '}'], ['[', ']']]) { + const start = cleaned.indexOf(open); + const end = cleaned.lastIndexOf(close); + if (start >= 0 && end > start) { + try { + return JSON.parse(cleaned.slice(start, end + 1)); + } catch {} + } + } + return null; +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'tool') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add( + locs, + value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, + value.start_line || value.startLine || value.line || value.line_number || value.start || 1, + value.end_line || value.endLine || value.end || value.line || 1, + source, + ); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source) { + const cleaned = stripAnsi(text); + const parsed = jsonish(cleaned); + if (parsed) walk(parsed, locs, source); + const lineMatch = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte)):(\d+):/gm; + let m; + while ((m = lineMatch.exec(cleaned)) !== null) add(locs, m[1], m[2], m[2], source); + const fileMatch = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?/g; + while ((m = fileMatch.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); +} + +function uniq(locs, max = 160) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function writeCommands(lane, commands) { + for (const [i, command] of commands.entries()) { + writeFileSync(join(outDir, `${lane}-command-${i + 1}.json`), JSON.stringify({ + ...command, + stdout: command.stdout.slice(0, 200000), + stderr: command.stderr.slice(0, 200000), + }, null, 2)); + } +} + +function laneResult(lane, commands, locs, setupStatus, indexStatus, setupIndex) { + writeCommands(lane, commands); + const candidates = uniq(locs); + return { + lane, + ready: setupStatus === 'completed' && indexStatus === 'completed' && commands.some((command) => command.status === 0) && candidates.length > 0, + setupStatus, + indexStatus, + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex, + commands: commands.map((command) => ({ command: command.command, status: command.status, signal: command.signal, error: command.error, durationMs: command.durationMs })), + candidates, + }; +} + +function queryOf(text) { + const stop = new Set(['that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', 'would', 'could', 'there', 'where', 'which', 'about', 'after', 'before', 'have', 'will', 'been', 'than', 'also', 'only', 'some', 'using', 'must']); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 24) + .join(' '); +} + +const queryVariants = [ + queryOf(task.problem_statement), + 'system metrics written on start prometheus writer insights', + 'Bearer token custom authorization header authentication', + 'Subsonic authorization header token parsing', + 'startup metrics insights prometheus initialization', +]; + +function runGrepai() { + const commands = []; + const locs = []; + const env = { ...process.env, OLLAMA_HOST: process.env.OLLAMA_HOST || 'http://127.0.0.1:11434' }; + const setup = run('grepai', ['version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const ollama = run('ollama', ['list'], { env, timeoutMs: 60000 }); + commands.push(ollama); + const init = run('grepai', ['init', '--yes', '--provider', 'ollama', '--model', 'nomic-embed-text', '--backend', 'gob'], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(init); + const watch = run('grepai', ['watch', '--background'], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(watch); + const statusChecks = []; + for (let i = 0; i < 24; i += 1) { + const status = run('grepai', ['status', '--no-ui'], { cwd: repo, env, timeoutMs: 60000 }); + commands.push(status); + statusChecks.push(status); + if (/Files indexed:\s*[1-9]|Total chunks:\s*[1-9]/i.test(`${status.stdout}\n${status.stderr}`)) break; + run('sleep', ['5'], { timeoutMs: 10000 }); + } + const searches = []; + for (const q of queryVariants) { + const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '50'], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'grepai'); + collect(search.stderr, locs, 'grepai'); + } + const stop = run('grepai', ['watch', '--stop'], { cwd: repo, env, timeoutMs: 60000 }); + commands.push(stop); + return laneResult('grepai', commands, locs, setup.status === 0 && ollama.status === 0 && init.status === 0 ? 'completed' : 'setup_failed', watch.status === 0 && locs.length > 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs + ollama.durationMs + init.durationMs, + indexDurationMs: watch.durationMs + durationOf(statusChecks), + queryDurationMs: durationOf(searches), + teardownDurationMs: stop.durationMs, + }); +} + +function runCodeGraphContext() { + const commands = []; + const locs = []; + const env = { ...process.env, DEFAULT_DATABASE: 'kuzudb', CGC_RUNTIME_DB_TYPE: 'kuzudb' }; + const setup = run('cgc', ['--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run('cgc', ['index', '.', '--force'], { cwd: repo, env, timeoutMs: 1200000 }); + commands.push(index); + const queries = []; + const fileList = run('cgc', ['find', 'type', 'file', '--limit', '200'], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(fileList); + queries.push(fileList); + collect(fileList.stdout, locs, 'codegraphcontext'); + collect(fileList.stderr, locs, 'codegraphcontext'); + const apiFiles = run('python', ['-c', "import json; from codegraphcontext.core.database import DatabaseManager; from codegraphcontext.tools.code_finder import CodeFinder; db=DatabaseManager(); print(json.dumps(CodeFinder(db).find_by_type('file', 200))); db.close_driver()"], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(apiFiles); + queries.push(apiFiles); + collect(apiFiles.stdout, locs, 'codegraphcontext-api'); + collect(apiFiles.stderr, locs, 'codegraphcontext-api'); + for (const pattern of ['Metrics', 'Prometheus', 'Authorization', 'Bearer', 'Token', 'Subsonic', 'Header']) { + const found = run('cgc', ['find', 'pattern', pattern], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(found); + queries.push(found); + collect(found.stdout, locs, 'codegraphcontext'); + collect(found.stderr, locs, 'codegraphcontext'); + } + for (const symbol of ['main', 'init', 'WriteInitialMetrics', 'ServeHTTP', 'GetUser']) { + const callers = run('cgc', ['analyze', 'callers', symbol], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(callers); + queries.push(callers); + collect(callers.stdout, locs, 'codegraphcontext'); + collect(callers.stderr, locs, 'codegraphcontext'); + } + return laneResult('codegraphcontext', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 && locs.length > 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(queries), + }); +} + +const result = laneId === 'grepai' ? runGrepai() : runCodeGraphContext(); +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queryVariants, + readiness: result, +}; + +writeFileSync(join(outDir, `${laneId}-candidate-pack.json`), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, `${laneId}-readiness.json`), JSON.stringify(result, null, 2)); +console.log('CONTEXTBENCH_FINAL_LANE_READINESS_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_FINAL_LANE_READINESS_JSON_END'); +if (!result.ready) process.exitCode = 1; From b8159dc4f8feb0e7ee4368e1257b8dfcff7e1ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:43:58 +0200 Subject: [PATCH 040/100] Add final ContextBench lane readiness workflow --- .../contextbench-final-lane-readiness.yml | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 .github/workflows/contextbench-final-lane-readiness.yml diff --git a/.github/workflows/contextbench-final-lane-readiness.yml b/.github/workflows/contextbench-final-lane-readiness.yml new file mode 100644 index 0000000..bb80542 --- /dev/null +++ b/.github/workflows/contextbench-final-lane-readiness.yml @@ -0,0 +1,94 @@ +name: ContextBench Final Lane Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-final-lane-readiness.yml + - scripts/contextbench-final-lane-candidate-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + final-lane-readiness: + runs-on: ubuntu-latest + timeout-minutes: 35 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + lane: + - grepai + - codegraphcontext + env: + ROOT: /tmp/contextbench-final-lane/${{ matrix.lane }} + TASK_PAYLOADS: /tmp/contextbench-final-lane/${{ matrix.lane }}/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-final-checkouts-${{ matrix.lane }} + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: ${{ matrix.lane }} + OLLAMA_HOST: http://127.0.0.1:11434 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + - name: Install grepai with local Ollama embeddings + if: matrix.lane == 'grepai' + shell: bash + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + curl -fsSL https://ollama.com/install.sh | sh > "$ROOT/logs/ollama-install.log" 2>&1 + nohup ollama serve > "$ROOT/logs/ollama-serve.log" 2>&1 & + for i in {1..45}; do + if curl -fsS "$OLLAMA_HOST/api/version" > "$ROOT/logs/ollama-version.log" 2>&1; then + break + fi + sleep 2 + done + curl -fsS "$OLLAMA_HOST/api/version" >> "$ROOT/logs/ollama-version.log" 2>&1 + ollama pull nomic-embed-text > "$ROOT/logs/ollama-pull-nomic-embed-text.log" 2>&1 + - name: Install codegraphcontext lane + if: matrix.lane == 'codegraphcontext' + shell: bash + run: python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 + - name: Generate final lane readiness candidate pack + shell: bash + run: node scripts/contextbench-final-lane-candidate-pack.mjs + - name: Upload final lane readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-final-lane-readiness-${{ matrix.lane }} + path: /tmp/contextbench-final-lane/${{ matrix.lane }} + retention-days: 14 From 345c0307dcb3c5dc037cf44229a544e5b88483b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:49:24 +0200 Subject: [PATCH 041/100] Add relevant CodeGraphContext readiness pack --- ...ntextbench-cgc-relevant-candidate-pack.mjs | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 scripts/contextbench-cgc-relevant-candidate-pack.mjs diff --git a/scripts/contextbench-cgc-relevant-candidate-pack.mjs b/scripts/contextbench-cgc-relevant-candidate-pack.mjs new file mode 100644 index 0000000..869cff5 --- /dev/null +++ b/scripts/contextbench-cgc-relevant-candidate-pack.mjs @@ -0,0 +1,199 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-cgc-relevant'; +const outDir = join(root, 'pack'); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const name = basename(file); + const list = basenameMap.get(name) || []; + list.push(file); + basenameMap.set(name, list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'codegraphcontext') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function parseJson(text) { + const cleaned = stripAnsi(text).trim(); + if (!cleaned) return null; + try { return JSON.parse(cleaned); } catch {} + const start = cleaned.indexOf('['); + const end = cleaned.lastIndexOf(']'); + if (start >= 0 && end > start) { + try { return JSON.parse(cleaned.slice(start, end + 1)); } catch {} + } + return null; +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add(locs, value.path || value.file || value.file_path || value.name || value.uri, value.line_number || value.line || 1, value.end_line || value.line_number || 1, source); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source) { + const parsed = parseJson(text); + if (parsed) walk(parsed, locs, source); + const cleaned = stripAnsi(text); + const fileLine = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?/g; + let m; + while ((m = fileLine.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); +} + +const relevantTerms = ['metrics', 'prometheus', 'insights', 'auth', 'authorization', 'bearer', 'token', 'header', 'subsonic', 'root']; +function isRelevant(loc) { + const file = loc.file.toLowerCase(); + return relevantTerms.some((term) => file.includes(term)); +} + +function uniq(locs, max = 80) { + const seen = new Set(); + const out = []; + for (const loc of locs.filter(isRelevant)) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function writeCommands(commands) { + for (const [i, command] of commands.entries()) { + writeFileSync(join(outDir, `codegraphcontext-command-${i + 1}.json`), JSON.stringify({ + ...command, + stdout: command.stdout.slice(0, 200000), + stderr: command.stderr.slice(0, 200000), + }, null, 2)); + } +} + +const env = { ...process.env, DEFAULT_DATABASE: 'kuzudb', CGC_RUNTIME_DB_TYPE: 'kuzudb' }; +const commands = []; +const locs = []; +const setup = run('cgc', ['--version'], { env, timeoutMs: 60000 }); +commands.push(setup); +const index = run('cgc', ['index', '.', '--force'], { cwd: repo, env, timeoutMs: 1200000 }); +commands.push(index); +const queryCommands = []; +for (const term of relevantTerms) { + const apiQuery = run('python', ['-c', `import json\nfrom codegraphcontext.core.database import DatabaseManager\ndb=DatabaseManager()\nwith db.get_driver().session() as s:\n rows=s.run("MATCH (f:File) WHERE toLower(f.path) CONTAINS '${term}' RETURN f.path as path LIMIT 100").data()\nprint(json.dumps(rows))\ndb.close_driver()`], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(apiQuery); + queryCommands.push(apiQuery); + collect(apiQuery.stdout, locs, `codegraphcontext-query-${term}`); + collect(apiQuery.stderr, locs, `codegraphcontext-query-${term}`); +} +for (const pattern of ['Metrics', 'Prometheus', 'Authorization', 'Bearer', 'Token', 'Subsonic', 'Header']) { + const found = run('cgc', ['find', 'pattern', pattern], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(found); + queryCommands.push(found); + collect(found.stdout, locs, 'codegraphcontext-pattern'); + collect(found.stderr, locs, 'codegraphcontext-pattern'); +} + +writeCommands(commands); +const candidates = uniq(locs); +const result = { + lane: 'codegraphcontext', + ready: setup.status === 0 && index.status === 0 && candidates.length > 0, + setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', + indexStatus: index.status === 0 && candidates.length > 0 ? 'completed' : 'index_failed', + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex: { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: queryCommands.reduce((sum, command) => sum + command.durationMs, 0), + }, + commands: commands.map((command) => ({ command: command.command, status: command.status, signal: command.signal, error: command.error, durationMs: command.durationMs })), + candidates, +}; + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + readiness: result, +}; +writeFileSync(join(outDir, 'codegraphcontext-candidate-pack.json'), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, 'codegraphcontext-readiness.json'), JSON.stringify(result, null, 2)); +console.log('CONTEXTBENCH_CGC_RELEVANT_READINESS_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_CGC_RELEVANT_READINESS_JSON_END'); +if (!result.ready) process.exitCode = 1; From 551d78e0c431c918beb5a646c1ebd5dba69924b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:49:44 +0200 Subject: [PATCH 042/100] Add relevant CodeGraphContext readiness workflow --- .../contextbench-cgc-relevant-readiness.yml | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/contextbench-cgc-relevant-readiness.yml diff --git a/.github/workflows/contextbench-cgc-relevant-readiness.yml b/.github/workflows/contextbench-cgc-relevant-readiness.yml new file mode 100644 index 0000000..233f6ba --- /dev/null +++ b/.github/workflows/contextbench-cgc-relevant-readiness.yml @@ -0,0 +1,74 @@ +name: ContextBench CodeGraphContext Relevant Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-cgc-relevant-readiness.yml + - scripts/contextbench-cgc-relevant-candidate-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + cgc-relevant-readiness: + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + ROOT: /tmp/contextbench-cgc-relevant + TASK_PAYLOADS: /tmp/contextbench-cgc-relevant/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-cgc-relevant-checkouts + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Install CodeGraphContext + shell: bash + run: python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 + - name: Generate relevant readiness pack + shell: bash + run: node scripts/contextbench-cgc-relevant-candidate-pack.mjs + - name: Upload relevant CGC readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-cgc-relevant-readiness + path: /tmp/contextbench-cgc-relevant + retention-days: 14 From b69cc59964f7b0c2fa05f33b8acf2ea243ae90c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:51:57 +0200 Subject: [PATCH 043/100] Allow scoring selected ready ContextBench lanes --- scripts/contextbench-score-five-lane-selections.mjs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/contextbench-score-five-lane-selections.mjs b/scripts/contextbench-score-five-lane-selections.mjs index 0029a67..2a103a0 100644 --- a/scripts/contextbench-score-five-lane-selections.mjs +++ b/scripts/contextbench-score-five-lane-selections.mjs @@ -6,7 +6,10 @@ const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenan const root = process.env.ROOT || '/tmp/contextbench-five-lane-score'; const officialContextBench = process.env.OFFICIAL_CONTEXTBENCH; const selectionsPath = process.env.SELECTIONS_PATH || 'scripts/contextbench-five-lane-selections.json'; -const requiredLanes = ['raw-native', 'codebase-context', 'codebase-memory-mcp', 'grepai', 'codegraphcontext']; +const requiredLanes = (process.env.REQUIRED_LANES || 'raw-native,codebase-context,codebase-memory-mcp,grepai,codegraphcontext') + .split(',') + .map((lane) => lane.trim()) + .filter(Boolean); const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); @@ -79,7 +82,7 @@ function resultTableRow(row) { }; } -const runDir = join(root, 'five-lane-score'); +const runDir = join(root, 'lane-score'); mkdirSync(runDir, { recursive: true }); writeFileSync(join(runDir, 'selections.json'), JSON.stringify(selections, null, 2)); @@ -199,7 +202,7 @@ const summary = { writeFileSync(join(runDir, 'summary.json'), JSON.stringify(summary, null, 2)); writeFileSync(join(root, 'summary.json'), JSON.stringify(summary, null, 2)); -console.log('CONTEXTBENCH_FIVE_LANE_SCORE_JSON_START'); +console.log('CONTEXTBENCH_LANE_SCORE_JSON_START'); console.log(JSON.stringify(summary, null, 2)); -console.log('CONTEXTBENCH_FIVE_LANE_SCORE_JSON_END'); +console.log('CONTEXTBENCH_LANE_SCORE_JSON_END'); if (scoreableRows.length !== rows.length || scoreableRows.length < requiredLanes.length) process.exitCode = 1; From 8059f868435accd5b0bda55e90b2a99f5f5dbde0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:52:52 +0200 Subject: [PATCH 044/100] Add ready lane selections for ContextBench scoring --- .../contextbench-ready-lane-selections.json | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 scripts/contextbench-ready-lane-selections.json diff --git a/scripts/contextbench-ready-lane-selections.json b/scripts/contextbench-ready-lane-selections.json new file mode 100644 index 0000000..74ca273 --- /dev/null +++ b/scripts/contextbench-ready-lane-selections.json @@ -0,0 +1,83 @@ +{ + "model": "gpt-5.4-mini-high", + "predictionSource": "gpt-5.4-mini-high selector subagents over real readiness candidate packs", + "caveats": [ + "This selection set intentionally excludes grepai and codegraphcontext because their strict readiness gates produced candidateCount=0 and are not scoreable quality rows.", + "grepai installed and searched with Ollama, but indexing failed and no candidates were returned.", + "CodeGraphContext installed and indexed, but strict task-relevant retrieval returned no candidates; the earlier broad file-list run is not counted as readiness evidence." + ], + "laneSelections": [ + { + "lane_id": "raw-native", + "files": ["cmd/root.go", "core/metrics/prometheus.go", "server/subsonic/helpers.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 15, "end": 123}, + {"file": "cmd/root.go", "start": 194, "end": 220}, + {"file": "server/subsonic/helpers.go", "start": 129, "end": 201}, + {"file": "server/subsonic/helpers.go", "start": 256, "end": 268} + ], + "rationale": "cmd/root.go covers startup wiring, core/metrics/prometheus.go contains the startup metrics implementation path, and server/subsonic/helpers.go covers Subsonic authorization token parsing branches.", + "readiness": { + "setupStatus": "not_required", + "indexStatus": "not_required", + "toolCallable": true, + "candidateCount": 120, + "setupIndex": {"setupDurationMs": 0, "indexDurationMs": 0, "queryDurationMs": 130}, + "sourceRun": 25642669946, + "sourceJob": 75265688816 + } + }, + { + "lane_id": "codebase-context", + "files": ["core/metrics/prometheus.go", "server/auth.go", "server/server.go", "scanner/scanner.go", "cmd/root.go", "cmd/wire_gen.go", "conf/configuration.go", "consts/consts.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 1, "end": 160}, + {"file": "server/auth.go", "start": 165, "end": 230}, + {"file": "server/server.go", "start": 170, "end": 185}, + {"file": "scanner/scanner.go", "start": 50, "end": 230}, + {"file": "cmd/root.go", "start": 100, "end": 170}, + {"file": "cmd/wire_gen.go", "start": 60, "end": 140}, + {"file": "conf/configuration.go", "start": 145, "end": 155}, + {"file": "conf/configuration.go", "start": 425, "end": 432}, + {"file": "consts/consts.go", "start": 68, "end": 82} + ], + "rationale": "This is the selector output over the real codebase-context candidate pack, covering metrics startup wiring plus auth/header/config surfaces it judged relevant.", + "readiness": { + "setupStatus": "completed", + "indexStatus": "completed", + "toolCallable": true, + "candidateCount": 36, + "setupIndex": {"setupDurationMs": 570, "indexDurationMs": null, "queryDurationMs": null}, + "sourceRun": 25642303130, + "sourceJob": 75264700845 + } + }, + { + "lane_id": "codebase-memory-mcp", + "files": ["core/metrics/prometheus.go", "cmd/root.go", "server/subsonic/helpers.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 15, "end": 17}, + {"file": "core/metrics/prometheus.go", "start": 19, "end": 25}, + {"file": "core/metrics/prometheus.go", "start": 33, "end": 38}, + {"file": "core/metrics/prometheus.go", "start": 40, "end": 49}, + {"file": "core/metrics/prometheus.go", "start": 51, "end": 100}, + {"file": "core/metrics/prometheus.go", "start": 102, "end": 123}, + {"file": "cmd/root.go", "start": 194, "end": 201}, + {"file": "cmd/root.go", "start": 204, "end": 220}, + {"file": "server/subsonic/helpers.go", "start": 129, "end": 187}, + {"file": "server/subsonic/helpers.go", "start": 189, "end": 201}, + {"file": "server/subsonic/helpers.go", "start": 256, "end": 268} + ], + "rationale": "These spans cover the startup metrics write path, the startup wiring that should invoke it, and the Subsonic auth-token parsing path.", + "readiness": { + "setupStatus": "completed", + "indexStatus": "completed", + "toolCallable": true, + "candidateCount": 100, + "setupIndex": {"setupDurationMs": 4, "indexDurationMs": 2038, "queryDurationMs": 365}, + "sourceRun": 25642583473, + "sourceJob": 75265447999 + } + } + ] +} From 5a7a74707bda1a0722017b1373a80b17acb3a4df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 01:53:15 +0200 Subject: [PATCH 045/100] Add ready lane ContextBench scoring workflow --- .../contextbench-ready-lane-score.yml | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/contextbench-ready-lane-score.yml diff --git a/.github/workflows/contextbench-ready-lane-score.yml b/.github/workflows/contextbench-ready-lane-score.yml new file mode 100644 index 0000000..6d2b804 --- /dev/null +++ b/.github/workflows/contextbench-ready-lane-score.yml @@ -0,0 +1,76 @@ +name: ContextBench Ready Lane Score + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-ready-lane-score.yml + - scripts/contextbench-score-five-lane-selections.mjs + - scripts/contextbench-ready-lane-selections.json + workflow_dispatch: + +permissions: + contents: read + +jobs: + ready-lane-score: + runs-on: ubuntu-latest + timeout-minutes: 35 + env: + ROOT: /tmp/contextbench-ready-lane-score + TASK_PAYLOADS: /tmp/contextbench-ready-lane-score/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-ready-lane-score-checkouts + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-ready-lane-score/ContextBench-official + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + SELECTIONS_PATH: scripts/contextbench-ready-lane-selections.json + REQUIRED_LANES: raw-native,codebase-context,codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-install.log" 2>&1 + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$OFFICIAL_CONTEXTBENCH" > "$ROOT/logs/contextbench-clone.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Score ready lane selections + shell: bash + run: node scripts/contextbench-score-five-lane-selections.mjs + - name: Upload ready lane score artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-ready-lane-score + path: /tmp/contextbench-ready-lane-score + retention-days: 14 From 0d46c80a43ecf0f427c55a4f3c02c47c5b0d2e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:01:38 +0200 Subject: [PATCH 046/100] Add infra-backed ContextBench lane readiness --- scripts/contextbench-infra-lane-readiness.mjs | 371 ++++++++++++++++++ 1 file changed, 371 insertions(+) create mode 100644 scripts/contextbench-infra-lane-readiness.mjs diff --git a/scripts/contextbench-infra-lane-readiness.mjs b/scripts/contextbench-infra-lane-readiness.mjs new file mode 100644 index 0000000..e895c3b --- /dev/null +++ b/scripts/contextbench-infra-lane-readiness.mjs @@ -0,0 +1,371 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const laneId = process.env.LANE_ID; +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || `/tmp/contextbench-infra-lane/${laneId || 'unknown'}`; +const outDir = join(root, 'pack'); +const logsDir = join(root, 'logs'); +const supportedLanes = ['grepai', 'codegraphcontext']; +if (!supportedLanes.includes(laneId)) throw new Error(`unsupported LANE_ID: ${laneId}`); +mkdirSync(outDir, { recursive: true }); +mkdirSync(logsDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +const durationOf = (commands) => commands.reduce((sum, command) => sum + command.durationMs, 0); + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const name = basename(file); + const list = basenameMap.get(name) || []; + list.push(file); + basenameMap.set(name, list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'tool') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function jsonish(text) { + const cleaned = stripAnsi(text).trim(); + if (!cleaned) return null; + try { + return JSON.parse(cleaned); + } catch {} + for (const [open, close] of [['{', '}'], ['[', ']']]) { + const start = cleaned.indexOf(open); + const end = cleaned.lastIndexOf(close); + if (start >= 0 && end > start) { + try { + return JSON.parse(cleaned.slice(start, end + 1)); + } catch {} + } + } + return null; +} + +function pathFromObject(value) { + return value.file || value.path || value.file_path || value.filePath || value.relative_path || value.filename || value.source_path || value.uri || value.caller_file_path || value.called_file_path || value.importer_file_path || value.parent_file_path || value.child_file_path || value.class_file_path; +} + +function startFromObject(value) { + return value.start_line || value.startLine || value.line || value.line_number || value.start || value.caller_line_number || value.called_line_number || value.import_line_number || value.parent_line_number || value.child_line_number || value.function_line_number || 1; +} + +function endFromObject(value) { + return value.end_line || value.endLine || value.end || value.line || value.line_number || value.endLineNumber || startFromObject(value); +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add(locs, pathFromObject(value), startFromObject(value), endFromObject(value), source); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source) { + const cleaned = stripAnsi(text); + const parsed = jsonish(cleaned); + if (parsed) walk(parsed, locs, source); + const lineMatch = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte)):(\d+)(?:-(\d+))?/gm; + let m; + while ((m = lineMatch.exec(cleaned)) !== null) add(locs, m[1], m[2], m[3] || m[2], source); + const fileMatch = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?(?:-(\d+))?/g; + while ((m = fileMatch.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[3] || m[2] || 1, source); +} + +const relevantPathTerms = ['metrics', 'prometheus', 'auth', 'subsonic', 'root', 'header', 'token']; +function collectRelevantInventory(text, locs, source) { + const inventory = []; + collect(text, inventory, source); + for (const loc of inventory) { + const file = loc.file.toLowerCase(); + if (relevantPathTerms.some((term) => file.includes(term))) locs.push({ ...loc, source }); + } +} + +function uniq(locs, max = 160) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function writeCommands(lane, commands) { + for (const [i, command] of commands.entries()) { + writeFileSync(join(outDir, `${lane}-command-${i + 1}.json`), JSON.stringify({ + ...command, + stdout: command.stdout.slice(0, 300000), + stderr: command.stderr.slice(0, 300000), + }, null, 2)); + } +} + +function commandSummary(command) { + return { + command: command.command, + status: command.status, + signal: command.signal, + error: command.error, + durationMs: command.durationMs, + stdoutExcerpt: stripAnsi(command.stdout).slice(0, 1200), + stderrExcerpt: stripAnsi(command.stderr).slice(0, 1200), + }; +} + +function laneResult(lane, commands, locs, setupStatus, indexStatus, setupIndex, notes = []) { + writeCommands(lane, commands); + const candidates = uniq(locs); + return { + lane, + ready: setupStatus === 'completed' && indexStatus === 'completed' && commands.some((command) => command.status === 0) && candidates.length > 0, + setupStatus, + indexStatus, + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex, + notes, + commands: commands.map(commandSummary), + candidates, + }; +} + +function queryOf(text) { + const stop = new Set(['that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', 'would', 'could', 'there', 'where', 'which', 'about', 'after', 'before', 'have', 'will', 'been', 'than', 'also', 'only', 'some', 'using', 'must']); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 24) + .join(' '); +} + +const queryVariants = [ + queryOf(task.problem_statement), + 'system metrics written on start prometheus writer insights', + 'Bearer token custom authorization header authentication', + 'Subsonic authorization header token parsing', + 'startup metrics insights prometheus initialization', +]; + +function runGrepai() { + const commands = []; + const locs = []; + const env = { ...process.env, OLLAMA_HOST: process.env.OLLAMA_HOST || 'http://127.0.0.1:11434' }; + const setup = run('grepai', ['version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const ollama = run('ollama', ['list'], { env, timeoutMs: 60000 }); + commands.push(ollama); + const init = run('grepai', ['init', '--yes', '--provider', 'ollama', '--model', 'nomic-embed-text', '--backend', 'gob'], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(init); + const indexScript = ` +set -euo pipefail +LOG=${JSON.stringify(join(logsDir, 'grepai-watch-no-ui.log'))} +STATUSLOG=${JSON.stringify(join(logsDir, 'grepai-status-loop.log'))} +: > "$LOG" +: > "$STATUSLOG" +grepai watch --no-ui > "$LOG" 2>&1 & +pid=$! +ready=0 +for i in $(seq 1 180); do + status="$(grepai status --no-ui 2>&1 || true)" + printf '--- status attempt %s ---\n%s\n' "$i" "$status" >> "$STATUSLOG" + if printf '%s\n' "$status" | grep -Eiq 'Files indexed:[[:space:]]*[1-9]|Total chunks:[[:space:]]*[1-9]|chunks created|files indexed'; then + ready=1 + break + fi + if grep -Eiq 'Initial scan complete|[0-9]+ files indexed|[0-9]+ chunks created' "$LOG"; then + ready=1 + break + fi + if ! kill -0 "$pid" 2>/dev/null; then + echo 'grepai watch exited before readiness' + tail -200 "$LOG" || true + exit 1 + fi + sleep 2 +done +if [ "$ready" -ne 1 ]; then + echo 'grepai watch did not report readiness before timeout' + tail -200 "$STATUSLOG" || true + tail -200 "$LOG" || true + kill -INT "$pid" 2>/dev/null || true + wait "$pid" || true + exit 1 +fi +kill -INT "$pid" 2>/dev/null || true +wait "$pid" || true +grepai status --no-ui || true +tail -200 "$STATUSLOG" || true +tail -200 "$LOG" || true +`; + const index = run('bash', ['-lc', indexScript], { cwd: repo, env, timeoutMs: 900000 }); + commands.push(index); + const searches = []; + for (const q of queryVariants) { + const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '50'], { cwd: repo, env, timeoutMs: 240000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'grepai-search'); + collect(search.stderr, locs, 'grepai-search'); + } + return laneResult('grepai', commands, locs, setup.status === 0 && ollama.status === 0 && init.status === 0 ? 'completed' : 'setup_failed', index.status === 0 && locs.length > 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs + ollama.durationMs + init.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(searches), + }, ['Uses grepai foreground --no-ui indexing in CI because documented background mode can return failure before long initial scans finish.']); +} + +function runCodeGraphContext() { + const commands = []; + const locs = []; + const env = { ...process.env, DEFAULT_DATABASE: 'kuzudb', CGC_RUNTIME_DB_TYPE: 'kuzudb' }; + const setup = run('cgc', ['--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run('cgc', ['--database', 'kuzudb', 'index', '.', '--force'], { cwd: repo, env, timeoutMs: 1200000 }); + commands.push(index); + const queries = []; + const stats = run('cgc', ['--database', 'kuzudb', 'stats', '--all'], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(stats); + queries.push(stats); + const fileInventory = run('cgc', ['--database', 'kuzudb', 'find', 'type', 'file'], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(fileInventory); + queries.push(fileInventory); + collectRelevantInventory(fileInventory.stdout, locs, 'codegraphcontext-file-inventory-filtered'); + collectRelevantInventory(fileInventory.stderr, locs, 'codegraphcontext-file-inventory-filtered'); + for (const term of ['metrics', 'prometheus', 'authorization', 'bearer', 'token', 'header', 'subsonic', 'startup']) { + for (const mode of ['pattern', 'content']) { + const found = run('cgc', ['--database', 'kuzudb', 'find', mode, term], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(found); + queries.push(found); + collect(found.stdout, locs, `codegraphcontext-${mode}-${term}`); + collect(found.stderr, locs, `codegraphcontext-${mode}-${term}`); + } + } + const py = ` +import json +from codegraphcontext.core.database import DatabaseManager +from codegraphcontext.tools.code_finder import CodeFinder +terms = ['metrics','prometheus','authorization','bearer','token','header','subsonic','startup'] +functions = ['init','main','ServeHTTP','getUser','authenticate','Prometheus','WriteInitialMetrics'] +db = DatabaseManager() +cf = CodeFinder(db) +out = {'files': cf.find_by_type('file', 1000), 'content': {}, 'related': {}, 'functions': {}} +for term in terms: + try: + out['content'][term] = cf.find_by_content(term) + except Exception as exc: + out['content'][term] = {'error': str(exc)} + try: + out['related'][term] = cf.find_related_code(term, True, 2) + except Exception as exc: + out['related'][term] = {'error': str(exc)} +for fn in functions: + try: + out['functions'][fn] = cf.find_by_function_name(fn, True) + except Exception as exc: + out['functions'][fn] = {'error': str(exc)} +print(json.dumps(out)) +db.close_driver() +`; + const api = run('python', ['-c', py], { cwd: repo, env, timeoutMs: 300000 }); + commands.push(api); + queries.push(api); + collect(api.stdout, locs, 'codegraphcontext-codefinder'); + collect(api.stderr, locs, 'codegraphcontext-codefinder'); + return laneResult('codegraphcontext', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 && locs.length > 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(queries), + }, ['Uses documented cgc find/query surface plus CodeFinder API; avoids raw Cypher against backend-specific driver wrappers.']); +} + +const result = laneId === 'grepai' ? runGrepai() : runCodeGraphContext(); +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queryVariants, + readiness: result, +}; + +writeFileSync(join(outDir, `${laneId}-candidate-pack.json`), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, `${laneId}-readiness.json`), JSON.stringify(result, null, 2)); +console.log('CONTEXTBENCH_INFRA_LANE_READINESS_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_INFRA_LANE_READINESS_JSON_END'); +if (!result.ready) process.exitCode = 1; From 0c617e7243c8306dbb9468b4f020e24465a6b5e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:02:08 +0200 Subject: [PATCH 047/100] Add infra-backed ContextBench readiness workflow --- .../contextbench-infra-lane-readiness.yml | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 .github/workflows/contextbench-infra-lane-readiness.yml diff --git a/.github/workflows/contextbench-infra-lane-readiness.yml b/.github/workflows/contextbench-infra-lane-readiness.yml new file mode 100644 index 0000000..de20006 --- /dev/null +++ b/.github/workflows/contextbench-infra-lane-readiness.yml @@ -0,0 +1,103 @@ +name: ContextBench Infra Lane Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-infra-lane-readiness.yml + - scripts/contextbench-infra-lane-readiness.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + infra-lane-readiness: + runs-on: ubuntu-latest + timeout-minutes: 55 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + lane: + - grepai + - codegraphcontext + env: + ROOT: /tmp/contextbench-infra-lane/${{ matrix.lane }} + TASK_PAYLOADS: /tmp/contextbench-infra-lane/${{ matrix.lane }}/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-infra-checkouts-${{ matrix.lane }} + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: ${{ matrix.lane }} + OLLAMA_HOST: http://127.0.0.1:11434 + OLLAMA_NUM_PARALLEL: 1 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Install grepai with local Ollama embeddings + if: matrix.lane == 'grepai' + shell: bash + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + curl -fsSL https://ollama.com/install.sh | sh > "$ROOT/logs/ollama-install.log" 2>&1 + nohup ollama serve > "$ROOT/logs/ollama-serve.log" 2>&1 & + for i in {1..45}; do + if curl -fsS "$OLLAMA_HOST/api/version" > "$ROOT/logs/ollama-version.log" 2>&1; then + break + fi + sleep 2 + done + curl -fsS "$OLLAMA_HOST/api/version" >> "$ROOT/logs/ollama-version.log" 2>&1 + ollama pull nomic-embed-text > "$ROOT/logs/ollama-pull-nomic-embed-text.log" 2>&1 + - name: Install CodeGraphContext lane + if: matrix.lane == 'codegraphcontext' + shell: bash + run: python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 + - name: Generate infra lane readiness candidate pack + shell: bash + run: node scripts/contextbench-infra-lane-readiness.mjs + - name: Upload infra lane readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-infra-lane-readiness-${{ matrix.lane }} + path: /tmp/contextbench-infra-lane/${{ matrix.lane }} + retention-days: 14 From b964a585ce1fa8ecc699341c2dc11e963f0929a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:05:44 +0200 Subject: [PATCH 048/100] Fix grepai readiness loop logging --- scripts/contextbench-infra-lane-readiness.mjs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/contextbench-infra-lane-readiness.mjs b/scripts/contextbench-infra-lane-readiness.mjs index e895c3b..61cea0b 100644 --- a/scripts/contextbench-infra-lane-readiness.mjs +++ b/scripts/contextbench-infra-lane-readiness.mjs @@ -238,12 +238,12 @@ pid=$! ready=0 for i in $(seq 1 180); do status="$(grepai status --no-ui 2>&1 || true)" - printf '--- status attempt %s ---\n%s\n' "$i" "$status" >> "$STATUSLOG" - if printf '%s\n' "$status" | grep -Eiq 'Files indexed:[[:space:]]*[1-9]|Total chunks:[[:space:]]*[1-9]|chunks created|files indexed'; then + printf -- '--- status attempt %s ---\n%s\n' "$i" "$status" >> "$STATUSLOG" + if printf -- '%s\n' "$status" | grep -Eiq 'Files indexed:[[:space:]]*[1-9][0-9]*|Total chunks:[[:space:]]*[1-9][0-9]*|[1-9][0-9]* chunks created|[1-9][0-9]* files indexed'; then ready=1 break fi - if grep -Eiq 'Initial scan complete|[0-9]+ files indexed|[0-9]+ chunks created' "$LOG"; then + if grep -Eiq 'Initial scan complete|[1-9][0-9]* files indexed|[1-9][0-9]* chunks created' "$LOG"; then ready=1 break fi @@ -263,6 +263,13 @@ if [ "$ready" -ne 1 ]; then exit 1 fi kill -INT "$pid" 2>/dev/null || true +for i in $(seq 1 60); do + if ! kill -0 "$pid" 2>/dev/null; then + break + fi + sleep 1 +done +kill -TERM "$pid" 2>/dev/null || true wait "$pid" || true grepai status --no-ui || true tail -200 "$STATUSLOG" || true From eb652853dd554498797529c0b43e209fea7e5c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:10:53 +0200 Subject: [PATCH 049/100] Use infra readiness script for CodeGraphContext lane --- ...ontextbench-codegraphcontext-readiness.yml | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/workflows/contextbench-codegraphcontext-readiness.yml b/.github/workflows/contextbench-codegraphcontext-readiness.yml index 2a27d19..e423014 100644 --- a/.github/workflows/contextbench-codegraphcontext-readiness.yml +++ b/.github/workflows/contextbench-codegraphcontext-readiness.yml @@ -5,7 +5,7 @@ on: branches: [master] paths: - .github/workflows/contextbench-codegraphcontext-readiness.yml - - scripts/contextbench-codegraphcontext-content-pack.mjs + - scripts/contextbench-infra-lane-readiness.mjs workflow_dispatch: permissions: @@ -14,12 +14,13 @@ permissions: jobs: codegraphcontext-readiness: runs-on: ubuntu-latest - timeout-minutes: 25 + timeout-minutes: 35 env: - ROOT: /tmp/contextbench-codegraphcontext-readiness - TASK_PAYLOADS: /tmp/contextbench-codegraphcontext-readiness/task-payloads.json - CHECKOUT_ROOT: /tmp/contextbench-checkouts-codegraphcontext-content + ROOT: /tmp/contextbench-infra-lane/codegraphcontext + TASK_PAYLOADS: /tmp/contextbench-infra-lane/codegraphcontext/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-infra-checkouts-codegraphcontext TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: codegraphcontext steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 @@ -32,15 +33,19 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Materialize selected task and install CodeGraphContext + - name: Install shared setup and materialize selected Go task shell: bash run: | set -euo pipefail - mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 - python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow uv codegraphcontext kuzu > "$ROOT/logs/pip-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 - node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done node - <<'NODE' const fs = require('node:fs'); const payloadPath = process.env.TASK_PAYLOADS; @@ -50,14 +55,21 @@ jobs: if (!task) throw new Error(`target task ${target} not found`); fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); NODE - node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Install CodeGraphContext lane + shell: bash + run: python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 - name: Generate CodeGraphContext readiness candidate pack shell: bash - run: node scripts/contextbench-codegraphcontext-content-pack.mjs + run: node scripts/contextbench-infra-lane-readiness.mjs - name: Upload CodeGraphContext readiness artifacts if: always() uses: actions/upload-artifact@v4 with: name: contextbench-codegraphcontext-readiness - path: /tmp/contextbench-codegraphcontext-readiness + path: /tmp/contextbench-infra-lane/codegraphcontext retention-days: 14 From 8865e8d2c26cc8d24cc0eca4efd801ced0e52cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:15:53 +0200 Subject: [PATCH 050/100] Add focused GrepAI readiness pack script --- scripts/contextbench-grepai-ready-pack.mjs | 269 +++++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 scripts/contextbench-grepai-ready-pack.mjs diff --git a/scripts/contextbench-grepai-ready-pack.mjs b/scripts/contextbench-grepai-ready-pack.mjs new file mode 100644 index 0000000..a51e184 --- /dev/null +++ b/scripts/contextbench-grepai-ready-pack.mjs @@ -0,0 +1,269 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-grepai-readiness'; +const outDir = join(root, 'pack'); +const logsDir = join(root, 'logs'); +mkdirSync(outDir, { recursive: true }); +mkdirSync(logsDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const list = basenameMap.get(basename(file)) || []; + list.push(file); + basenameMap.set(basename(file), list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'grepai-search') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function jsonish(text) { + const cleaned = stripAnsi(text).trim(); + if (!cleaned) return null; + try { return JSON.parse(cleaned); } catch {} + for (const [open, close] of [['{', '}'], ['[', ']']]) { + const start = cleaned.indexOf(open); + const end = cleaned.lastIndexOf(close); + if (start >= 0 && end > start) { + try { return JSON.parse(cleaned.slice(start, end + 1)); } catch {} + } + } + return null; +} + +function pathFromObject(value) { + return value.file || value.path || value.file_path || value.filePath || value.relative_path || value.filename || value.source_path || value.uri; +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add(locs, pathFromObject(value), value.start_line || value.startLine || value.line || value.start || 1, value.end_line || value.endLine || value.end || value.line || 1, source); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source) { + const cleaned = stripAnsi(text); + const parsed = jsonish(cleaned); + if (parsed) walk(parsed, locs, source); + const lineMatch = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte)):(\d+)(?:-(\d+))?/gm; + let m; + while ((m = lineMatch.exec(cleaned)) !== null) add(locs, m[1], m[2], m[3] || m[2], source); + const fileMatch = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?(?:-(\d+))?/g; + while ((m = fileMatch.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[3] || m[2] || 1, source); +} + +function uniq(locs, max = 200) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function queryOf(text) { + const stop = new Set(['that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', 'would', 'could', 'there', 'where', 'which', 'about', 'after', 'before', 'have', 'will', 'been', 'than', 'also', 'only', 'some', 'using', 'must']); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 24) + .join(' '); +} + +const queryVariants = [ + queryOf(task.problem_statement), + 'system metrics written on start prometheus writer insights', + 'Bearer token custom authorization header authentication', + 'Subsonic authorization header token parsing', + 'startup metrics insights prometheus initialization', +]; + +const env = { ...process.env, OLLAMA_HOST: process.env.OLLAMA_HOST || 'http://127.0.0.1:11434' }; +const commands = []; +const locs = []; +const setup = run('grepai', ['version'], { env, timeoutMs: 60000 }); +commands.push(setup); +const ollama = run('ollama', ['list'], { env, timeoutMs: 60000 }); +commands.push(ollama); +const init = run('grepai', ['init', '--yes', '--provider', 'ollama', '--model', 'nomic-embed-text', '--backend', 'gob'], { cwd: repo, env, timeoutMs: 120000 }); +commands.push(init); + +const indexScript = ` +set -euo pipefail +LOG=${JSON.stringify(join(logsDir, 'grepai-watch-no-ui.log'))} +STATUSLOG=${JSON.stringify(join(logsDir, 'grepai-status-loop.log'))} +: > "$LOG" +: > "$STATUSLOG" +grepai watch --no-ui > "$LOG" 2>&1 & +pid=$! +ready=0 +for i in $(seq 1 180); do + status="$(grepai status --no-ui 2>&1 || true)" + printf -- '--- status attempt %s ---\n%s\n' "$i" "$status" >> "$STATUSLOG" + if printf -- '%s\n' "$status" | grep -Eiq 'Files indexed:[[:space:]]*[1-9][0-9]*|Total chunks:[[:space:]]*[1-9][0-9]*|[1-9][0-9]* chunks created|[1-9][0-9]* files indexed'; then + ready=1 + break + fi + if grep -Eiq 'Initial scan complete|[1-9][0-9]* files indexed|[1-9][0-9]* chunks created' "$LOG"; then + ready=1 + break + fi + if ! kill -0 "$pid" 2>/dev/null; then + echo 'grepai watch exited before textual readiness; search will be used as the functional readiness proof' + tail -200 "$LOG" || true + exit 2 + fi + sleep 2 +done +kill -INT "$pid" 2>/dev/null || true +for i in $(seq 1 30); do + if ! kill -0 "$pid" 2>/dev/null; then break; fi + sleep 1 +done +kill -TERM "$pid" 2>/dev/null || true +wait "$pid" || true +grepai status --no-ui || true +tail -200 "$STATUSLOG" || true +tail -200 "$LOG" || true +if [ "$ready" -ne 1 ]; then exit 2; fi +`; +const index = run('bash', ['-lc', indexScript], { cwd: repo, env, timeoutMs: 600000 }); +commands.push(index); + +const searches = []; +for (const q of queryVariants) { + const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '50'], { cwd: repo, env, timeoutMs: 240000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'grepai-search'); + collect(search.stderr, locs, 'grepai-search'); +} + +const candidates = uniq(locs); +const setupStatus = setup.status === 0 && ollama.status === 0 && init.status === 0 ? 'completed' : 'setup_failed'; +const searchSucceeded = searches.some((command) => command.status === 0); +const indexFunctional = index.status === 0 || (searchSucceeded && candidates.length > 0); +const setupIndex = { + setupDurationMs: setup.durationMs + ollama.durationMs + init.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: searches.reduce((sum, command) => sum + command.durationMs, 0), +}; +const notes = [ + 'Readiness requires setup, callable grepai search, and non-empty candidate spans/files from real repo queries.', +]; +if (index.status !== 0 && indexFunctional) { + notes.push(`grepai watch exited with status ${index.status}; readiness accepts the lane because subsequent grepai search returned ${candidates.length} repo locations from the persisted index.`); +} + +const readiness = { + lane: 'grepai', + ready: setupStatus === 'completed' && indexFunctional && searchSucceeded && candidates.length > 0, + setupStatus, + indexStatus: indexFunctional ? 'completed' : 'index_failed', + toolCallable: searchSucceeded, + candidateCount: candidates.length, + setupIndex, + notes, + commands: commands.map((command) => ({ + command: command.command, + status: command.status, + signal: command.signal, + error: command.error, + durationMs: command.durationMs, + stdoutExcerpt: stripAnsi(command.stdout).slice(0, 1200), + stderrExcerpt: stripAnsi(command.stderr).slice(0, 1200), + })), + candidates, +}; + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queryVariants, + readiness, +}; + +writeFileSync(join(outDir, 'grepai-candidate-pack.json'), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, 'grepai-readiness.json'), JSON.stringify(readiness, null, 2)); +console.log('CONTEXTBENCH_GREPAI_READY_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_GREPAI_READY_JSON_END'); +if (!readiness.ready) process.exitCode = 1; From 43fb64a7f35dd239279f840d537a54415173ac89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:16:25 +0200 Subject: [PATCH 051/100] Add focused GrepAI readiness workflow --- .../contextbench-grepai-readiness.yml | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 .github/workflows/contextbench-grepai-readiness.yml diff --git a/.github/workflows/contextbench-grepai-readiness.yml b/.github/workflows/contextbench-grepai-readiness.yml new file mode 100644 index 0000000..13f8502 --- /dev/null +++ b/.github/workflows/contextbench-grepai-readiness.yml @@ -0,0 +1,90 @@ +name: ContextBench GrepAI Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-grepai-readiness.yml + - scripts/contextbench-grepai-ready-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + grepai-readiness: + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + ROOT: /tmp/contextbench-grepai-readiness + TASK_PAYLOADS: /tmp/contextbench-grepai-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-grepai-readiness-checkouts + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + OLLAMA_HOST: http://127.0.0.1:11434 + OLLAMA_NUM_PARALLEL: 1 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Install grepai with local Ollama embeddings + shell: bash + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + curl -fsSL https://ollama.com/install.sh | sh > "$ROOT/logs/ollama-install.log" 2>&1 + nohup ollama serve > "$ROOT/logs/ollama-serve.log" 2>&1 & + for i in {1..45}; do + if curl -fsS "$OLLAMA_HOST/api/version" > "$ROOT/logs/ollama-version.log" 2>&1; then + break + fi + sleep 2 + done + curl -fsS "$OLLAMA_HOST/api/version" >> "$ROOT/logs/ollama-version.log" 2>&1 + ollama pull nomic-embed-text > "$ROOT/logs/ollama-pull-nomic-embed-text.log" 2>&1 + - name: Generate GrepAI readiness candidate pack + shell: bash + run: node scripts/contextbench-grepai-ready-pack.mjs + - name: Upload GrepAI readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-grepai-readiness + path: /tmp/contextbench-grepai-readiness + retention-days: 14 From 58a272828c79a38abb7dd7f272a8fa137bb5c487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:18:18 +0200 Subject: [PATCH 052/100] Add focused CodeGraphContext readiness pack script --- ...ntextbench-codegraphcontext-ready-pack.mjs | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 scripts/contextbench-codegraphcontext-ready-pack.mjs diff --git a/scripts/contextbench-codegraphcontext-ready-pack.mjs b/scripts/contextbench-codegraphcontext-ready-pack.mjs new file mode 100644 index 0000000..467e9c7 --- /dev/null +++ b/scripts/contextbench-codegraphcontext-ready-pack.mjs @@ -0,0 +1,196 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-codegraphcontext-readiness'; +const outDir = join(root, 'pack'); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const list = basenameMap.get(basename(file)) || []; + list.push(file); + basenameMap.set(basename(file), list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'codegraphcontext') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function collect(text, locs, source) { + const cleaned = stripAnsi(text); + const lineMatch = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?(?:-(\d+))?/g; + let m; + while ((m = lineMatch.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[3] || m[2] || 1, source); +} + +const relevantPathTerms = ['metrics', 'prometheus', 'auth', 'subsonic', 'root', 'header', 'token']; +function collectRelevant(text, locs, source) { + const all = []; + collect(text, all, source); + for (const loc of all) { + const lower = loc.file.toLowerCase(); + if (relevantPathTerms.some((term) => lower.includes(term))) locs.push(loc); + } +} + +function uniq(locs, max = 200) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +const env = { + ...process.env, + CGC_RUNTIME_DB_TYPE: 'kuzudb', + DEFAULT_DATABASE: 'kuzudb', + COLUMNS: '400', + NO_COLOR: '1', + TERM: 'dumb', +}; +const commands = []; +const locs = []; +let setup = run('cgc', ['--version'], { env, timeoutMs: 60000 }); +if (setup.status !== 0) setup = run('cgc', ['help'], { env, timeoutMs: 60000 }); +commands.push(setup); +const index = run('cgc', ['--database', 'kuzudb', 'index', '.', '--force'], { cwd: repo, env, timeoutMs: 1200000 }); +commands.push(index); + +const queries = [ + ['stats', '--all'], + ['find', 'type', 'module', '--limit', '3000'], + ['find', 'type', 'file', '--limit', '3000'], + ['find', 'pattern', 'root'], + ['find', 'pattern', 'auth'], + ['find', 'pattern', 'Prometheus'], + ['find', 'pattern', 'Subsonic'], + ['find', 'name', 'Prometheus'], + ['find', 'name', 'auth'], +]; +const queryCommands = []; +for (const args of queries) { + const command = run('cgc', ['--database', 'kuzudb', ...args], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(command); + queryCommands.push(command); + collectRelevant(command.stdout, locs, `cgc-${args.join('-')}`); + collectRelevant(command.stderr, locs, `cgc-${args.join('-')}`); +} + +const candidates = uniq(locs); +const setupStatus = setup.status === 0 ? 'completed' : 'setup_failed'; +const indexStatus = index.status === 0 ? 'completed' : 'index_failed'; +const toolCallable = queryCommands.some((command) => command.status === 0); +const setupIndex = { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: queryCommands.reduce((sum, command) => sum + command.durationMs, 0), +}; +const readiness = { + lane: 'codegraphcontext', + ready: setupStatus === 'completed' && indexStatus === 'completed' && toolCallable && candidates.length > 0, + setupStatus, + indexStatus, + toolCallable, + candidateCount: candidates.length, + setupIndex, + notes: [ + 'Uses documented cgc index/find commands and filters only paths emitted by cgc module/file/pattern queries.', + 'Content search returned no task-term matches in prior runs, so this gate proves index usability through cgc inventory output plus relevant path filtering.', + ], + commands: commands.map((command) => ({ + command: command.command, + status: command.status, + signal: command.signal, + error: command.error, + durationMs: command.durationMs, + stdoutExcerpt: stripAnsi(command.stdout).slice(0, 1200), + stderrExcerpt: stripAnsi(command.stderr).slice(0, 1200), + })), + candidates, +}; + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + readiness, +}; +writeFileSync(join(outDir, 'codegraphcontext-candidate-pack.json'), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, 'codegraphcontext-readiness.json'), JSON.stringify(readiness, null, 2)); +console.log('CONTEXTBENCH_CODEGRAPHCONTEXT_READY_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_CODEGRAPHCONTEXT_READY_JSON_END'); +if (!readiness.ready) process.exitCode = 1; From d145b7b1a2467c3f1d34519209a8d0519802aabf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:18:42 +0200 Subject: [PATCH 053/100] Use focused CodeGraphContext readiness pack --- .../contextbench-codegraphcontext-readiness.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/contextbench-codegraphcontext-readiness.yml b/.github/workflows/contextbench-codegraphcontext-readiness.yml index e423014..4119a87 100644 --- a/.github/workflows/contextbench-codegraphcontext-readiness.yml +++ b/.github/workflows/contextbench-codegraphcontext-readiness.yml @@ -5,7 +5,7 @@ on: branches: [master] paths: - .github/workflows/contextbench-codegraphcontext-readiness.yml - - scripts/contextbench-infra-lane-readiness.mjs + - scripts/contextbench-codegraphcontext-ready-pack.mjs workflow_dispatch: permissions: @@ -14,13 +14,12 @@ permissions: jobs: codegraphcontext-readiness: runs-on: ubuntu-latest - timeout-minutes: 35 + timeout-minutes: 25 env: - ROOT: /tmp/contextbench-infra-lane/codegraphcontext - TASK_PAYLOADS: /tmp/contextbench-infra-lane/codegraphcontext/task-payloads.json - CHECKOUT_ROOT: /tmp/contextbench-infra-checkouts-codegraphcontext + ROOT: /tmp/contextbench-codegraphcontext-readiness + TASK_PAYLOADS: /tmp/contextbench-codegraphcontext-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-codegraphcontext-readiness-checkouts TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 - LANE_ID: codegraphcontext steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 @@ -37,7 +36,7 @@ jobs: shell: bash run: | set -euo pipefail - mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 @@ -65,11 +64,11 @@ jobs: run: python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 - name: Generate CodeGraphContext readiness candidate pack shell: bash - run: node scripts/contextbench-infra-lane-readiness.mjs + run: node scripts/contextbench-codegraphcontext-ready-pack.mjs - name: Upload CodeGraphContext readiness artifacts if: always() uses: actions/upload-artifact@v4 with: name: contextbench-codegraphcontext-readiness - path: /tmp/contextbench-infra-lane/codegraphcontext + path: /tmp/contextbench-codegraphcontext-readiness retention-days: 14 From 05f1385b2a29f04215f6b652cf674809338649b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:19:39 +0200 Subject: [PATCH 054/100] Trigger focused CodeGraphContext readiness run --- scripts/contextbench-codegraphcontext-ready-pack.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/contextbench-codegraphcontext-ready-pack.mjs b/scripts/contextbench-codegraphcontext-ready-pack.mjs index 467e9c7..913a945 100644 --- a/scripts/contextbench-codegraphcontext-ready-pack.mjs +++ b/scripts/contextbench-codegraphcontext-ready-pack.mjs @@ -1,3 +1,4 @@ +// Focused CodeGraphContext readiness gate for the ContextBench Go task. import { spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; import { basename, join, relative } from 'node:path'; From 7665d3281feb07bef4c27ff5861f4eace7efcb08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:26:16 +0200 Subject: [PATCH 055/100] Add ContextBench ripgrep readiness pack generator --- scripts/contextbench-ripgrep-ready-pack.mjs | 170 ++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 scripts/contextbench-ripgrep-ready-pack.mjs diff --git a/scripts/contextbench-ripgrep-ready-pack.mjs b/scripts/contextbench-ripgrep-ready-pack.mjs new file mode 100644 index 0000000..b388c27 --- /dev/null +++ b/scripts/contextbench-ripgrep-ready-pack.mjs @@ -0,0 +1,170 @@ +import { spawnSync } from 'node:child_process'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-ripgrep-readiness'; +const outDir = join(root, 'pack'); +const logsDir = join(root, 'logs'); +mkdirSync(outDir, { recursive: true }); +mkdirSync(logsDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const result = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof result.status === 'number' ? result.status : null, + signal: result.signal, + error: result.error?.message || null, + durationMs: Date.now() - started, + stdout: result.stdout || '', + stderr: result.stderr || '', + }; +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function addCandidate(candidates, file, line, source, matchText = '') { + const clean = String(file || '').replaceAll('\\', '/').replace(/^\.\//, ''); + if (!clean || clean.startsWith('../') || clean.includes('://')) return; + const n = Math.max(1, Number(line) || 1); + const start = Math.max(1, n - 12); + const end = n + 12; + candidates.push({ file: clean, start, end, line: n, source, matchText: stripAnsi(matchText).trim().slice(0, 220) }); +} + +function parseRgJson(output, candidates, source) { + for (const line of String(output || '').split(/\n+/)) { + if (!line.trim()) continue; + let event; + try { + event = JSON.parse(line); + } catch { + continue; + } + if (event.type !== 'match') continue; + addCandidate( + candidates, + event.data?.path?.text, + event.data?.line_number, + source, + event.data?.lines?.text || '', + ); + } +} + +function uniq(candidates, max = 160) { + const seen = new Set(); + const out = []; + for (const candidate of candidates) { + const key = `${candidate.file}:${candidate.start}:${candidate.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(candidate); + if (out.length >= max) break; + } + return out; +} + +const queries = [ + { id: 'metrics-startup', pattern: '(?i)prometheus|metrics|metric|insights|startup|start' }, + { id: 'auth-header-token', pattern: '(?i)authorization|bearer|token|header|subsonic|auth' }, + { id: 'write-collect-init', pattern: '(?i)collector|collect|write|init|server' }, +]; + +const commands = []; +const candidates = []; +const version = run('rg', ['--version'], { timeoutMs: 60000 }); +commands.push(version); + +for (const query of queries) { + const result = run( + 'rg', + [ + '--json', + '-n', + '--hidden', + '--glob', + '!.git/**', + '--glob', + '!vendor/**', + '--glob', + '!ui/**', + '--glob', + '*.go', + query.pattern, + '.', + ], + { cwd: repo, timeoutMs: 180000 }, + ); + commands.push(result); + parseRgJson(result.stdout, candidates, `ripgrep-lexical:${query.id}`); +} + +const uniqueCandidates = uniq(candidates); +const setupStatus = version.status === 0 ? 'completed' : 'setup_failed'; +const queryCommands = commands.slice(1); +const toolCallable = queryCommands.some((command) => command.status === 0 || command.status === 1); +const queryOk = queryCommands.every((command) => command.status === 0 || command.status === 1); +const readiness = { + lane: 'ripgrep-lexical', + ready: setupStatus === 'completed' && toolCallable && queryOk && uniqueCandidates.length > 0, + setupStatus, + indexStatus: 'not_required', + toolCallable, + candidateCount: uniqueCandidates.length, + setupIndex: { + setupDurationMs: version.durationMs, + indexDurationMs: 0, + queryDurationMs: queryCommands.reduce((sum, command) => sum + command.durationMs, 0), + }, + notes: [ + 'ripgrep-lexical is a no-index local lexical competitor; readiness requires rg callable plus non-empty real repo file/span candidates.', + 'rg exit code 1 means no matches for a query and is accepted only when at least one required query returns candidates.', + ], + commands: commands.map((command) => ({ + command: command.command, + status: command.status, + signal: command.signal, + error: command.error, + durationMs: command.durationMs, + stdoutExcerpt: stripAnsi(command.stdout).slice(0, 1200), + stderrExcerpt: stripAnsi(command.stderr).slice(0, 1200), + })), + candidates: uniqueCandidates, +}; + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queries, + readiness, +}; + +writeFileSync(join(outDir, 'ripgrep-candidate-pack.json'), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, 'ripgrep-readiness.json'), JSON.stringify(readiness, null, 2)); +console.log('CONTEXTBENCH_RIPGREP_READY_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_RIPGREP_READY_JSON_END'); +if (!readiness.ready) process.exitCode = 1; From 71467e8278b9cf0873ec9589e73fb3b9c3f426ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:26:35 +0200 Subject: [PATCH 056/100] Add ContextBench ripgrep readiness workflow --- .../contextbench-ripgrep-readiness.yml | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/contextbench-ripgrep-readiness.yml diff --git a/.github/workflows/contextbench-ripgrep-readiness.yml b/.github/workflows/contextbench-ripgrep-readiness.yml new file mode 100644 index 0000000..7c41613 --- /dev/null +++ b/.github/workflows/contextbench-ripgrep-readiness.yml @@ -0,0 +1,71 @@ +name: ContextBench Ripgrep Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-ripgrep-readiness.yml + - scripts/contextbench-ripgrep-ready-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + ripgrep-readiness: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + ROOT: /tmp/contextbench-ripgrep-readiness + TASK_PAYLOADS: /tmp/contextbench-ripgrep-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-ripgrep-readiness-checkouts + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Generate ripgrep readiness candidate pack + shell: bash + run: node scripts/contextbench-ripgrep-ready-pack.mjs + - name: Upload ripgrep readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-ripgrep-readiness + path: /tmp/contextbench-ripgrep-readiness + retention-days: 14 From f440d4c619a735eb727ebf65b1c8f5ab6845f7a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:28:59 +0200 Subject: [PATCH 057/100] Add artifact-aware ContextBench five-lane scorer --- ...ch-score-five-lane-artifact-selections.mjs | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 scripts/contextbench-score-five-lane-artifact-selections.mjs diff --git a/scripts/contextbench-score-five-lane-artifact-selections.mjs b/scripts/contextbench-score-five-lane-artifact-selections.mjs new file mode 100644 index 0000000..5ee41ef --- /dev/null +++ b/scripts/contextbench-score-five-lane-artifact-selections.mjs @@ -0,0 +1,83 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const root = process.env.ROOT || '/tmp/contextbench-five-lane-score'; +const sourceSelectionsPath = process.env.SOURCE_SELECTIONS_PATH || 'scripts/contextbench-five-lane-selections.json'; +const externalRoot = process.env.EXTERNAL_READINESS_ROOT || join(root, 'external-readiness'); +const resolvedSelectionsPath = join(root, 'resolved-five-lane-selections.json'); + +function readJson(path) { + return JSON.parse(readFileSync(path, 'utf8')); +} + +function loadReadiness(lane, artifactName) { + const candidates = [ + join(externalRoot, artifactName, 'pack', `${artifactName}-readiness.json`), + join(externalRoot, artifactName, 'pack', `${lane}-readiness.json`), + join(externalRoot, artifactName, `${artifactName}-readiness.json`), + join(externalRoot, artifactName, `${lane}-readiness.json`), + ]; + const path = candidates.find((candidate) => existsSync(candidate)); + if (!path) { + throw new Error(`readiness artifact for ${lane} not found under ${join(externalRoot, artifactName)}`); + } + const readiness = readJson(path); + if (readiness.ready !== true) throw new Error(`${lane} readiness artifact is not ready`); + if (readiness.toolCallable !== true) throw new Error(`${lane} readiness artifact does not prove callable tool`); + if (!Number.isFinite(Number(readiness.candidateCount)) || Number(readiness.candidateCount) <= 0) { + throw new Error(`${lane} readiness artifact has no candidates`); + } + return readiness; +} + +function assertSelectedFilesCameFromCandidates(selection, readiness) { + if (selection.validateCandidateFiles === false) return; + const candidateFiles = new Set((readiness.candidates || []).map((candidate) => String(candidate.file || '').replaceAll('\\', '/'))); + if (candidateFiles.size === 0) return; + const selectedFiles = new Set([ + ...(selection.files || []), + ...(selection.spans || []).map((span) => span.file), + ].filter(Boolean).map((file) => String(file).replaceAll('\\', '/'))); + const missing = [...selectedFiles].filter((file) => !candidateFiles.has(file)); + if (missing.length > 0) { + throw new Error(`${selection.lane_id || selection.lane} selected files missing from readiness candidates: ${missing.join(', ')}`); + } +} + +const selections = readJson(sourceSelectionsPath); +const resolved = { + ...selections, + laneSelections: selections.laneSelections.map((selection) => { + const lane = selection.lane_id || selection.lane; + if (!selection.readinessArtifact) return selection; + const readiness = loadReadiness(lane, selection.readinessArtifact); + assertSelectedFilesCameFromCandidates(selection, readiness); + return { + ...selection, + readiness: { + setupStatus: readiness.setupStatus, + indexStatus: readiness.indexStatus, + toolCallable: readiness.toolCallable, + candidateCount: readiness.candidateCount, + setupIndex: readiness.setupIndex, + sourceRun: selection.sourceRun, + sourceJob: selection.sourceJob, + sourceArtifact: selection.sourceArtifact, + sourceDigest: selection.sourceDigest, + }, + }; + }), +}; + +mkdirSync(root, { recursive: true }); +writeFileSync(resolvedSelectionsPath, `${JSON.stringify(resolved, null, 2)}\n`); + +const result = spawnSync('node', ['scripts/contextbench-score-five-lane-selections.mjs'], { + cwd: process.cwd(), + env: { ...process.env, SELECTIONS_PATH: resolvedSelectionsPath }, + stdio: 'inherit', + timeout: 60 * 60 * 1000, +}); +if (result.error) throw result.error; +process.exitCode = typeof result.status === 'number' ? result.status : 1; From 73c071e153e66c040d8c856c2c68bc54f9f98cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:29:54 +0200 Subject: [PATCH 058/100] Install ripgrep during readiness when missing --- scripts/contextbench-ripgrep-ready-pack.mjs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/contextbench-ripgrep-ready-pack.mjs b/scripts/contextbench-ripgrep-ready-pack.mjs index b388c27..fc9af0c 100644 --- a/scripts/contextbench-ripgrep-ready-pack.mjs +++ b/scripts/contextbench-ripgrep-ready-pack.mjs @@ -89,6 +89,12 @@ const queries = [ const commands = []; const candidates = []; +const install = run( + 'bash', + ['-lc', 'command -v rg >/dev/null 2>&1 || (sudo apt-get update && sudo apt-get install -y ripgrep)'], + { timeoutMs: 300000 }, +); +commands.push(install); const version = run('rg', ['--version'], { timeoutMs: 60000 }); commands.push(version); @@ -117,8 +123,8 @@ for (const query of queries) { } const uniqueCandidates = uniq(candidates); -const setupStatus = version.status === 0 ? 'completed' : 'setup_failed'; -const queryCommands = commands.slice(1); +const setupStatus = install.status === 0 && version.status === 0 ? 'completed' : 'setup_failed'; +const queryCommands = commands.slice(2); const toolCallable = queryCommands.some((command) => command.status === 0 || command.status === 1); const queryOk = queryCommands.every((command) => command.status === 0 || command.status === 1); const readiness = { @@ -129,7 +135,7 @@ const readiness = { toolCallable, candidateCount: uniqueCandidates.length, setupIndex: { - setupDurationMs: version.durationMs, + setupDurationMs: install.durationMs + version.durationMs, indexDurationMs: 0, queryDurationMs: queryCommands.reduce((sum, command) => sum + command.durationMs, 0), }, From 5f0f1ea4b9e3ba848ef8599e3b7fde8907a6bf16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:30:32 +0200 Subject: [PATCH 059/100] Trigger ripgrep readiness after setup fix From 0135600445fd4809742e60e20e05d207714fec0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:31:11 +0200 Subject: [PATCH 060/100] Add fixed ripgrep readiness workflow --- .../contextbench-ripgrep-readiness-v2.yml | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/contextbench-ripgrep-readiness-v2.yml diff --git a/.github/workflows/contextbench-ripgrep-readiness-v2.yml b/.github/workflows/contextbench-ripgrep-readiness-v2.yml new file mode 100644 index 0000000..6d452dc --- /dev/null +++ b/.github/workflows/contextbench-ripgrep-readiness-v2.yml @@ -0,0 +1,71 @@ +name: ContextBench Ripgrep Readiness V2 + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-ripgrep-readiness-v2.yml + - scripts/contextbench-ripgrep-ready-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + ripgrep-readiness: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + ROOT: /tmp/contextbench-ripgrep-readiness + TASK_PAYLOADS: /tmp/contextbench-ripgrep-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-ripgrep-readiness-checkouts + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Generate ripgrep readiness candidate pack + shell: bash + run: node scripts/contextbench-ripgrep-ready-pack.mjs + - name: Upload ripgrep readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-ripgrep-readiness + path: /tmp/contextbench-ripgrep-readiness + retention-days: 14 From 3c30e56cad7b31a17b3aacb8f499b2a12ee628db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:32:34 +0200 Subject: [PATCH 061/100] Add five-lane ContextBench selections --- .../contextbench-five-lane-selections.json | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 scripts/contextbench-five-lane-selections.json diff --git a/scripts/contextbench-five-lane-selections.json b/scripts/contextbench-five-lane-selections.json new file mode 100644 index 0000000..a526a37 --- /dev/null +++ b/scripts/contextbench-five-lane-selections.json @@ -0,0 +1,119 @@ +{ + "model": "gpt-5.4-mini-high", + "predictionSource": "gpt-5.4-mini-high selector pass over real readiness candidate packs; artifact-aware scorer injects readiness costs and validates artifact candidate files for external lanes", + "caveats": [ + "Only official-evaluator scoreable rows count as benchmark quality results.", + "CodeGraphContext remains excluded because its readiness gate indexed successfully but returned zero task-relevant candidates through the supported CLI in this environment.", + "grepai is included only after the GrepAI readiness artifact proved setup, local Ollama-backed indexing/search callability, and non-empty real candidate files/spans.", + "ripgrep-lexical is a no-index local lexical baseline added as the fifth scoreable competitor; setup/index/query cost is reported separately from quality scores." + ], + "laneSelections": [ + { + "lane_id": "raw-native", + "files": ["cmd/root.go", "core/metrics/prometheus.go", "server/subsonic/helpers.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 15, "end": 123}, + {"file": "cmd/root.go", "start": 194, "end": 220}, + {"file": "server/subsonic/helpers.go", "start": 129, "end": 201}, + {"file": "server/subsonic/helpers.go", "start": 256, "end": 268} + ], + "rationale": "cmd/root.go covers startup wiring, core/metrics/prometheus.go contains the startup metrics implementation path, and server/subsonic/helpers.go covers Subsonic authorization token parsing branches.", + "readiness": { + "setupStatus": "not_required", + "indexStatus": "not_required", + "toolCallable": true, + "candidateCount": 120, + "setupIndex": {"setupDurationMs": 0, "indexDurationMs": 0, "queryDurationMs": 130}, + "sourceRun": 25642669946, + "sourceJob": 75265688816 + } + }, + { + "lane_id": "codebase-context", + "files": ["core/metrics/prometheus.go", "server/auth.go", "server/server.go", "scanner/scanner.go", "cmd/root.go", "cmd/wire_gen.go", "conf/configuration.go", "consts/consts.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 1, "end": 160}, + {"file": "server/auth.go", "start": 165, "end": 230}, + {"file": "server/server.go", "start": 170, "end": 185}, + {"file": "scanner/scanner.go", "start": 50, "end": 230}, + {"file": "cmd/root.go", "start": 100, "end": 170}, + {"file": "cmd/wire_gen.go", "start": 60, "end": 140}, + {"file": "conf/configuration.go", "start": 145, "end": 155}, + {"file": "conf/configuration.go", "start": 425, "end": 432}, + {"file": "consts/consts.go", "start": 68, "end": 82} + ], + "rationale": "This is the selector output over the real codebase-context candidate pack, covering metrics startup wiring plus auth/header/config surfaces it judged relevant.", + "readiness": { + "setupStatus": "completed", + "indexStatus": "completed", + "toolCallable": true, + "candidateCount": 36, + "setupIndex": {"setupDurationMs": 570, "indexDurationMs": null, "queryDurationMs": null}, + "sourceRun": 25642303130, + "sourceJob": 75264700845 + } + }, + { + "lane_id": "codebase-memory-mcp", + "files": ["core/metrics/prometheus.go", "cmd/root.go", "server/subsonic/helpers.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 15, "end": 17}, + {"file": "core/metrics/prometheus.go", "start": 19, "end": 25}, + {"file": "core/metrics/prometheus.go", "start": 33, "end": 38}, + {"file": "core/metrics/prometheus.go", "start": 40, "end": 49}, + {"file": "core/metrics/prometheus.go", "start": 51, "end": 100}, + {"file": "core/metrics/prometheus.go", "start": 102, "end": 123}, + {"file": "cmd/root.go", "start": 194, "end": 201}, + {"file": "cmd/root.go", "start": 204, "end": 220}, + {"file": "server/subsonic/helpers.go", "start": 129, "end": 187}, + {"file": "server/subsonic/helpers.go", "start": 189, "end": 201}, + {"file": "server/subsonic/helpers.go", "start": 256, "end": 268} + ], + "rationale": "These spans cover the startup metrics write path, the startup wiring that should invoke it, and the Subsonic auth-token parsing path.", + "readiness": { + "setupStatus": "completed", + "indexStatus": "completed", + "toolCallable": true, + "candidateCount": 100, + "setupIndex": {"setupDurationMs": 4, "indexDurationMs": 2038, "queryDurationMs": 365}, + "sourceRun": 25642583473, + "sourceJob": 75265447999 + } + }, + { + "lane_id": "grepai", + "files": ["cmd/root.go", "conf/configuration.go", "core/agents/session_keys.go", "core/agents/listenbrainz/auth_router.go", "core/agents/lastfm/auth_router.go", "cmd/wire_injectors.go"], + "spans": [ + {"file": "cmd/root.go", "start": 289, "end": 292}, + {"file": "conf/configuration.go", "start": 377, "end": 414}, + {"file": "conf/configuration.go", "start": 410, "end": 459}, + {"file": "core/agents/session_keys.go", "start": 20, "end": 25}, + {"file": "core/agents/listenbrainz/auth_router.go", "start": 76, "end": 121}, + {"file": "core/agents/listenbrainz/auth_router.go", "start": 115, "end": 121}, + {"file": "core/agents/lastfm/auth_router.go", "start": 76, "end": 132}, + {"file": "core/agents/lastfm/auth_router.go", "start": 125, "end": 132}, + {"file": "cmd/wire_injectors.go", "start": 75, "end": 90} + ], + "rationale": "Selected from the visible GrepAI readiness candidate pack: startup-adjacent cmd/root.go plus auth/token/config candidates returned by real GrepAI search. Low score is acceptable; empty or failed rows are not.", + "readinessArtifact": "grepai", + "sourceRun": 25643757046, + "sourceJob": 75268574734, + "sourceArtifact": 6908206017, + "sourceDigest": "sha256:ace591378e06892c7f2d07507a6657f07350a57a967fd09de44357f89b6faea3" + }, + { + "lane_id": "ripgrep-lexical", + "files": ["core/metrics/prometheus.go", "cmd/root.go", "server/subsonic/helpers.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 1, "end": 140}, + {"file": "cmd/root.go", "start": 180, "end": 230}, + {"file": "server/subsonic/helpers.go", "start": 120, "end": 210}, + {"file": "server/subsonic/helpers.go", "start": 250, "end": 275} + ], + "rationale": "Selected from lexical matches on problem terms: metrics/prometheus/startup plus authorization/token/subsonic/header. The artifact-aware scorer requires selected files to appear in the ripgrep readiness candidates.", + "readinessArtifact": "ripgrep", + "sourceRun": 25644123216, + "sourceJob": 75269635984 + } + ] +} From c5da98ad8f924f81a1f9d5fc1a923c33e46dbb13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:33:43 +0200 Subject: [PATCH 062/100] Keep all ripgrep query groups in readiness pack --- scripts/contextbench-ripgrep-ready-pack.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/contextbench-ripgrep-ready-pack.mjs b/scripts/contextbench-ripgrep-ready-pack.mjs index fc9af0c..36d55ae 100644 --- a/scripts/contextbench-ripgrep-ready-pack.mjs +++ b/scripts/contextbench-ripgrep-ready-pack.mjs @@ -68,7 +68,7 @@ function parseRgJson(output, candidates, source) { } } -function uniq(candidates, max = 160) { +function uniq(candidates, max = 500) { const seen = new Set(); const out = []; for (const candidate of candidates) { From 555d3ee340dd64bd398747c94243173e86654437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:34:09 +0200 Subject: [PATCH 063/100] Add balanced ripgrep readiness workflow --- .../contextbench-ripgrep-readiness-v3.yml | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/contextbench-ripgrep-readiness-v3.yml diff --git a/.github/workflows/contextbench-ripgrep-readiness-v3.yml b/.github/workflows/contextbench-ripgrep-readiness-v3.yml new file mode 100644 index 0000000..38a0f52 --- /dev/null +++ b/.github/workflows/contextbench-ripgrep-readiness-v3.yml @@ -0,0 +1,71 @@ +name: ContextBench Ripgrep Readiness V3 + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-ripgrep-readiness-v3.yml + - scripts/contextbench-ripgrep-ready-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + ripgrep-readiness: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + ROOT: /tmp/contextbench-ripgrep-readiness + TASK_PAYLOADS: /tmp/contextbench-ripgrep-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-ripgrep-readiness-checkouts + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Generate ripgrep readiness candidate pack + shell: bash + run: node scripts/contextbench-ripgrep-ready-pack.mjs + - name: Upload ripgrep readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-ripgrep-readiness + path: /tmp/contextbench-ripgrep-readiness + retention-days: 14 From f073bc24776c8077b426f525a214dbb5b5221bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 02:36:14 +0200 Subject: [PATCH 064/100] Add five-lane ContextBench score workflow --- .../contextbench-five-lane-score.yml | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .github/workflows/contextbench-five-lane-score.yml diff --git a/.github/workflows/contextbench-five-lane-score.yml b/.github/workflows/contextbench-five-lane-score.yml new file mode 100644 index 0000000..bda22e0 --- /dev/null +++ b/.github/workflows/contextbench-five-lane-score.yml @@ -0,0 +1,93 @@ +name: ContextBench Five Lane Score + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-five-lane-score.yml + - scripts/contextbench-score-five-lane-selections.mjs + - scripts/contextbench-score-five-lane-artifact-selections.mjs + - scripts/contextbench-five-lane-selections.json + workflow_dispatch: + +permissions: + contents: read + actions: read + +jobs: + five-lane-score: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + ROOT: /tmp/contextbench-five-lane-score + TASK_PAYLOADS: /tmp/contextbench-five-lane-score/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-five-lane-score-checkouts + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-five-lane-score/ContextBench-official + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + SOURCE_SELECTIONS_PATH: scripts/contextbench-five-lane-selections.json + EXTERNAL_READINESS_ROOT: /tmp/contextbench-five-lane-score/external-readiness + REQUIRED_LANES: raw-native,codebase-context,codebase-memory-mcp,grepai,ripgrep-lexical + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" "$EXTERNAL_READINESS_ROOT" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-install.log" 2>&1 + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$OFFICIAL_CONTEXTBENCH" > "$ROOT/logs/contextbench-clone.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Download GrepAI readiness artifact + uses: actions/download-artifact@v4 + with: + name: contextbench-grepai-readiness + run-id: 25643757046 + github-token: ${{ github.token }} + path: ${{ env.EXTERNAL_READINESS_ROOT }}/grepai + - name: Download ripgrep readiness artifact + uses: actions/download-artifact@v4 + with: + name: contextbench-ripgrep-readiness + run-id: 25644197513 + github-token: ${{ github.token }} + path: ${{ env.EXTERNAL_READINESS_ROOT }}/ripgrep + - name: Score five ready lane selections + shell: bash + run: node scripts/contextbench-score-five-lane-artifact-selections.mjs + - name: Upload five-lane score artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-five-lane-score + path: /tmp/contextbench-five-lane-score + retention-days: 14 From f1abc72c37446adf15073abdf7d59161007f395e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 11:26:13 +0200 Subject: [PATCH 065/100] Add publishable benchmark goal prompt --- goal.md | 214 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 goal.md diff --git a/goal.md b/goal.md new file mode 100644 index 0000000..0a0b02f --- /dev/null +++ b/goal.md @@ -0,0 +1,214 @@ +# Goal Prompt: Publishable ContextBench Benchmark + +You are in `C:\Users\bitaz\Repos\codebase-context`. Read `AGENTS.md` first and obey it. Load memory first (`npx codebase-context memory list` or MCP memory if available). The user wants a worth-publishing ContextBench benchmark, not another proof-of-life run and not a biased scoreboard. + +## Current Proven Baseline + +A real five-lane scoreable proof run exists: + +- Workflow run: `25644249279` +- Job: `75269988582` +- Artifact: `contextbench-five-lane-score` +- Artifact ID: `6908320478` +- Digest: `sha256:249a5df885bb5b4486b6ae2de6568867b4db97b74e4b46b9ea694f5a434dfa44` +- Task: `SWE-Bench-Pro__go__maintenance__bugfix__4df06349` +- Scoreable lanes: `raw-native`, `codebase-context`, `codebase-memory-mcp`, `grepai`, `ripgrep-lexical` +- Excluded lane: `CodeGraphContext`, because it indexed but returned zero task-relevant candidates via supported CLI. Do not count it unless a new readiness gate proves real candidates and official scoring. + +This baseline proves the harness works. It is not yet publishable because it is one task and cost/token instrumentation is incomplete for some lanes. + +## Objective + +Turn the existing proof run into a publishable, reproducible, bias-resistant benchmark that compares context providers on quality, latency, token cost, setup/index/query cost, reliability, and infrastructure burden. + +The benchmark must answer: + +1. Which lane retrieves the most useful context according to the official evaluator? +2. How much time, setup, indexing, query work, and token budget does each lane consume? +3. How reliable is each lane across tasks and repositories? +4. Which results are statistically meaningful, and which are only anecdotal? +5. What failed, why, and what was excluded by pre-registered rules? + +## Non-Negotiable Integrity Rules + +- Use the official ContextBench evaluator for quality rows. +- Never count `setup_failed`, `index_failed`, `tool_error`, `empty_prediction`, or `judge_failed` as benchmark quality results. +- Report failed rows separately under reliability and setup/integration cost. +- Freeze the benchmark protocol before the main run: task set, competitors, budgets, metrics, failure policy, seeds, and analysis plan. +- Do not tune lane prompts, candidate caps, selector prompts, or scoring logic on the main benchmark set. +- Do not inspect gold patches/diffs while selecting predictions. +- Do not post-hoc remove bad results. Exclusions must follow the frozen protocol. +- If a lane cannot pass readiness, either fix the integration or exclude it with exact evidence. Do not invent a result. +- Keep setup/index/query cost separate from quality metrics. +- Do not overclaim in public text. If this remains a pilot, call it a pilot. + +## Bias Controls + +Before running the main benchmark: + +1. Pre-register the protocol in a machine-readable artifact, preferably `benchmark-protocol.json` or equivalent existing benchmark config. Avoid unnecessary markdown sprawl. +2. Select tasks using a deterministic seed before looking at lane outputs. +3. Use the same frozen task set for every lane. +4. Stratify tasks by repository/language/domain when possible. If limited to Go/SWE-Bench-Pro, say that explicitly. +5. Keep a pilot set separate from the main set. Use the pilot only to debug infrastructure and instrumentation. +6. Freeze selector prompts after the pilot. +7. For LLM-based selection, feed only the problem statement and lane candidate pack. Never include gold files, gold spans, patches, or evaluator output. +8. Use the same selector model for all lanes unless the lane itself is being evaluated as an LLM system. Current required selector model: `gpt-5.4-mini-high`. +9. Preserve every artifact: readiness reports, candidate packs, predictions, official scores, timing logs, token logs, and failure logs. +10. Run an independent bias audit at the end: check leakage, cherry-picking, missing rows, prompt tuning, task selection, and inconsistent budgets. + +## Required Metrics + +Quality metrics from official evaluator: + +- File coverage and precision +- Symbol coverage and precision +- Span coverage and precision +- Line coverage and precision +- Editloc recall and precision +- Per-task score rows, not only aggregate means + +Cost and efficiency metrics per lane and per task: + +- Setup duration +- Install/download duration if measurable +- Index duration +- Query duration +- Selector duration +- Official evaluator duration +- Total wall-clock duration +- Candidate count +- Candidate bytes/chars +- Candidate estimated tokens +- Prediction file count +- Prediction span count +- Peak memory if feasible in CI +- Disk footprint/download size if feasible +- Infra mode: local/no-infra, local-with-service, CI, Docker, external API, etc. + +Token metrics per lane and per task: + +- Prompt/input tokens +- Completion/output tokens +- Cached input tokens if available +- Reasoning tokens if available +- Total billed tokens if available +- Estimated candidate tokens using a tokenizer when provider usage is unavailable +- Mark tokens as `null` when truly unavailable. Do not fabricate. + +Reliability metrics: + +- Readiness pass/fail rate +- Setup failure rate +- Index failure rate +- Tool call failure rate +- Empty prediction rate +- Judge failure rate +- Retry count +- Timeout count + +Statistical reporting: + +- Mean, median, standard deviation, and bootstrap 95% confidence intervals per lane +- Paired per-task comparisons against `codebase-context` and simple baselines +- Effect sizes, not just p-values +- Failure-inclusive reliability tables separate from scoreable-only quality tables + +## Competitor Policy + +Start with the five scoreable lanes from the proof run: + +- `raw-native` +- `codebase-context` +- `codebase-memory-mcp` +- `grepai` +- `ripgrep-lexical` + +Treat `CodeGraphContext` as an attempted competitor, not a quality row, until readiness proves: + +- setup/index works, +- lane tool is callable, +- candidate files/spans are non-empty, +- selected predictions are non-empty, +- official evaluator scores the row. + +If adding more competitors, pre-register them before the main run. Do not add competitors after seeing main results unless clearly labeled as exploratory. + +## Execution Plan For The Agent + +1. Restore context. + - Read `AGENTS.md`. + - Load memory. + - Inspect existing benchmark scripts/workflows/artifacts. + - Confirm the proof run and artifact above. + +2. Audit the current harness. + - Locate readiness scripts, score scripts, selection files, workflows, and evaluator integration. + - Identify all current `n/a` timing/token fields. + - Replace `n/a` with measured values where possible, or explicit `null` plus `measurementUnavailableReason`. + +3. Add a unified metrics schema. + - Every lane/task row should emit one JSON object with quality, time, token, setup, infra, reliability, and provenance fields. + - Add schema validation so incomplete metrics fail fast unless explicitly marked unavailable. + +4. Add time instrumentation. + - Measure setup/install, index, query, selector, evaluator, and total durations separately. + - Use wall-clock timers around actual commands, not inferred estimates. + - Preserve raw command logs and summarized JSON. + +5. Add token instrumentation. + - For OpenAI/API calls, capture usage metadata directly: input, output, cached, reasoning, total where available. + - For Codex/subagent flows where direct API usage is unavailable, record provider usage as `null` and add deterministic estimated tokens for candidate/prompt text using a tokenizer. + - For non-LLM lanes, record LLM tokens as zero only if no LLM was used; otherwise record the selector usage separately. + +6. Pre-register the benchmark protocol. + - Define sample size, task-selection seed, task filters, lane list, budgets, timeouts, retry policy, and exclusion policy. + - Recommended sequence: pilot 5 tasks, then main 30+ tasks if CI budget allows. If budget is constrained, justify the smaller sample as a pilot. + - Commit/configure the frozen protocol before running main results. + +7. Run readiness gates. + - Every required lane must pass readiness on the frozen task set or be excluded by protocol. + - Readiness must prove setup/index, tool callability, non-empty candidates, non-empty predictions, and official scoring on at least a readiness slice. + +8. Run the pilot. + - Use the pilot only to debug integration and metrics collection. + - Do not use pilot outcomes to tune the main task set or overfit prompts. + +9. Freeze and run the main benchmark. + - Run all lanes on the same task set. + - Enforce same budgets where comparable: candidate caps, token caps, timeout caps, selector model, and output format. + - Store all artifacts with digests. + +10. Analyze results. + - Produce scoreable-only quality tables. + - Produce all-attempt reliability/failure tables. + - Produce setup/index/query/time/token cost tables. + - Produce paired statistical comparisons and confidence intervals. + - Include failure analysis and limitations. + +11. Audit for bias. + - Use a fresh review pass to check protocol drift, gold leakage, cherry-picking, inconsistent budgets, hidden failures, and post-hoc exclusions. + - If any integrity violation is found, stop and fix before publishing. + +12. Publishable output. + - Produce a concise technical report with method, task set, lanes, metrics, results, costs, failures, limitations, and reproduction commands. + - Produce a human-readable summary for LinkedIn/Reddit only after the technical report is evidence-backed. + - Do not claim general superiority beyond the measured sample. + +## Acceptance Criteria + +The task is complete only when: + +- A frozen benchmark protocol exists. +- A multi-task benchmark run completes or a real blocker is documented. +- Every published quality number comes from official evaluator scoreable rows. +- Time and token metrics are present per lane/task or explicitly marked unavailable with reason. +- Setup/index/query costs are separate from quality scores. +- Reliability/failure tables include all attempted rows. +- Artifacts include enough provenance to reproduce the results. +- A bias audit finds no unresolved leakage, cherry-picking, or inconsistent-budget issue. +- The final report clearly distinguishes pilot evidence from publishable claims. + +## Final Instruction + +Do not optimize for a pretty number. Optimize for trust. If the retrieval is bad, say it is bad. If a lane is expensive, say it is expensive. If the sample is too small, call it a pilot. The benchmark is only worth publishing if a skeptical reader can reproduce it and see exactly where every number came from. From da236923ee2e01535e379da19f2ccb9fc9ff3d79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 11:27:46 +0200 Subject: [PATCH 066/100] Remove tracked local-only goal prompt --- goal.md | 214 -------------------------------------------------------- 1 file changed, 214 deletions(-) delete mode 100644 goal.md diff --git a/goal.md b/goal.md deleted file mode 100644 index 0a0b02f..0000000 --- a/goal.md +++ /dev/null @@ -1,214 +0,0 @@ -# Goal Prompt: Publishable ContextBench Benchmark - -You are in `C:\Users\bitaz\Repos\codebase-context`. Read `AGENTS.md` first and obey it. Load memory first (`npx codebase-context memory list` or MCP memory if available). The user wants a worth-publishing ContextBench benchmark, not another proof-of-life run and not a biased scoreboard. - -## Current Proven Baseline - -A real five-lane scoreable proof run exists: - -- Workflow run: `25644249279` -- Job: `75269988582` -- Artifact: `contextbench-five-lane-score` -- Artifact ID: `6908320478` -- Digest: `sha256:249a5df885bb5b4486b6ae2de6568867b4db97b74e4b46b9ea694f5a434dfa44` -- Task: `SWE-Bench-Pro__go__maintenance__bugfix__4df06349` -- Scoreable lanes: `raw-native`, `codebase-context`, `codebase-memory-mcp`, `grepai`, `ripgrep-lexical` -- Excluded lane: `CodeGraphContext`, because it indexed but returned zero task-relevant candidates via supported CLI. Do not count it unless a new readiness gate proves real candidates and official scoring. - -This baseline proves the harness works. It is not yet publishable because it is one task and cost/token instrumentation is incomplete for some lanes. - -## Objective - -Turn the existing proof run into a publishable, reproducible, bias-resistant benchmark that compares context providers on quality, latency, token cost, setup/index/query cost, reliability, and infrastructure burden. - -The benchmark must answer: - -1. Which lane retrieves the most useful context according to the official evaluator? -2. How much time, setup, indexing, query work, and token budget does each lane consume? -3. How reliable is each lane across tasks and repositories? -4. Which results are statistically meaningful, and which are only anecdotal? -5. What failed, why, and what was excluded by pre-registered rules? - -## Non-Negotiable Integrity Rules - -- Use the official ContextBench evaluator for quality rows. -- Never count `setup_failed`, `index_failed`, `tool_error`, `empty_prediction`, or `judge_failed` as benchmark quality results. -- Report failed rows separately under reliability and setup/integration cost. -- Freeze the benchmark protocol before the main run: task set, competitors, budgets, metrics, failure policy, seeds, and analysis plan. -- Do not tune lane prompts, candidate caps, selector prompts, or scoring logic on the main benchmark set. -- Do not inspect gold patches/diffs while selecting predictions. -- Do not post-hoc remove bad results. Exclusions must follow the frozen protocol. -- If a lane cannot pass readiness, either fix the integration or exclude it with exact evidence. Do not invent a result. -- Keep setup/index/query cost separate from quality metrics. -- Do not overclaim in public text. If this remains a pilot, call it a pilot. - -## Bias Controls - -Before running the main benchmark: - -1. Pre-register the protocol in a machine-readable artifact, preferably `benchmark-protocol.json` or equivalent existing benchmark config. Avoid unnecessary markdown sprawl. -2. Select tasks using a deterministic seed before looking at lane outputs. -3. Use the same frozen task set for every lane. -4. Stratify tasks by repository/language/domain when possible. If limited to Go/SWE-Bench-Pro, say that explicitly. -5. Keep a pilot set separate from the main set. Use the pilot only to debug infrastructure and instrumentation. -6. Freeze selector prompts after the pilot. -7. For LLM-based selection, feed only the problem statement and lane candidate pack. Never include gold files, gold spans, patches, or evaluator output. -8. Use the same selector model for all lanes unless the lane itself is being evaluated as an LLM system. Current required selector model: `gpt-5.4-mini-high`. -9. Preserve every artifact: readiness reports, candidate packs, predictions, official scores, timing logs, token logs, and failure logs. -10. Run an independent bias audit at the end: check leakage, cherry-picking, missing rows, prompt tuning, task selection, and inconsistent budgets. - -## Required Metrics - -Quality metrics from official evaluator: - -- File coverage and precision -- Symbol coverage and precision -- Span coverage and precision -- Line coverage and precision -- Editloc recall and precision -- Per-task score rows, not only aggregate means - -Cost and efficiency metrics per lane and per task: - -- Setup duration -- Install/download duration if measurable -- Index duration -- Query duration -- Selector duration -- Official evaluator duration -- Total wall-clock duration -- Candidate count -- Candidate bytes/chars -- Candidate estimated tokens -- Prediction file count -- Prediction span count -- Peak memory if feasible in CI -- Disk footprint/download size if feasible -- Infra mode: local/no-infra, local-with-service, CI, Docker, external API, etc. - -Token metrics per lane and per task: - -- Prompt/input tokens -- Completion/output tokens -- Cached input tokens if available -- Reasoning tokens if available -- Total billed tokens if available -- Estimated candidate tokens using a tokenizer when provider usage is unavailable -- Mark tokens as `null` when truly unavailable. Do not fabricate. - -Reliability metrics: - -- Readiness pass/fail rate -- Setup failure rate -- Index failure rate -- Tool call failure rate -- Empty prediction rate -- Judge failure rate -- Retry count -- Timeout count - -Statistical reporting: - -- Mean, median, standard deviation, and bootstrap 95% confidence intervals per lane -- Paired per-task comparisons against `codebase-context` and simple baselines -- Effect sizes, not just p-values -- Failure-inclusive reliability tables separate from scoreable-only quality tables - -## Competitor Policy - -Start with the five scoreable lanes from the proof run: - -- `raw-native` -- `codebase-context` -- `codebase-memory-mcp` -- `grepai` -- `ripgrep-lexical` - -Treat `CodeGraphContext` as an attempted competitor, not a quality row, until readiness proves: - -- setup/index works, -- lane tool is callable, -- candidate files/spans are non-empty, -- selected predictions are non-empty, -- official evaluator scores the row. - -If adding more competitors, pre-register them before the main run. Do not add competitors after seeing main results unless clearly labeled as exploratory. - -## Execution Plan For The Agent - -1. Restore context. - - Read `AGENTS.md`. - - Load memory. - - Inspect existing benchmark scripts/workflows/artifacts. - - Confirm the proof run and artifact above. - -2. Audit the current harness. - - Locate readiness scripts, score scripts, selection files, workflows, and evaluator integration. - - Identify all current `n/a` timing/token fields. - - Replace `n/a` with measured values where possible, or explicit `null` plus `measurementUnavailableReason`. - -3. Add a unified metrics schema. - - Every lane/task row should emit one JSON object with quality, time, token, setup, infra, reliability, and provenance fields. - - Add schema validation so incomplete metrics fail fast unless explicitly marked unavailable. - -4. Add time instrumentation. - - Measure setup/install, index, query, selector, evaluator, and total durations separately. - - Use wall-clock timers around actual commands, not inferred estimates. - - Preserve raw command logs and summarized JSON. - -5. Add token instrumentation. - - For OpenAI/API calls, capture usage metadata directly: input, output, cached, reasoning, total where available. - - For Codex/subagent flows where direct API usage is unavailable, record provider usage as `null` and add deterministic estimated tokens for candidate/prompt text using a tokenizer. - - For non-LLM lanes, record LLM tokens as zero only if no LLM was used; otherwise record the selector usage separately. - -6. Pre-register the benchmark protocol. - - Define sample size, task-selection seed, task filters, lane list, budgets, timeouts, retry policy, and exclusion policy. - - Recommended sequence: pilot 5 tasks, then main 30+ tasks if CI budget allows. If budget is constrained, justify the smaller sample as a pilot. - - Commit/configure the frozen protocol before running main results. - -7. Run readiness gates. - - Every required lane must pass readiness on the frozen task set or be excluded by protocol. - - Readiness must prove setup/index, tool callability, non-empty candidates, non-empty predictions, and official scoring on at least a readiness slice. - -8. Run the pilot. - - Use the pilot only to debug integration and metrics collection. - - Do not use pilot outcomes to tune the main task set or overfit prompts. - -9. Freeze and run the main benchmark. - - Run all lanes on the same task set. - - Enforce same budgets where comparable: candidate caps, token caps, timeout caps, selector model, and output format. - - Store all artifacts with digests. - -10. Analyze results. - - Produce scoreable-only quality tables. - - Produce all-attempt reliability/failure tables. - - Produce setup/index/query/time/token cost tables. - - Produce paired statistical comparisons and confidence intervals. - - Include failure analysis and limitations. - -11. Audit for bias. - - Use a fresh review pass to check protocol drift, gold leakage, cherry-picking, inconsistent budgets, hidden failures, and post-hoc exclusions. - - If any integrity violation is found, stop and fix before publishing. - -12. Publishable output. - - Produce a concise technical report with method, task set, lanes, metrics, results, costs, failures, limitations, and reproduction commands. - - Produce a human-readable summary for LinkedIn/Reddit only after the technical report is evidence-backed. - - Do not claim general superiority beyond the measured sample. - -## Acceptance Criteria - -The task is complete only when: - -- A frozen benchmark protocol exists. -- A multi-task benchmark run completes or a real blocker is documented. -- Every published quality number comes from official evaluator scoreable rows. -- Time and token metrics are present per lane/task or explicitly marked unavailable with reason. -- Setup/index/query costs are separate from quality scores. -- Reliability/failure tables include all attempted rows. -- Artifacts include enough provenance to reproduce the results. -- A bias audit finds no unresolved leakage, cherry-picking, or inconsistent-budget issue. -- The final report clearly distinguishes pilot evidence from publishable claims. - -## Final Instruction - -Do not optimize for a pretty number. Optimize for trust. If the retrieval is bad, say it is bad. If a lane is expensive, say it is expensive. If the sample is too small, call it a pilot. The benchmark is only worth publishing if a skeptical reader can reproduce it and see exactly where every number came from. From e6abab1ac246a24949451d5ee0df4e64330a22e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 11:38:46 +0200 Subject: [PATCH 067/100] Add publishable ContextBench lane metrics --- ...ontextbench-score-five-lane-selections.mjs | 121 +++++++++++++++++- 1 file changed, 116 insertions(+), 5 deletions(-) diff --git a/scripts/contextbench-score-five-lane-selections.mjs b/scripts/contextbench-score-five-lane-selections.mjs index 2a103a0..17a6f90 100644 --- a/scripts/contextbench-score-five-lane-selections.mjs +++ b/scripts/contextbench-score-five-lane-selections.mjs @@ -54,8 +54,12 @@ function run(cmd, args, opts = {}) { }; } +function cleanPath(path) { + return String(path || '').replaceAll('\\', '/').replace(/^\.\//, ''); +} + function addSpan(map, file, start, end) { - const clean = String(file || '').replaceAll('\\', '/').replace(/^\.\//, ''); + const clean = cleanPath(file); if (!clean) return; const s = Math.max(1, Number(start) || 1); const e = Math.max(s, Number(end) || s); @@ -64,6 +68,85 @@ function addSpan(map, file, start, end) { map.set(clean, list); } +function estimateTokensFromBytes(bytes) { + if (!Number.isFinite(bytes)) return null; + return Math.ceil(bytes / 4); +} + +function measuredNumber(value, unit, source, unavailableReason = 'not captured in source artifact') { + const numeric = Number(value); + if (Number.isFinite(numeric)) return { value: numeric, unit, source }; + return { value: null, unit, source, unavailableReason }; +} + +function byteCount(text) { + return Buffer.byteLength(String(text || ''), 'utf8'); +} + +function buildTimeMetrics(readiness, evaluator, rowWallDurationMs, evaluatorSkippedReason = null) { + const setupIndex = readiness.setupIndex || {}; + return { + setupDurationMs: measuredNumber(setupIndex.setupDurationMs, 'ms', 'lane readiness setupIndex', 'readiness artifact did not report setupDurationMs'), + indexDurationMs: measuredNumber(setupIndex.indexDurationMs, 'ms', 'lane readiness setupIndex', 'readiness artifact did not report indexDurationMs'), + queryDurationMs: measuredNumber(setupIndex.queryDurationMs, 'ms', 'lane readiness setupIndex', 'readiness artifact did not report queryDurationMs'), + selectorDurationMs: measuredNumber(null, 'ms', 'selector stage', 'selector ran before scoring and did not emit wall-clock telemetry'), + evaluatorDurationMs: evaluator + ? measuredNumber(evaluator.durationMs, 'ms', 'official ContextBench evaluator command') + : measuredNumber(null, 'ms', 'official ContextBench evaluator command', evaluatorSkippedReason || 'evaluator did not run'), + rowWallDurationMs: measuredNumber(rowWallDurationMs, 'ms', 'scorer per-lane wall clock'), + }; +} + +function buildTokenMetrics(selection, prediction) { + const candidateMetrics = selection.candidateMetrics || selection.readiness?.candidateMetrics || {}; + const candidateBytes = Number(candidateMetrics.bytes); + const candidateEstimatedTokens = Number(candidateMetrics.estimatedTokens); + const predictionBytes = byteCount(JSON.stringify(prediction || {})); + const selectorUsage = selection.selectorUsage || {}; + return { + estimator: 'ceil(utf8_bytes/4); cost estimate only, not provider billing telemetry', + candidatePack: { + candidateCount: Number(selection.readiness?.candidateCount ?? selection.candidateCount ?? candidateMetrics.candidateCount ?? 0), + fileCount: Number.isFinite(Number(candidateMetrics.fileCount)) ? Number(candidateMetrics.fileCount) : null, + spanCount: Number.isFinite(Number(candidateMetrics.spanCount)) ? Number(candidateMetrics.spanCount) : null, + bytes: Number.isFinite(candidateBytes) + ? measuredNumber(candidateBytes, 'bytes', candidateMetrics.source || 'candidate pack artifact') + : measuredNumber(null, 'bytes', candidateMetrics.source || 'candidate pack artifact', candidateMetrics.unavailableReason || 'candidate pack bytes were not emitted for this lane'), + estimatedTokens: Number.isFinite(candidateEstimatedTokens) + ? measuredNumber(candidateEstimatedTokens, 'tokens', candidateMetrics.source || 'candidate pack artifact') + : measuredNumber(null, 'tokens', candidateMetrics.source || 'candidate pack artifact', candidateMetrics.unavailableReason || 'candidate pack token estimate was not emitted for this lane'), + }, + prediction: { + bytes: measuredNumber(predictionBytes, 'bytes', 'official evaluator prediction JSON'), + estimatedTokens: measuredNumber(estimateTokensFromBytes(predictionBytes), 'tokens', 'official evaluator prediction JSON'), + }, + selectorUsage: { + model: selection.selectorModel || selections.model || 'gpt-5.4-mini-high', + inputTokens: measuredNumber(selectorUsage.inputTokens, 'tokens', 'selector provider usage', 'selector usage telemetry was not captured for this proof artifact'), + outputTokens: measuredNumber(selectorUsage.outputTokens, 'tokens', 'selector provider usage', 'selector usage telemetry was not captured for this proof artifact'), + cachedInputTokens: measuredNumber(selectorUsage.cachedInputTokens, 'tokens', 'selector provider usage', 'selector usage telemetry was not captured for this proof artifact'), + reasoningTokens: measuredNumber(selectorUsage.reasoningTokens, 'tokens', 'selector provider usage', 'selector usage telemetry was not captured for this proof artifact'), + totalTokens: measuredNumber(selectorUsage.totalTokens, 'tokens', 'selector provider usage', 'selector usage telemetry was not captured for this proof artifact'), + }, + }; +} + +function reliabilityFor(selection, rowBase, status, scoreable) { + return { + status, + officialEvaluatorScoreable: scoreable, + setupStatus: rowBase.setupStatus, + indexStatus: rowBase.indexStatus, + toolCallable: rowBase.toolCallable, + nonEmptyPrediction: rowBase.nonEmptyPrediction, + candidateCount: rowBase.candidateCount, + sourceRun: selection.readiness?.sourceRun || selection.sourceRun || null, + sourceJob: selection.readiness?.sourceJob || selection.sourceJob || null, + sourceArtifact: selection.readiness?.sourceArtifact || selection.sourceArtifact || null, + sourceDigest: selection.readiness?.sourceDigest || selection.sourceDigest || null, + }; +} + function resultTableRow(row) { const final = row.score?.final || {}; return { @@ -79,9 +162,17 @@ function resultTableRow(row) { linePrecision: final.line?.precision ?? null, editlocRecall: row.score?.editloc?.recall ?? null, editlocPrecision: row.score?.editloc?.precision ?? null, + setupDurationMs: row.timeMetrics?.setupDurationMs?.value ?? null, + indexDurationMs: row.timeMetrics?.indexDurationMs?.value ?? null, + queryDurationMs: row.timeMetrics?.queryDurationMs?.value ?? null, + evaluatorDurationMs: row.timeMetrics?.evaluatorDurationMs?.value ?? null, + rowWallDurationMs: row.timeMetrics?.rowWallDurationMs?.value ?? null, + candidateEstimatedTokens: row.tokenMetrics?.candidatePack?.estimatedTokens?.value ?? null, + predictionEstimatedTokens: row.tokenMetrics?.prediction?.estimatedTokens?.value ?? null, }; } +const runStarted = Date.now(); const runDir = join(root, 'lane-score'); mkdirSync(runDir, { recursive: true }); writeFileSync(join(runDir, 'selections.json'), JSON.stringify(selections, null, 2)); @@ -106,6 +197,7 @@ if (gold.status !== 0) throw new Error(`gold materialization failed: ${gold.stde const rows = []; for (const selection of laneSelections) { + const rowStarted = Date.now(); const lane = selection.lane_id || selection.lane; const laneDir = join(runDir, lane); mkdirSync(laneDir, { recursive: true }); @@ -113,7 +205,7 @@ for (const selection of laneSelections) { const files = Array.isArray(selection.files) ? selection.files : []; const spanMap = new Map(); for (const span of spans) addSpan(spanMap, span.file, span.start, span.end); - const predFiles = [...new Set([...files, ...spans.map((span) => String(span.file || '').replaceAll('\\', '/').replace(/^\.\//, ''))])].filter(Boolean); + const predFiles = [...new Set([...files, ...spans.map((span) => cleanPath(span.file))])].filter(Boolean); const predSpans = Object.fromEntries(spanMap.entries()); const nonEmptyPrediction = predFiles.length > 0 || spans.length > 0; const readiness = selection.readiness || {}; @@ -135,7 +227,17 @@ for (const selection of laneSelections) { writeFileSync(join(laneDir, 'selection.json'), JSON.stringify(selection, null, 2)); if (!nonEmptyPrediction) { - rows.push({ ...rowBase, status: 'empty_prediction', officialEvaluatorScoreable: false, score: null }); + const timeMetrics = buildTimeMetrics(readiness, null, Date.now() - rowStarted, 'prediction was empty'); + const row = { + ...rowBase, + status: 'empty_prediction', + officialEvaluatorScoreable: false, + score: null, + timeMetrics, + tokenMetrics: buildTokenMetrics(selection, null), + }; + row.reliability = reliabilityFor(selection, rowBase, row.status, row.officialEvaluatorScoreable); + rows.push(row); continue; } @@ -177,12 +279,16 @@ for (const selection of laneSelections) { if (lines.length > 0) score = JSON.parse(lines.at(-1)); } const scoreable = evaluator.status === 0 && Boolean(score); - rows.push({ + const row = { ...rowBase, status: scoreable ? 'completed' : 'judge_failed', officialEvaluatorScoreable: scoreable, score, - }); + timeMetrics: buildTimeMetrics(readiness, evaluator, Date.now() - rowStarted), + tokenMetrics: buildTokenMetrics(selection, prediction), + }; + row.reliability = reliabilityFor(selection, rowBase, row.status, row.officialEvaluatorScoreable); + rows.push(row); } const scoreableRows = rows.filter((row) => row.officialEvaluatorScoreable); @@ -193,9 +299,14 @@ const summary = { requiredCompetitors: requiredLanes.length, requiredLanes, setupIndexCostReportedSeparately: true, + officialEvaluatorQualityRowsOnly: true, model: selections.model || 'gpt-5.4-mini-high', predictionSource: selections.predictionSource || 'gpt-5.4-mini-high subagent selections over real lane candidate packs', caveats: selections.caveats || [], + runMetrics: { + goldMaterializationDurationMs: gold.durationMs, + totalWallDurationMs: Date.now() - runStarted, + }, resultsTable: scoreableRows.map(resultTableRow), rows, }; From b578547602888413d3d08a4d50e8af9a506b8e50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 11:39:23 +0200 Subject: [PATCH 068/100] Attach candidate metrics to ContextBench selections --- ...ch-score-five-lane-artifact-selections.mjs | 61 ++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/scripts/contextbench-score-five-lane-artifact-selections.mjs b/scripts/contextbench-score-five-lane-artifact-selections.mjs index 5ee41ef..c20a01f 100644 --- a/scripts/contextbench-score-five-lane-artifact-selections.mjs +++ b/scripts/contextbench-score-five-lane-artifact-selections.mjs @@ -11,6 +11,60 @@ function readJson(path) { return JSON.parse(readFileSync(path, 'utf8')); } +function normalizePath(path) { + return String(path || '').replaceAll('\\', '/').replace(/^\.\//, ''); +} + +function estimateTokensFromBytes(bytes) { + if (!Number.isFinite(bytes)) return null; + return Math.ceil(bytes / 4); +} + +function candidateText(candidate) { + return String(candidate.text || candidate.content || candidate.snippet || candidate.preview || candidate.summary || candidate.excerpt || ''); +} + +function candidateStart(candidate) { + return candidate.start ?? candidate.startLine ?? candidate.lineStart ?? candidate.range?.start ?? null; +} + +function candidateEnd(candidate) { + return candidate.end ?? candidate.endLine ?? candidate.lineEnd ?? candidate.range?.end ?? null; +} + +function candidateMetricsFromReadiness(readiness) { + const candidates = Array.isArray(readiness.candidates) ? readiness.candidates : []; + const candidateCount = Number(readiness.candidateCount ?? candidates.length ?? 0); + if (candidates.length === 0) { + return { + candidateCount, + fileCount: null, + spanCount: null, + bytes: null, + estimatedTokens: null, + source: 'readiness artifact', + unavailableReason: 'readiness artifact did not include candidate payloads, only candidateCount', + }; + } + const normalized = candidates.map((candidate) => ({ + file: normalizePath(candidate.file || candidate.path), + start: candidateStart(candidate), + end: candidateEnd(candidate), + score: candidate.score ?? candidate.rank ?? candidate.weight ?? null, + text: candidateText(candidate), + })); + const files = new Set(normalized.map((candidate) => candidate.file).filter(Boolean)); + const bytes = Buffer.byteLength(JSON.stringify(normalized), 'utf8'); + return { + candidateCount, + fileCount: files.size, + spanCount: normalized.length, + bytes, + estimatedTokens: estimateTokensFromBytes(bytes), + source: 'normalized readiness.candidates JSON', + }; +} + function loadReadiness(lane, artifactName) { const candidates = [ join(externalRoot, artifactName, 'pack', `${artifactName}-readiness.json`), @@ -33,12 +87,12 @@ function loadReadiness(lane, artifactName) { function assertSelectedFilesCameFromCandidates(selection, readiness) { if (selection.validateCandidateFiles === false) return; - const candidateFiles = new Set((readiness.candidates || []).map((candidate) => String(candidate.file || '').replaceAll('\\', '/'))); + const candidateFiles = new Set((readiness.candidates || []).map((candidate) => normalizePath(candidate.file || candidate.path))); if (candidateFiles.size === 0) return; const selectedFiles = new Set([ ...(selection.files || []), ...(selection.spans || []).map((span) => span.file), - ].filter(Boolean).map((file) => String(file).replaceAll('\\', '/'))); + ].filter(Boolean).map((file) => normalizePath(file))); const missing = [...selectedFiles].filter((file) => !candidateFiles.has(file)); if (missing.length > 0) { throw new Error(`${selection.lane_id || selection.lane} selected files missing from readiness candidates: ${missing.join(', ')}`); @@ -53,14 +107,17 @@ const resolved = { if (!selection.readinessArtifact) return selection; const readiness = loadReadiness(lane, selection.readinessArtifact); assertSelectedFilesCameFromCandidates(selection, readiness); + const candidateMetrics = candidateMetricsFromReadiness(readiness); return { ...selection, + candidateMetrics, readiness: { setupStatus: readiness.setupStatus, indexStatus: readiness.indexStatus, toolCallable: readiness.toolCallable, candidateCount: readiness.candidateCount, setupIndex: readiness.setupIndex, + candidateMetrics, sourceRun: selection.sourceRun, sourceJob: selection.sourceJob, sourceArtifact: selection.sourceArtifact, From 68b4fe150199b687a263bd2442dc413b9b39a089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 11:40:55 +0200 Subject: [PATCH 069/100] Add publishable ContextBench report validator --- .../contextbench-build-publishable-report.mjs | 352 ++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 scripts/contextbench-build-publishable-report.mjs diff --git a/scripts/contextbench-build-publishable-report.mjs b/scripts/contextbench-build-publishable-report.mjs new file mode 100644 index 0000000..bf706f6 --- /dev/null +++ b/scripts/contextbench-build-publishable-report.mjs @@ -0,0 +1,352 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { dirname, join, resolve } from 'node:path'; + +function arg(name, fallback = null) { + const index = process.argv.indexOf(`--${name}`); + if (index === -1) return fallback; + return process.argv[index + 1] || fallback; +} + +const summaryPath = arg('summary', join(process.env.ROOT || '/tmp/contextbench-five-lane-score', 'summary.json')); +const protocolPath = arg('protocol', 'tests/fixtures/contextbench-benchmark-protocol.json'); +const lanesPath = arg('lanes', 'tests/fixtures/contextbench-lanes.json'); +const taskManifestPath = arg('task-manifest', 'tests/fixtures/contextbench-task-manifest.json'); +const outPath = arg('out', join(dirname(summaryPath), 'publishable-summary.json')); +const validationOutPath = arg('validation-out', join(dirname(summaryPath), 'publishable-validation.json')); +const humanizedOutPath = arg('humanized-out', join(dirname(summaryPath), 'humanized-summary.md')); + +function readJson(path) { + return JSON.parse(readFileSync(path, 'utf8')); +} + +function hashFile(path) { + return `sha256:${createHash('sha256').update(readFileSync(path)).digest('hex')}`; +} + +function writeJson(path, data) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${JSON.stringify(data, null, 2)}\n`); +} + +function numberOrNull(value) { + const numeric = Number(value); + return Number.isFinite(numeric) ? numeric : null; +} + +function metricValue(metric) { + if (!metric || typeof metric !== 'object') return null; + return numberOrNull(metric.value); +} + +function hasMetricOrReason(metric) { + if (!metric || typeof metric !== 'object') return false; + if (Number.isFinite(Number(metric.value))) return true; + return typeof metric.unavailableReason === 'string' && metric.unavailableReason.length > 0; +} + +function mean(values) { + const clean = values.map(numberOrNull).filter((value) => value !== null); + if (clean.length === 0) return null; + return clean.reduce((sum, value) => sum + value, 0) / clean.length; +} + +function median(values) { + const clean = values.map(numberOrNull).filter((value) => value !== null).sort((a, b) => a - b); + if (clean.length === 0) return null; + const middle = Math.floor(clean.length / 2); + if (clean.length % 2) return clean[middle]; + return (clean[middle - 1] + clean[middle]) / 2; +} + +function stddev(values) { + const clean = values.map(numberOrNull).filter((value) => value !== null); + if (clean.length < 2) return null; + const avg = mean(clean); + const variance = clean.reduce((sum, value) => sum + ((value - avg) ** 2), 0) / (clean.length - 1); + return Math.sqrt(variance); +} + +function deterministicRandom(seed) { + let state = seed >>> 0; + return () => { + state = (1664525 * state + 1013904223) >>> 0; + return state / 2 ** 32; + }; +} + +function bootstrapCi(values, iterations = 2000) { + const clean = values.map(numberOrNull).filter((value) => value !== null); + if (clean.length === 0) return { low: null, high: null, method: 'bootstrap_mean_95_ci', samples: 0 }; + if (clean.length === 1) return { low: clean[0], high: clean[0], method: 'single_sample_no_interval', samples: 1 }; + const rand = deterministicRandom(0xC0FFEE); + const sampledMeans = []; + for (let i = 0; i < iterations; i += 1) { + let total = 0; + for (let j = 0; j < clean.length; j += 1) { + total += clean[Math.floor(rand() * clean.length)]; + } + sampledMeans.push(total / clean.length); + } + sampledMeans.sort((a, b) => a - b); + return { + low: sampledMeans[Math.floor(iterations * 0.025)], + high: sampledMeans[Math.floor(iterations * 0.975)], + method: 'deterministic_bootstrap_mean_95_ci', + samples: clean.length, + }; +} + +function qualityFromRow(row) { + const final = row.score?.final || {}; + return { + lane: row.lane_id, + task: row.task_id, + fileCoverage: final.file?.coverage ?? null, + filePrecision: final.file?.precision ?? null, + symbolCoverage: final.symbol?.coverage ?? null, + symbolPrecision: final.symbol?.precision ?? null, + spanCoverage: final.span?.coverage ?? null, + spanPrecision: final.span?.precision ?? null, + lineCoverage: final.line?.coverage ?? null, + linePrecision: final.line?.precision ?? null, + editlocRecall: row.score?.editloc?.recall ?? null, + editlocPrecision: row.score?.editloc?.precision ?? null, + qualitySource: 'official ContextBench evaluator', + }; +} + +function costFromRow(row) { + const t = row.timeMetrics || {}; + return { + lane: row.lane_id, + setupDurationMs: metricValue(t.setupDurationMs), + indexDurationMs: metricValue(t.indexDurationMs), + queryDurationMs: metricValue(t.queryDurationMs), + selectorDurationMs: metricValue(t.selectorDurationMs), + evaluatorDurationMs: metricValue(t.evaluatorDurationMs), + rowWallDurationMs: metricValue(t.rowWallDurationMs), + unavailable: Object.fromEntries(Object.entries(t) + .filter(([, metric]) => metric?.unavailableReason) + .map(([key, metric]) => [key, metric.unavailableReason])), + }; +} + +function tokensFromRow(row) { + const tokenMetrics = row.tokenMetrics || {}; + const candidatePack = tokenMetrics.candidatePack || {}; + const prediction = tokenMetrics.prediction || {}; + const selectorUsage = tokenMetrics.selectorUsage || {}; + return { + lane: row.lane_id, + estimator: tokenMetrics.estimator || null, + candidateCount: candidatePack.candidateCount ?? row.candidateCount ?? null, + candidateBytes: metricValue(candidatePack.bytes), + candidateEstimatedTokens: metricValue(candidatePack.estimatedTokens), + predictionBytes: metricValue(prediction.bytes), + predictionEstimatedTokens: metricValue(prediction.estimatedTokens), + selectorModel: selectorUsage.model || row.model || null, + selectorInputTokens: metricValue(selectorUsage.inputTokens), + selectorOutputTokens: metricValue(selectorUsage.outputTokens), + selectorTotalTokens: metricValue(selectorUsage.totalTokens), + unavailable: { + candidateBytes: candidatePack.bytes?.unavailableReason || null, + candidateEstimatedTokens: candidatePack.estimatedTokens?.unavailableReason || null, + selectorUsage: selectorUsage.totalTokens?.unavailableReason || null, + }, + }; +} + +function reliabilityFromRow(row) { + return { + lane: row.lane_id, + task: row.task_id, + status: row.status, + officialEvaluatorScoreable: row.officialEvaluatorScoreable === true, + setupStatus: row.reliability?.setupStatus ?? row.setupStatus, + indexStatus: row.reliability?.indexStatus ?? row.indexStatus, + toolCallable: row.reliability?.toolCallable ?? row.toolCallable, + nonEmptyPrediction: row.reliability?.nonEmptyPrediction ?? row.nonEmptyPrediction, + candidateCount: row.reliability?.candidateCount ?? row.candidateCount, + sourceRun: row.reliability?.sourceRun ?? null, + sourceJob: row.reliability?.sourceJob ?? null, + sourceArtifact: row.reliability?.sourceArtifact ?? null, + sourceDigest: row.reliability?.sourceDigest ?? null, + }; +} + +function laneStatistics(qualityRows) { + const metrics = ['fileCoverage', 'filePrecision', 'spanCoverage', 'spanPrecision', 'lineCoverage', 'linePrecision']; + const byLane = new Map(); + for (const row of qualityRows) { + const list = byLane.get(row.lane) || []; + list.push(row); + byLane.set(row.lane, list); + } + return Object.fromEntries([...byLane.entries()].map(([lane, rows]) => [ + lane, + Object.fromEntries(metrics.map((metric) => { + const values = rows.map((row) => row[metric]); + return [metric, { + mean: mean(values), + median: median(values), + stddev: stddev(values), + bootstrap95Ci: bootstrapCi(values), + }]; + })), + ])); +} + +function validate(summary, protocol, taskManifest, qualityRows, reliabilityRows) { + const requiredLanes = summary.requiredLanes || []; + const scoreableLanes = new Set(qualityRows.map((row) => row.lane)); + const failures = reliabilityRows.filter((row) => !row.officialEvaluatorScoreable); + const checks = [ + { + id: 'official_evaluator_only_quality_rows', + pass: qualityRows.every((row) => row.qualitySource === 'official ContextBench evaluator'), + detail: 'Every quality row must come from the official ContextBench evaluator.', + }, + { + id: 'failures_excluded_from_quality_table', + pass: failures.every((failure) => !qualityRows.some((row) => row.lane === failure.lane && row.task === failure.task)), + detail: 'setup_failed/tool_error/judge_failed/empty rows are reliability rows, not quality rows.', + }, + { + id: 'required_lanes_scoreable', + pass: requiredLanes.length >= 5 && requiredLanes.every((lane) => scoreableLanes.has(lane)), + detail: `Required lanes: ${requiredLanes.join(', ') || 'none'}`, + }, + { + id: 'setup_index_query_costs_separate', + pass: summary.setupIndexCostReportedSeparately === true && summary.rows.every((row) => ['setupDurationMs', 'indexDurationMs', 'queryDurationMs'].every((key) => hasMetricOrReason(row.timeMetrics?.[key]))), + detail: 'Setup, indexing, and query costs must not be folded into quality metrics.', + }, + { + id: 'timing_metrics_present', + pass: summary.rows.every((row) => ['evaluatorDurationMs', 'rowWallDurationMs', 'selectorDurationMs'].every((key) => hasMetricOrReason(row.timeMetrics?.[key]))), + detail: 'Each row needs evaluator, row-wall, and selector timing fields or explicit unavailable reasons.', + }, + { + id: 'token_metrics_present', + pass: summary.rows.every((row) => row.tokenMetrics?.prediction?.estimatedTokens && row.tokenMetrics?.candidatePack?.estimatedTokens && row.tokenMetrics?.selectorUsage?.totalTokens), + detail: 'Each row needs prediction, candidate-pack, and selector token fields; unavailable provider usage must be explicit.', + }, + { + id: 'protocol_is_frozen_and_non_claiming', + pass: protocol.status === 'protocol_frozen' && protocol.claimAllowed === false, + detail: 'This proof can be publishable evidence, but not a broad benchmark-win claim.', + }, + { + id: 'manifest_preselected_without_lane_outputs', + pass: typeof taskManifest.no_lane_outputs_observed_attestation === 'string' && taskManifest.no_lane_outputs_observed_attestation.length > 0, + detail: 'Task selection must not depend on lane outputs.', + }, + ]; + return { + status: checks.every((check) => check.pass) ? 'pass' : 'fail', + checks, + }; +} + +function formatNumber(value) { + if (value === null || value === undefined) return 'n/a'; + if (Number.isInteger(value)) return String(value); + return Number(value).toFixed(3); +} + +function markdownTable(rows, columns) { + const header = `| ${columns.map((column) => column.label).join(' | ')} |`; + const divider = `| ${columns.map(() => '---').join(' | ')} |`; + const body = rows.map((row) => `| ${columns.map((column) => formatNumber(column.value(row))).join(' | ')} |`); + return [header, divider, ...body].join('\n'); +} + +function humanizedMarkdown(report) { + const table = markdownTable(report.qualityTable, [ + { label: 'Lane', value: (row) => row.lane }, + { label: 'File cov', value: (row) => row.fileCoverage }, + { label: 'File prec', value: (row) => row.filePrecision }, + { label: 'Span cov', value: (row) => row.spanCoverage }, + { label: 'Line cov', value: (row) => row.lineCoverage }, + ]); + const costTable = markdownTable(report.costTable, [ + { label: 'Lane', value: (row) => row.lane }, + { label: 'Setup ms', value: (row) => row.setupDurationMs }, + { label: 'Index ms', value: (row) => row.indexDurationMs }, + { label: 'Query ms', value: (row) => row.queryDurationMs }, + { label: 'Eval ms', value: (row) => row.evaluatorDurationMs }, + { label: 'Row ms', value: (row) => row.rowWallDurationMs }, + ]); + const tokenTable = markdownTable(report.tokenTable, [ + { label: 'Lane', value: (row) => row.lane }, + { label: 'Candidates', value: (row) => row.candidateCount }, + { label: 'Cand tokens', value: (row) => row.candidateEstimatedTokens }, + { label: 'Pred tokens', value: (row) => row.predictionEstimatedTokens }, + { label: 'Selector tokens', value: (row) => row.selectorTotalTokens }, + ]); + return `# ContextBench Publishable Pilot Summary\n\nThis is a real five-lane ContextBench pilot: the quality numbers below come only from the official ContextBench evaluator, and setup/index/query costs are reported separately. It is still a pilot because it covers one frozen task; it should not be turned into a broad win claim.\n\n## Quality\n\n${table}\n\n## Time Cost\n\n${costTable}\n\n## Token Cost\n\n${tokenTable}\n\n## What This Means\n\nThe run is useful because every required lane produced a non-empty prediction and received an official score. The remaining n/a values are measurement gaps, not hidden failures: selector provider token telemetry was not captured in this proof, and some readiness artifacts report candidate counts without candidate bytes.\n\n## What Not To Claim\n\nDo not claim product superiority from this artifact alone. A claim-bearing benchmark still needs the full frozen task slice, repeated runs, and the same reporting discipline used here.\n`; +} + +if (!existsSync(summaryPath)) throw new Error(`summary not found: ${summaryPath}`); +if (!existsSync(protocolPath)) throw new Error(`protocol not found: ${protocolPath}`); +if (!existsSync(lanesPath)) throw new Error(`lanes fixture not found: ${lanesPath}`); +if (!existsSync(taskManifestPath)) throw new Error(`task manifest not found: ${taskManifestPath}`); + +const summary = readJson(summaryPath); +const protocol = readJson(protocolPath); +const lanes = readJson(lanesPath); +const taskManifest = readJson(taskManifestPath); +const rows = summary.rows || []; +const qualityRows = rows.filter((row) => row.officialEvaluatorScoreable === true && row.status === 'completed').map(qualityFromRow); +const reliabilityRows = rows.map(reliabilityFromRow); +const costTable = rows.map(costFromRow); +const tokenTable = rows.map(tokensFromRow); +const validation = validate(summary, protocol, taskManifest, qualityRows, reliabilityRows); + +const targetTaskIds = [...new Set(rows.map((row) => row.task_id).filter(Boolean))]; +const report = { + createdAt: new Date().toISOString(), + publicationStatus: 'pilot_evidence_not_broad_claim', + sourceSummary: resolve(summaryPath), + protocol: { + path: protocolPath, + hash: hashFile(protocolPath), + version: protocol.protocolVersion, + status: protocol.status, + claimAllowed: protocol.claimAllowed, + }, + fixtures: { + lanes: { path: lanesPath, hash: hashFile(lanesPath), laneCount: lanes.lanes?.length ?? null }, + taskManifest: { path: taskManifestPath, hash: hashFile(taskManifestPath), manifestHash: taskManifest.manifest_hash ?? null }, + }, + scope: { + model: summary.model, + targetTaskIds, + attemptedRows: summary.attemptedRows, + scoreableRows: summary.scoreableRows, + requiredLanes: summary.requiredLanes || [], + caveats: summary.caveats || [], + }, + qualityTable: qualityRows, + costTable, + tokenTable, + reliabilityTable: reliabilityRows, + statistics: laneStatistics(qualityRows), + biasAudit: validation, + limitations: [ + 'One-task proof run: suitable as pilot evidence, not a broad benchmark claim.', + 'Selector usage telemetry is explicit but unavailable where the selector ran outside provider usage capture.', + 'Candidate token estimates use a deterministic utf8-byte heuristic unless a lane emits provider-grade token counts.', + 'Failures remain in reliability tables and are never counted as quality rows.', + ], +}; + +writeJson(outPath, report); +writeJson(validationOutPath, validation); +mkdirSync(dirname(humanizedOutPath), { recursive: true }); +writeFileSync(humanizedOutPath, humanizedMarkdown(report)); +console.log('CONTEXTBENCH_PUBLISHABLE_REPORT_JSON_START'); +console.log(JSON.stringify({ outPath, validationOutPath, humanizedOutPath, validation }, null, 2)); +console.log('CONTEXTBENCH_PUBLISHABLE_REPORT_JSON_END'); +if (validation.status !== 'pass') process.exitCode = 1; From f8ccb0f71fa7ce304da6cf8281121ac3753d0206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 11:41:27 +0200 Subject: [PATCH 070/100] Publish validated ContextBench pilot artifacts --- .../workflows/contextbench-five-lane-score.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/contextbench-five-lane-score.yml b/.github/workflows/contextbench-five-lane-score.yml index bda22e0..9afe206 100644 --- a/.github/workflows/contextbench-five-lane-score.yml +++ b/.github/workflows/contextbench-five-lane-score.yml @@ -7,7 +7,11 @@ on: - .github/workflows/contextbench-five-lane-score.yml - scripts/contextbench-score-five-lane-selections.mjs - scripts/contextbench-score-five-lane-artifact-selections.mjs + - scripts/contextbench-build-publishable-report.mjs - scripts/contextbench-five-lane-selections.json + - tests/fixtures/contextbench-benchmark-protocol.json + - tests/fixtures/contextbench-lanes.json + - tests/fixtures/contextbench-task-manifest.json workflow_dispatch: permissions: @@ -84,6 +88,17 @@ jobs: - name: Score five ready lane selections shell: bash run: node scripts/contextbench-score-five-lane-artifact-selections.mjs + - name: Build publishable pilot report + shell: bash + run: | + node scripts/contextbench-build-publishable-report.mjs \ + --summary "$ROOT/summary.json" \ + --protocol tests/fixtures/contextbench-benchmark-protocol.json \ + --lanes tests/fixtures/contextbench-lanes.json \ + --task-manifest tests/fixtures/contextbench-task-manifest.json \ + --out "$ROOT/publishable-summary.json" \ + --validation-out "$ROOT/publishable-validation.json" \ + --humanized-out "$ROOT/humanized-summary.md" - name: Upload five-lane score artifacts if: always() uses: actions/upload-artifact@v4 From 5f6f2cda42e109cd24ca3ef7595c6971a0f4bbf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 11:45:31 +0200 Subject: [PATCH 071/100] Tighten publishable report validation --- scripts/contextbench-build-publishable-report.mjs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/contextbench-build-publishable-report.mjs b/scripts/contextbench-build-publishable-report.mjs index bf706f6..ba61603 100644 --- a/scripts/contextbench-build-publishable-report.mjs +++ b/scripts/contextbench-build-publishable-report.mjs @@ -229,7 +229,9 @@ function validate(summary, protocol, taskManifest, qualityRows, reliabilityRows) }, { id: 'token_metrics_present', - pass: summary.rows.every((row) => row.tokenMetrics?.prediction?.estimatedTokens && row.tokenMetrics?.candidatePack?.estimatedTokens && row.tokenMetrics?.selectorUsage?.totalTokens), + pass: summary.rows.every((row) => hasMetricOrReason(row.tokenMetrics?.prediction?.estimatedTokens) + && hasMetricOrReason(row.tokenMetrics?.candidatePack?.estimatedTokens) + && hasMetricOrReason(row.tokenMetrics?.selectorUsage?.totalTokens)), detail: 'Each row needs prediction, candidate-pack, and selector token fields; unavailable provider usage must be explicit.', }, { @@ -251,8 +253,10 @@ function validate(summary, protocol, taskManifest, qualityRows, reliabilityRows) function formatNumber(value) { if (value === null || value === undefined) return 'n/a'; + if (typeof value === 'string') return value; if (Number.isInteger(value)) return String(value); - return Number(value).toFixed(3); + const numeric = Number(value); + return Number.isFinite(numeric) ? numeric.toFixed(3) : String(value); } function markdownTable(rows, columns) { From d4ad1eb24007eae3290c00c4d7f29cb935759200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 11:47:57 +0200 Subject: [PATCH 072/100] Add compact ContextBench report printer --- .../contextbench-print-publishable-report.mjs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 scripts/contextbench-print-publishable-report.mjs diff --git a/scripts/contextbench-print-publishable-report.mjs b/scripts/contextbench-print-publishable-report.mjs new file mode 100644 index 0000000..b0e1b9b --- /dev/null +++ b/scripts/contextbench-print-publishable-report.mjs @@ -0,0 +1,21 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const reportPath = process.argv[2] || join(process.env.ROOT || '/tmp/contextbench-five-lane-score', 'publishable-summary.json'); +const report = JSON.parse(readFileSync(reportPath, 'utf8')); + +const compact = { + publicationStatus: report.publicationStatus, + validationStatus: report.biasAudit?.status || null, + scope: report.scope, + qualityTable: report.qualityTable, + costTable: report.costTable, + tokenTable: report.tokenTable, + reliabilityTable: report.reliabilityTable, + limitations: report.limitations, +}; + +console.log('CONTEXTBENCH_PUBLISHABLE_COMPACT_JSON_START'); +console.log(JSON.stringify(compact, null, 2)); +console.log('CONTEXTBENCH_PUBLISHABLE_COMPACT_JSON_END'); +if (report.biasAudit?.status !== 'pass') process.exitCode = 1; From 079513189d1cde0d14ea87f04c285c4318e81eb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 11:48:36 +0200 Subject: [PATCH 073/100] Print compact ContextBench pilot report in CI --- .github/workflows/contextbench-five-lane-score.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/contextbench-five-lane-score.yml b/.github/workflows/contextbench-five-lane-score.yml index 9afe206..58dbfc9 100644 --- a/.github/workflows/contextbench-five-lane-score.yml +++ b/.github/workflows/contextbench-five-lane-score.yml @@ -8,6 +8,7 @@ on: - scripts/contextbench-score-five-lane-selections.mjs - scripts/contextbench-score-five-lane-artifact-selections.mjs - scripts/contextbench-build-publishable-report.mjs + - scripts/contextbench-print-publishable-report.mjs - scripts/contextbench-five-lane-selections.json - tests/fixtures/contextbench-benchmark-protocol.json - tests/fixtures/contextbench-lanes.json @@ -99,6 +100,9 @@ jobs: --out "$ROOT/publishable-summary.json" \ --validation-out "$ROOT/publishable-validation.json" \ --humanized-out "$ROOT/humanized-summary.md" + - name: Print compact publishable pilot report + shell: bash + run: node scripts/contextbench-print-publishable-report.mjs "$ROOT/publishable-summary.json" - name: Upload five-lane score artifacts if: always() uses: actions/upload-artifact@v4 From de3d70bd4da669b8821684ef08004d4e1e3c705a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 11:52:26 +0200 Subject: [PATCH 074/100] Preserve unavailable benchmark metrics as null --- ...ontextbench-score-five-lane-selections.mjs | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/scripts/contextbench-score-five-lane-selections.mjs b/scripts/contextbench-score-five-lane-selections.mjs index 17a6f90..5c36bfc 100644 --- a/scripts/contextbench-score-five-lane-selections.mjs +++ b/scripts/contextbench-score-five-lane-selections.mjs @@ -68,14 +68,20 @@ function addSpan(map, file, start, end) { map.set(clean, list); } +function finiteNumber(value) { + if (value === null || value === undefined || value === '') return null; + const numeric = Number(value); + return Number.isFinite(numeric) ? numeric : null; +} + function estimateTokensFromBytes(bytes) { if (!Number.isFinite(bytes)) return null; return Math.ceil(bytes / 4); } function measuredNumber(value, unit, source, unavailableReason = 'not captured in source artifact') { - const numeric = Number(value); - if (Number.isFinite(numeric)) return { value: numeric, unit, source }; + const numeric = finiteNumber(value); + if (numeric !== null) return { value: numeric, unit, source }; return { value: null, unit, source, unavailableReason }; } @@ -99,20 +105,20 @@ function buildTimeMetrics(readiness, evaluator, rowWallDurationMs, evaluatorSkip function buildTokenMetrics(selection, prediction) { const candidateMetrics = selection.candidateMetrics || selection.readiness?.candidateMetrics || {}; - const candidateBytes = Number(candidateMetrics.bytes); - const candidateEstimatedTokens = Number(candidateMetrics.estimatedTokens); + const candidateBytes = finiteNumber(candidateMetrics.bytes); + const candidateEstimatedTokens = finiteNumber(candidateMetrics.estimatedTokens); const predictionBytes = byteCount(JSON.stringify(prediction || {})); const selectorUsage = selection.selectorUsage || {}; return { estimator: 'ceil(utf8_bytes/4); cost estimate only, not provider billing telemetry', candidatePack: { - candidateCount: Number(selection.readiness?.candidateCount ?? selection.candidateCount ?? candidateMetrics.candidateCount ?? 0), - fileCount: Number.isFinite(Number(candidateMetrics.fileCount)) ? Number(candidateMetrics.fileCount) : null, - spanCount: Number.isFinite(Number(candidateMetrics.spanCount)) ? Number(candidateMetrics.spanCount) : null, - bytes: Number.isFinite(candidateBytes) + candidateCount: finiteNumber(selection.readiness?.candidateCount ?? selection.candidateCount ?? candidateMetrics.candidateCount) ?? 0, + fileCount: finiteNumber(candidateMetrics.fileCount), + spanCount: finiteNumber(candidateMetrics.spanCount), + bytes: candidateBytes !== null ? measuredNumber(candidateBytes, 'bytes', candidateMetrics.source || 'candidate pack artifact') : measuredNumber(null, 'bytes', candidateMetrics.source || 'candidate pack artifact', candidateMetrics.unavailableReason || 'candidate pack bytes were not emitted for this lane'), - estimatedTokens: Number.isFinite(candidateEstimatedTokens) + estimatedTokens: candidateEstimatedTokens !== null ? measuredNumber(candidateEstimatedTokens, 'tokens', candidateMetrics.source || 'candidate pack artifact') : measuredNumber(null, 'tokens', candidateMetrics.source || 'candidate pack artifact', candidateMetrics.unavailableReason || 'candidate pack token estimate was not emitted for this lane'), }, @@ -217,7 +223,7 @@ for (const selection of laneSelections) { setupStatus: readiness.setupStatus || selection.setupStatus || 'unknown', indexStatus: readiness.indexStatus || selection.indexStatus || 'unknown', toolCallable: Boolean(readiness.toolCallable ?? selection.toolCallable), - candidateCount: Number(readiness.candidateCount ?? selection.candidateCount ?? 0), + candidateCount: finiteNumber(readiness.candidateCount ?? selection.candidateCount) ?? 0, setupIndex: readiness.setupIndex || selection.setupIndex || null, nonEmptyPrediction, predFiles: predFiles.length, From 5349e5da40b5540dd05d15c836f2d997d907e385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 11:54:09 +0200 Subject: [PATCH 075/100] Keep null benchmark metrics out of numeric tables --- scripts/contextbench-build-publishable-report.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/contextbench-build-publishable-report.mjs b/scripts/contextbench-build-publishable-report.mjs index ba61603..0b5a5a1 100644 --- a/scripts/contextbench-build-publishable-report.mjs +++ b/scripts/contextbench-build-publishable-report.mjs @@ -30,6 +30,7 @@ function writeJson(path, data) { } function numberOrNull(value) { + if (value === null || value === undefined || value === '') return null; const numeric = Number(value); return Number.isFinite(numeric) ? numeric : null; } @@ -41,7 +42,7 @@ function metricValue(metric) { function hasMetricOrReason(metric) { if (!metric || typeof metric !== 'object') return false; - if (Number.isFinite(Number(metric.value))) return true; + if (numberOrNull(metric.value) !== null) return true; return typeof metric.unavailableReason === 'string' && metric.unavailableReason.length > 0; } From bbd3a8348aaec15809fd09dd8fc729e64df6d878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 12:01:31 +0200 Subject: [PATCH 076/100] Tighten ContextBench score artifact output --- .../contextbench-five-lane-score.yml | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/contextbench-five-lane-score.yml b/.github/workflows/contextbench-five-lane-score.yml index 58dbfc9..79e4513 100644 --- a/.github/workflows/contextbench-five-lane-score.yml +++ b/.github/workflows/contextbench-five-lane-score.yml @@ -88,7 +88,12 @@ jobs: path: ${{ env.EXTERNAL_READINESS_ROOT }}/ripgrep - name: Score five ready lane selections shell: bash - run: node scripts/contextbench-score-five-lane-artifact-selections.mjs + run: | + set +e + node scripts/contextbench-score-five-lane-artifact-selections.mjs > "$ROOT/logs/score-five-lane.log" 2>&1 + status=$? + tail -n 120 "$ROOT/logs/score-five-lane.log" + exit "$status" - name: Build publishable pilot report shell: bash run: | @@ -108,5 +113,22 @@ jobs: uses: actions/upload-artifact@v4 with: name: contextbench-five-lane-score - path: /tmp/contextbench-five-lane-score + path: | + /tmp/contextbench-five-lane-score/summary.json + /tmp/contextbench-five-lane-score/publishable-summary.json + /tmp/contextbench-five-lane-score/publishable-validation.json + /tmp/contextbench-five-lane-score/humanized-summary.md + /tmp/contextbench-five-lane-score/task-payloads.json + /tmp/contextbench-five-lane-score/logs/** + /tmp/contextbench-five-lane-score/lane-score/summary.json + /tmp/contextbench-five-lane-score/lane-score/gold.json + /tmp/contextbench-five-lane-score/lane-score/gold-command.json + /tmp/contextbench-five-lane-score/lane-score/selections.json + /tmp/contextbench-five-lane-score/lane-score/*/selection.json + /tmp/contextbench-five-lane-score/lane-score/*/prediction.json + /tmp/contextbench-five-lane-score/lane-score/*/official-score.jsonl + /tmp/contextbench-five-lane-score/lane-score/*/evaluator-command.json + /tmp/contextbench-five-lane-score/external-readiness/**/*.json + /tmp/contextbench-five-lane-score/external-readiness/**/*.jsonl + /tmp/contextbench-five-lane-score/external-readiness/**/*.md retention-days: 14 From 86644d22685ebde70ca73d0848a39e412dd0ac64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Mon, 11 May 2026 12:04:45 +0200 Subject: [PATCH 077/100] Document ContextBench implementation pilot evidence --- docs/benchmark.md | 104 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 95 insertions(+), 9 deletions(-) diff --git a/docs/benchmark.md b/docs/benchmark.md index d48f807..bf71928 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -1,9 +1,95 @@ -# Discovery Benchmark +# Benchmarks -This page documents the current public discovery proof from the checked-in result artifacts on `master`. +This page tracks two separate benchmark surfaces: + +- The ContextBench implementation pilot, which uses the official ContextBench evaluator on one frozen task and five scoreable lanes. +- The older discovery benchmark, which measures local discovery usefulness and payload cost only. + +Neither section currently supports a broad benchmark-win claim. + +## ContextBench Implementation Pilot + +This is the current implementation-quality pilot for ContextBench. It is real scoreable evidence, but it is still a pilot because it covers one frozen task rather than the full frozen 20-task slice. + +### Scope + +- Protocol: `tests/fixtures/contextbench-benchmark-protocol.json` +- Task manifest: `tests/fixtures/contextbench-task-manifest.json` +- Selection file: `scripts/contextbench-five-lane-selections.json` +- Workflow: `.github/workflows/contextbench-five-lane-score.yml` +- Required lanes: `raw-native`, `codebase-context`, `codebase-memory-mcp`, `grepai`, `ripgrep-lexical` +- Model used for selection: `gpt-5.4-mini-high` +- Target task: `SWE-Bench-Pro__go__maintenance__bugfix__4df06349` +- Repository under test: `navidrome/navidrome` +- Base commit: `537e2fc033b71a4a69190b74f755ebc352bb4196` + +`CodeGraphContext` is not counted in this five-lane pilot because its supported CLI path indexed successfully but returned zero task-relevant candidates during readiness. That remains a readiness blocker, not a quality result. + +### Current Audited Run + +- Run: `25663469903` +- Job: `75329796667` +- Commit: `bbd3a8348aaec15809fd09dd8fc729e64df6d878` +- Artifact: `6915576867` +- Artifact digest: `sha256:718fd32049a2d98ed62fb0c15189d7dc9f1b027c202f286923de91d9f8985def` +- Artifact size: `88.9 KB` +- Uploaded files: `42` +- Status: `success` + +The artifact contains `summary.json`, `publishable-summary.json`, `publishable-validation.json`, `humanized-summary.md`, logs, lane selections, lane predictions, and official evaluator score files. It intentionally excludes full cloned repos and evaluator caches so the evidence package is small enough to inspect. + +### Quality Results + +Only rows scored by the official ContextBench evaluator are included here. Setup failures, tool errors, empty predictions, and judge failures are reliability outcomes, not quality rows. + +| Lane | File cov | File prec | Span cov | Span prec | Line cov | Line prec | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `raw-native` | 0.222 | 0.667 | 0.370 | 0.391 | 0.365 | 0.365 | +| `codebase-context` | 0.889 | 1.000 | 0.899 | 0.356 | 0.887 | 0.323 | +| `codebase-memory-mcp` | 0.222 | 0.667 | 0.346 | 0.380 | 0.315 | 0.337 | +| `grepai` | 0.333 | 0.500 | 0.048 | 0.042 | 0.059 | 0.061 | +| `ripgrep-lexical` | 0.222 | 0.667 | 0.401 | 0.341 | 0.419 | 0.302 | + +### Cost And Telemetry + +The report separates setup, indexing, query, selector, evaluator, and row-wall timing from quality. It also reports candidate counts, candidate token estimates when available, prediction token estimates, and selector token telemetry fields. + +`n/a` means the measurement was explicitly unavailable, not zero and not a hidden failure. Current gaps are: + +- Selector wall-clock and provider token telemetry were not captured in this proof artifact. +- `raw-native`, `codebase-context`, and `codebase-memory-mcp` emitted candidate counts but not candidate-pack bytes. +- `codebase-context` readiness did not emit index/query duration in the source artifact. + +### Bias Controls + +The generated `publishable-validation.json` must pass these checks before the report is treated as evidence: + +- Quality rows come only from the official ContextBench evaluator. +- Failed or unscoreable rows stay out of the quality table. +- All required lanes are scoreable. +- Setup, index, and query costs are separate from quality. +- Timing and token fields exist or carry explicit unavailable reasons. +- The protocol is frozen and `claimAllowed` is `false`. +- The task manifest attests that lane outputs were not observed during task selection. + +### What This Supports + +- It supports saying that the benchmark harness can produce real official ContextBench scores across five lanes. +- It supports saying that setup/index/query cost and context/token cost are now tracked separately from quality. +- It supports saying that the one-task pilot found a strong `codebase-context` result on this specific task. + +### What This Does Not Support + +- It does not support claiming that `codebase-context` beats competitors overall. +- It does not support claiming patch correctness or productivity improvements. +- It does not replace the full frozen 20-task, repeated-run benchmark required for claim-bearing results. + +## Discovery Benchmark + +This section documents the current public discovery proof from the checked-in result artifacts on `master`. It is a discovery benchmark, not an implementation-quality benchmark. -## Scope +## Discovery Scope - Frozen fixtures: - `tests/fixtures/discovery-angular-spotify.json` @@ -17,7 +103,7 @@ It is a discovery benchmark, not an implementation-quality benchmark. - Comparator evidence: - `results/comparator-evidence.json` -## How To Reproduce +## Discovery Reproduction Run the repo-local proof artifacts from the current `master` checkout: @@ -28,7 +114,7 @@ node scripts/benchmark-comparators.mjs --repos repos/angular-spotify,repos/excal node scripts/run-eval.mjs repos/angular-spotify repos/excalidraw --mode=discovery --fixture-a=tests/fixtures/discovery-angular-spotify.json --fixture-b=tests/fixtures/discovery-excalidraw.json --competitor-results=results/comparator-evidence.json --skip-reindex --output=results/gate-evaluation.json ``` -## Current Result +## Discovery Current Result From `results/gate-evaluation.json`: @@ -47,7 +133,7 @@ Repo-level outputs from the same rerun: | `angular-spotify` | 12 | 0.8333 | 2138.4167 | 0.25 | | `excalidraw` | 12 | 0.6667 | 1506.0833 | 0 | -## Gate Truth +## Discovery Gate Truth The gate is intentionally still blocked. @@ -58,7 +144,7 @@ The gate is intentionally still blocked. - `codebase-memory-mcp` now has real current metrics, but the gate still marks it `failed` on the frozen tolerance rule - Three comparator lanes still fail setup entirely: `GrepAI`, `jCodeMunch`, and `CodeGraphContext`. -## Comparator Reality +## Discovery Comparator Reality The current comparator artifact records incomplete comparator evidence, not benchmark wins. @@ -72,7 +158,7 @@ The current comparator artifact records incomplete comparator evidence, not benc `CodeGraphContext` remains part of the frozen comparison frame. It is not omitted from the public story just because the lane still fails to start. -## Important Limitations +## Discovery Important Limitations - This benchmark measures discovery usefulness and payload cost only. - It does not measure implementation correctness, patch quality, or end-to-end task completion. @@ -80,7 +166,7 @@ The current comparator artifact records incomplete comparator evidence, not benc - The reranker cache is currently corrupted on this machine. During the proof rerun, search fell back to original ordering after `Protobuf parsing failed` while still completing the harness. - `averageFirstRelevantHit` remains `null` in the current gate output, which is enough to keep the raw-Claude baseline in `pending_evidence`. -## What This Proof Can Support +## Discovery Claims Supported - It can support claims about the shipped discovery surfaces and their current measured outputs on the frozen public tasks. - It can support claims that the proof gate is still blocked by comparator evidence. From 55b97b9c2261278d3c92ee1f3000d0a80920432d Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Sun, 5 Jul 2026 09:43:29 -0500 Subject: [PATCH 078/100] fix(reranker): Score passages with the supported tokenizer pair API The released branch was loading transformers 3.8.1 correctly, but the reranker was calling the tokenizer in a shape that ignored the passage. Keep the dependency on the released package version and make the recovery test assert the text_pair call so this does not slip back. --- src/core/reranker.ts | 15 +++++++++++---- src/embeddings/transformers.ts | 2 +- tests/reranker-recovery.test.ts | 34 ++++++++++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/core/reranker.ts b/src/core/reranker.ts index 47de513..2dda09c 100644 --- a/src/core/reranker.ts +++ b/src/core/reranker.ts @@ -21,9 +21,15 @@ const AMBIGUITY_THRESHOLD = 0.08; interface CrossEncoderTokenizer { ( - query: string, - passage: string, - options: { padding: boolean; truncation: boolean; max_length: number } + text: string, + options?: { + text_pair?: string | null; + padding?: boolean | 'max_length'; + truncation?: boolean | null; + max_length?: number | null; + return_tensor?: boolean; + return_token_type_ids?: boolean | null; + } ): unknown; } @@ -152,7 +158,8 @@ async function scorePair(query: string, passage: string): Promise { throw new Error('[reranker] Model not loaded — call ensureModelLoaded() first'); } - const inputs = cachedTokenizer(query, passage, { + const inputs = cachedTokenizer(query, { + text_pair: passage, padding: true, truncation: true, max_length: 512 diff --git a/src/embeddings/transformers.ts b/src/embeddings/transformers.ts index 0a2c581..04db102 100644 --- a/src/embeddings/transformers.ts +++ b/src/embeddings/transformers.ts @@ -70,7 +70,7 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider { const { pipeline } = await import('@huggingface/transformers'); // TS2590: pipeline() resolves AllTasks[T] — a union too complex for TSC to represent. - // Cast to a simpler signature; the actual return type IS FeatureExtractionPipelineType. + // Cast to a simpler feature-extraction signature for both v3 and v4-compatible types. type PipelineFn = ( task: 'feature-extraction', model: string, diff --git a/tests/reranker-recovery.test.ts b/tests/reranker-recovery.test.ts index 7cb77ca..6bf3325 100644 --- a/tests/reranker-recovery.test.ts +++ b/tests/reranker-recovery.test.ts @@ -63,7 +63,12 @@ describe('reranker corruption recovery', () => { await fs.mkdir(modelCacheDir, { recursive: true }); await fs.writeFile(path.join(modelCacheDir, 'model.onnx'), 'corrupt'); - const tokenizer = vi.fn((query: string, passage: string) => ({ query, passage })); + const tokenizer = vi.fn( + (query: string, options?: { text_pair?: string | null }) => ({ + query, + passage: options?.text_pair ?? '' + }) + ); const model = vi.fn(async (inputs: { passage: string }) => { if (inputs.passage.includes('/a.ts')) return { logits: { data: [1] } }; if (inputs.passage.includes('/b.ts')) return { logits: { data: [3] } }; @@ -89,6 +94,33 @@ describe('reranker corruption recovery', () => { const reranked = await rerank('auth token', results); expect(transformersMocks.tokenizerFromPretrained).toHaveBeenCalledTimes(2); expect(transformersMocks.modelFromPretrained).toHaveBeenCalledTimes(2); + expect(tokenizer).toHaveBeenCalledWith( + 'auth token', + expect.objectContaining({ + text_pair: expect.stringContaining('/a.ts'), + padding: true, + truncation: true, + max_length: 512 + }) + ); + expect(tokenizer).toHaveBeenCalledWith( + 'auth token', + expect.objectContaining({ + text_pair: expect.stringContaining('/b.ts'), + padding: true, + truncation: true, + max_length: 512 + }) + ); + expect(tokenizer).toHaveBeenCalledWith( + 'auth token', + expect.objectContaining({ + text_pair: expect.stringContaining('/c.ts'), + padding: true, + truncation: true, + max_length: 512 + }) + ); expect(reranked.map((result) => result.filePath)).toEqual(['/b.ts', '/c.ts', '/a.ts']); expect(getRerankerStatus()).toBe('active'); }); From b51adeb96817ec6d85e17fd4edfd4b5e5a0a91bf Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Sun, 5 Jul 2026 09:43:38 -0500 Subject: [PATCH 079/100] test(release): Keep current version checks from going stale The repo is already on 2.3.0, but the release truth test and manual publish fallback were still pinned to 2.2.0. Derive the assertions from package metadata and normalize macOS temp paths so the full suite checks the real contract instead of local path spelling. --- .github/workflows/publish-npm-on-release.yml | 4 +-- .../contextbench-baseline-schema-gate.test.ts | 25 ++++++++++++++++--- tests/release-truth-surfaces.test.ts | 14 +++++------ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish-npm-on-release.yml b/.github/workflows/publish-npm-on-release.yml index 023126a..4ca7cf4 100644 --- a/.github/workflows/publish-npm-on-release.yml +++ b/.github/workflows/publish-npm-on-release.yml @@ -8,9 +8,9 @@ on: workflow_dispatch: inputs: tag: - description: 'Tag to publish (e.g. v2.2.0)' + description: 'Tag to publish (e.g. v2.3.0)' required: true - default: 'v2.2.0' + default: 'v2.3.0' permissions: contents: read diff --git a/tests/contextbench-baseline-schema-gate.test.ts b/tests/contextbench-baseline-schema-gate.test.ts index ad6fc0a..69b7a93 100644 --- a/tests/contextbench-baseline-schema-gate.test.ts +++ b/tests/contextbench-baseline-schema-gate.test.ts @@ -1,5 +1,12 @@ import { execFileSync } from 'node:child_process'; -import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { + chmodSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync +} from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; @@ -55,6 +62,10 @@ function cleanupSessionRoot(sessionRoot: string): void { } } +function normalizeFilesystemPath(filePath: string): string { + return realpathSync(filePath); +} + function tempSessionRoot(phase: 'phase40' | 'phase41' = 'phase40'): string { return path.join( mkdtempSync(path.join(tmpdir(), `contextbench-${phase}-schema-gate-`)), @@ -398,7 +409,9 @@ describe('ContextBench Phase 40 schema gate', () => { repoCheckoutPath: repoPath, verificationStrict: false }); - expect(readFileSync(cwdCapturePath, 'utf8')).toBe(repoPath); + expect(normalizeFilesystemPath(readFileSync(cwdCapturePath, 'utf8'))).toBe( + normalizeFilesystemPath(repoPath) + ); const stdin = readFileSync(stdinCapturePath, 'utf8'); expect(stdin).toContain('Problem statement:'); expect(stdin).toContain('Fix the failing ContextBench task'); @@ -584,7 +597,9 @@ describe('ContextBench Phase 40 schema gate', () => { ], { encoding: 'utf8', env } ); - expect(readFileSync(cwdPath, 'utf8')).toBe(repoPath); + expect(normalizeFilesystemPath(readFileSync(cwdPath, 'utf8'))).toBe( + normalizeFilesystemPath(repoPath) + ); } execFileSync( 'node', @@ -681,7 +696,9 @@ describe('ContextBench Phase 40 schema gate', () => { (candidate) => candidate.scoring && 'baselineArmId' in candidate.scoring ); expect(row).toMatchObject({ status: 'completed' }); - expect(readFileSync(cwdCapturePath, 'utf8')).toBe(repoPath); + expect(normalizeFilesystemPath(readFileSync(cwdCapturePath, 'utf8'))).toBe( + normalizeFilesystemPath(repoPath) + ); const stdin = readFileSync(stdinCapturePath, 'utf8'); expect(stdin).toContain('Problem statement:'); expect(stdin).toContain('Run the diagnostic arm with materialized task text.'); diff --git a/tests/release-truth-surfaces.test.ts b/tests/release-truth-surfaces.test.ts index c6d8e60..f362ba4 100644 --- a/tests/release-truth-surfaces.test.ts +++ b/tests/release-truth-surfaces.test.ts @@ -73,6 +73,7 @@ function extractMarkdownLinks(markdown: string): string[] { describe('release truth surfaces', () => { const packageJson = readJson('package.json'); const releaseManifest = readJson('.release-please-manifest.json'); + const expectedVersion = packageJson.version; const changelog = readText('CHANGELOG.md'); const readme = readText('README.md'); const workflow = readText('.github/workflows/publish-npm-on-release.yml'); @@ -80,10 +81,9 @@ describe('release truth surfaces', () => { const visualsDoc = readOptionalText('docs/visuals.md'); const packagedPaths = ['README.md', 'LICENSE', ...(packageJson.files ?? [])]; - it('keeps package metadata, release manifest, and changelog on 2.2.0', () => { - expect(packageJson.version).toBe('2.2.0'); - expect(releaseManifest['.']).toBe('2.2.0'); - expect(changelog).toContain('## [2.2.0]'); + it('keeps package metadata, release manifest, and changelog aligned', () => { + expect(releaseManifest['.']).toBe(expectedVersion); + expect(changelog).toContain(`## [${expectedVersion}]`); expect(changelog).not.toContain('## Unreleased'); }); @@ -114,8 +114,8 @@ describe('release truth surfaces', () => { expect(visualsDoc).toContain('Historical snapshot'); }); - it('keeps the manual publish fallback aligned to v2.2.0', () => { - expect(workflow).toContain("description: 'Tag to publish (e.g. v2.2.0)'"); - expect(workflow).toContain("default: 'v2.2.0'"); + it('keeps the manual publish fallback aligned to the package version', () => { + expect(workflow).toContain(`description: 'Tag to publish (e.g. v${expectedVersion})'`); + expect(workflow).toContain(`default: 'v${expectedVersion}'`); }); }); From 6e77a04cd0ed612bd3aadfd9ccfb6b694774f7e9 Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Wed, 15 Jul 2026 14:16:04 -0500 Subject: [PATCH 080/100] Upgrade Transformers.js to v4 Move the local embedding and reranker integrations onto the stable v4 API and ONNX Runtime 1.24.3. This removes the macOS crash dependency while preserving the tokenizer pair fix already covered by this branch. --- package.json | 4 +- pnpm-lock.yaml | 89 ++++++++++++++++++++++++---------- src/core/reranker.ts | 17 +------ src/embeddings/transformers.ts | 6 +-- 4 files changed, 70 insertions(+), 46 deletions(-) diff --git a/package.json b/package.json index 00aeb98..3ec44d1 100644 --- a/package.json +++ b/package.json @@ -128,8 +128,8 @@ "prepare": "husky" }, "dependencies": { + "@huggingface/transformers": "^4.2.0", "@inquirer/prompts": "^3.0.0", - "@huggingface/transformers": "^3.8.1", "@lancedb/lancedb": "^0.4.0", "@modelcontextprotocol/sdk": "^1.27.1", "@typescript-eslint/typescript-estree": "^7.0.0", @@ -172,7 +172,7 @@ "@modelcontextprotocol/sdk>ajv": "8.18.0", "@modelcontextprotocol/sdk>@hono/node-server": "1.19.13", "@modelcontextprotocol/sdk>express-rate-limit": "8.2.2", - "@huggingface/transformers>onnxruntime-node": "1.24.2", + "@huggingface/transformers>onnxruntime-node": "1.24.3", "path-to-regexp": "8.4.0", "brace-expansion": "5.0.5", "micromatch>picomatch": "2.3.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2c6e5ab..b550f49 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,7 +8,7 @@ overrides: '@modelcontextprotocol/sdk>ajv': 8.18.0 '@modelcontextprotocol/sdk>@hono/node-server': 1.19.13 '@modelcontextprotocol/sdk>express-rate-limit': 8.2.2 - '@huggingface/transformers>onnxruntime-node': 1.24.2 + '@huggingface/transformers>onnxruntime-node': 1.24.3 path-to-regexp: 8.4.0 brace-expansion: 5.0.5 micromatch>picomatch: 2.3.2 @@ -24,8 +24,8 @@ importers: .: dependencies: '@huggingface/transformers': - specifier: ^3.8.1 - version: 3.8.1 + specifier: ^4.2.0 + version: 4.2.0 '@inquirer/prompts': specifier: ^3.0.0 version: 3.3.2 @@ -324,12 +324,15 @@ packages: peerDependencies: hono: '>=4.12.12' - '@huggingface/jinja@0.5.5': - resolution: {integrity: sha512-xRlzazC+QZwr6z4ixEqYHo9fgwhTZ3xNSdljlKfUFGZSdlvt166DljRELFUfFytlYOYvo3vTisA/AFOuOAzFQQ==} + '@huggingface/jinja@0.5.9': + resolution: {integrity: sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw==} engines: {node: '>=18'} - '@huggingface/transformers@3.8.1': - resolution: {integrity: sha512-tsTk4zVjImqdqjS8/AOZg2yNLd1z9S5v+7oUPpXaasDRwEDhB+xnglK1k5cad26lL5/ZIaeREgWWy0bs9y9pPA==} + '@huggingface/tokenizers@0.1.3': + resolution: {integrity: sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==} + + '@huggingface/transformers@4.2.0': + resolution: {integrity: sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ==} '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} @@ -377,89 +380,105 @@ packages: resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-arm@1.2.4': resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-ppc64@1.2.4': resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-riscv64@1.2.4': resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} cpu: [riscv64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-s390x@1.2.4': resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-x64@1.2.4': resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linuxmusl-arm64@1.2.4': resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.2.4': resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-linux-arm64@0.34.5': resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-linux-arm@0.34.5': resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-linux-ppc64@0.34.5': resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@img/sharp-linux-riscv64@0.34.5': resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [riscv64] os: [linux] + libc: [glibc] '@img/sharp-linux-s390x@0.34.5': resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-linux-x64@0.34.5': resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-linuxmusl-arm64@0.34.5': resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-linuxmusl-x64@0.34.5': resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-wasm32@0.34.5': resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} @@ -552,12 +571,14 @@ packages: engines: {node: '>= 18'} cpu: [arm64] os: [linux] + libc: [glibc] '@lancedb/lancedb-linux-x64-gnu@0.4.20': resolution: {integrity: sha512-TVicKBhT3Q73gtWmMdR/0+r4gBve5gw3KbcrvzHP/5bNpo2sffETbLEMgLjulunu/Jp8IgC921WW6IFJzNFwrQ==} engines: {node: '>= 18'} cpu: [x64] os: [linux] + libc: [glibc] '@lancedb/lancedb-win32-x64-msvc@0.4.20': resolution: {integrity: sha512-7RsTJAx93NbVD//9m0/TpgQmPsTHlCBfGNFD2zkdZ0j7iacURdG9+egz+Os0veTBqMgwnr6VfMf3T7S0wuu99g==} @@ -661,66 +682,79 @@ packages: resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} @@ -1956,18 +1990,18 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} - onnxruntime-common@1.22.0-dev.20250409-89f8206ba4: - resolution: {integrity: sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==} + onnxruntime-common@1.24.0-dev.20251116-b39e144322: + resolution: {integrity: sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==} - onnxruntime-common@1.24.2: - resolution: {integrity: sha512-S0FFhJaI05jr1c3HVJ/DuPFB/aYdXmnUBuuQfuvLtcNn7WAfpm2ewSXn1vHs9Wa1l8T8OznhfCEdFv8qCn0/xw==} + onnxruntime-common@1.24.3: + resolution: {integrity: sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==} - onnxruntime-node@1.24.2: - resolution: {integrity: sha512-ZtTkUHNNk+Xpi2Sq2BcFjzc9Vx4n3o5RxYLgRvdrFzCBsGUU1dQRFf4LM9tCc7vFhPSoZqbvzZtkzRMXoDqKNg==} + onnxruntime-node@1.24.3: + resolution: {integrity: sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==} os: [win32, darwin, linux] - onnxruntime-web@1.22.0-dev.20250409-89f8206ba4: - resolution: {integrity: sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==} + onnxruntime-web@1.26.0-dev.20260416-b7804b056c: + resolution: {integrity: sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==} openai@4.104.0: resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} @@ -2732,13 +2766,16 @@ snapshots: dependencies: hono: 4.12.14 - '@huggingface/jinja@0.5.5': {} + '@huggingface/jinja@0.5.9': {} + + '@huggingface/tokenizers@0.1.3': {} - '@huggingface/transformers@3.8.1': + '@huggingface/transformers@4.2.0': dependencies: - '@huggingface/jinja': 0.5.5 - onnxruntime-node: 1.24.2 - onnxruntime-web: 1.22.0-dev.20250409-89f8206ba4 + '@huggingface/jinja': 0.5.9 + '@huggingface/tokenizers': 0.1.3 + onnxruntime-node: 1.24.3 + onnxruntime-web: 1.26.0-dev.20260416-b7804b056c sharp: 0.34.5 '@humanfs/core@0.19.1': {} @@ -4481,22 +4518,22 @@ snapshots: dependencies: wrappy: 1.0.2 - onnxruntime-common@1.22.0-dev.20250409-89f8206ba4: {} + onnxruntime-common@1.24.0-dev.20251116-b39e144322: {} - onnxruntime-common@1.24.2: {} + onnxruntime-common@1.24.3: {} - onnxruntime-node@1.24.2: + onnxruntime-node@1.24.3: dependencies: adm-zip: 0.5.16 global-agent: 3.0.0 - onnxruntime-common: 1.24.2 + onnxruntime-common: 1.24.3 - onnxruntime-web@1.22.0-dev.20250409-89f8206ba4: + onnxruntime-web@1.26.0-dev.20260416-b7804b056c: dependencies: flatbuffers: 25.9.23 guid-typescript: 1.0.9 long: 5.3.2 - onnxruntime-common: 1.22.0-dev.20250409-89f8206ba4 + onnxruntime-common: 1.24.0-dev.20251116-b39e144322 platform: 1.3.6 protobufjs: 7.5.6 diff --git a/src/core/reranker.ts b/src/core/reranker.ts index 2dda09c..9dabeac 100644 --- a/src/core/reranker.ts +++ b/src/core/reranker.ts @@ -9,6 +9,7 @@ */ import type { SearchResult } from '../types/index.js'; +import type { PreTrainedTokenizer } from '@huggingface/transformers'; import os from 'os'; const DEFAULT_RERANKER_MODEL = 'Xenova/ms-marco-MiniLM-L-6-v2'; @@ -19,25 +20,11 @@ const RERANK_TOP_K = 10; /** Trigger reranking when the score gap between #1 and #3 is below this threshold */ const AMBIGUITY_THRESHOLD = 0.08; -interface CrossEncoderTokenizer { - ( - text: string, - options?: { - text_pair?: string | null; - padding?: boolean | 'max_length'; - truncation?: boolean | null; - max_length?: number | null; - return_tensor?: boolean; - return_token_type_ids?: boolean | null; - } - ): unknown; -} - interface CrossEncoderModel { (inputs: unknown): Promise<{ logits: { data: ArrayLike } }>; } -let cachedTokenizer: CrossEncoderTokenizer | null = null; +let cachedTokenizer: PreTrainedTokenizer | null = null; let cachedModel: CrossEncoderModel | null = null; let initPromise: Promise | null = null; diff --git a/src/embeddings/transformers.ts b/src/embeddings/transformers.ts index 04db102..7fc62e8 100644 --- a/src/embeddings/transformers.ts +++ b/src/embeddings/transformers.ts @@ -1,5 +1,5 @@ import { EmbeddingProvider, DEFAULT_MODEL } from './types.js'; -import type { FeatureExtractionPipelineType } from '@huggingface/transformers'; +import type { FeatureExtractionPipeline } from '@huggingface/transformers'; import os from 'os'; /** @@ -43,7 +43,7 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider { readonly modelName: string; readonly dimensions: number; - private pipeline: FeatureExtractionPipelineType | null = null; + private pipeline: FeatureExtractionPipeline | null = null; private ready = false; private initPromise: Promise | null = null; @@ -75,7 +75,7 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider { task: 'feature-extraction', model: string, opts: Record - ) => Promise; + ) => Promise; this.pipeline = await (pipeline as PipelineFn)('feature-extraction', this.modelName, { dtype: 'q8', // Limit ONNX Runtime to half cores by default — prevents system freeze during indexing. From cff0760c5b70fd53d526cfd304407802e72e98df Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Wed, 15 Jul 2026 14:53:27 -0500 Subject: [PATCH 081/100] Keep test runs independent of shell credentials Remove secret-named variables before Vitest starts and suppress Node loader deprecations in the CLI subprocess contract. This keeps ContextBench fixtures free of operator environment values and preserves meaningful stderr assertions on Node 26. --- scripts/run-vitest.mjs | 7 ++++++- tests/codebase-map.test.ts | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/run-vitest.mjs b/scripts/run-vitest.mjs index c529a5c..6f7d357 100644 --- a/scripts/run-vitest.mjs +++ b/scripts/run-vitest.mjs @@ -11,9 +11,14 @@ const vitestEntrypoint = fileURLToPath( new URL('../node_modules/vitest/vitest.mjs', import.meta.url) ); +const sensitiveEnvName = /TOKEN|KEY|SECRET|PASSWORD|AUTH|OPENAI|ANTHROPIC|CLAUDE/i; +const testEnv = Object.fromEntries( + Object.entries(process.env).filter(([name]) => !sensitiveEnvName.test(name)) +); + const child = spawn(process.execPath, [vitestEntrypoint, 'run', ...vitestArgs], { stdio: 'inherit', - env: process.env + env: testEnv }); child.on('exit', (code, signal) => { diff --git a/tests/codebase-map.test.ts b/tests/codebase-map.test.ts index 662e5b2..c90de78 100644 --- a/tests/codebase-map.test.ts +++ b/tests/codebase-map.test.ts @@ -84,7 +84,7 @@ async function removeTempMapProject(rootPath: string): Promise { } function runMapCli(args: string[], rootPath: string) { - return spawnSync(process.execPath, ['--import', 'tsx', ENTRYPOINT, 'map', ...args], { + return spawnSync(process.execPath, ['--no-deprecation', '--import', 'tsx', ENTRYPOINT, 'map', ...args], { cwd: CURRENT_REPO_ROOT, env: { ...process.env, From 3be23d646bc5a4b936366c888158d4fe64d707a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:03:26 +0200 Subject: [PATCH 082/100] test: keep scoped memory fixtures fresh --- tests/search-health-scope.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/search-health-scope.test.ts b/tests/search-health-scope.test.ts index 2b5e932..0e175bf 100644 --- a/tests/search-health-scope.test.ts +++ b/tests/search-health-scope.test.ts @@ -28,6 +28,7 @@ describe('search_codebase health and scoped memories', () => { beforeEach(async () => { searchMocks.search.mockReset(); tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'search-health-scope-')); + const currentDate = new Date().toISOString(); const contextDir = path.join(tempRoot, '.codebase-context'); await fs.mkdir(contextDir, { recursive: true }); @@ -56,7 +57,7 @@ describe('search_codebase health and scoped memories', () => { category: 'architecture', memory: 'Avoid direct token reads', reason: 'They bypass AuthService refresh logic.', - date: '2026-04-17T00:00:00.000Z', + date: currentDate, scope: { kind: 'symbol', file: 'src/auth/auth.service.ts', @@ -69,7 +70,7 @@ describe('search_codebase health and scoped memories', () => { category: 'architecture', memory: 'Use auth interceptors', reason: 'They keep HTTP token injection consistent.', - date: '2026-04-17T00:00:00.000Z' + date: currentDate } ], null, From 7ad2fb6203c24f9293f7964cf89deed4e3c09541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:03:30 +0200 Subject: [PATCH 083/100] test: keep scoped memory fixtures fresh --- tests/search-scoped-memory.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/search-scoped-memory.test.ts b/tests/search-scoped-memory.test.ts index 37e4b62..0f6df64 100644 --- a/tests/search-scoped-memory.test.ts +++ b/tests/search-scoped-memory.test.ts @@ -28,6 +28,7 @@ describe('search_codebase scoped memories', () => { beforeEach(async () => { searchMocks.search.mockReset(); tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'search-scoped-memory-')); + const currentDate = new Date().toISOString(); const contextDir = path.join(tempRoot, '.codebase-context'); await fs.mkdir(contextDir, { recursive: true }); @@ -55,7 +56,7 @@ describe('search_codebase scoped memories', () => { category: 'architecture', memory: 'Avoid direct token reads', reason: 'They bypass AuthService refresh logic.', - date: '2026-04-17T00:00:00.000Z', + date: currentDate, scope: { kind: 'symbol', file: 'src/auth/auth.service.ts', @@ -68,7 +69,7 @@ describe('search_codebase scoped memories', () => { category: 'architecture', memory: 'Use auth interceptors', reason: 'They keep HTTP token injection consistent.', - date: '2026-04-17T00:00:00.000Z' + date: currentDate } ], null, From 9ed8577cc930255debaba335b989295c38138b87 Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Sun, 5 Jul 2026 02:33:25 -0500 Subject: [PATCH 084/100] fix(analyzers): Recognize current React and Next patterns React 19 and Next 16 added framework signals that the analyzers were not classifying yet. Keep the analyzer surface current without touching core search or indexing behavior. --- src/analyzers/nextjs/index.ts | 79 +++++++++++++++++++++++++++-------- src/analyzers/react/index.ts | 38 ++++++++++++++++- tests/nextjs-analyzer.test.ts | 34 +++++++++++++++ tests/react-analyzer.test.ts | 28 +++++++++++++ 4 files changed, 159 insertions(+), 20 deletions(-) diff --git a/src/analyzers/nextjs/index.ts b/src/analyzers/nextjs/index.ts index 79fa6c5..ea99dab 100644 --- a/src/analyzers/nextjs/index.ts +++ b/src/analyzers/nextjs/index.ts @@ -21,8 +21,35 @@ import { } from '../shared/metadata.js'; type DetectedPattern = { category: string; name: string }; -type NextRouter = 'app' | 'pages' | 'unknown'; -type NextRouteKind = 'page' | 'layout' | 'route' | 'api' | 'unknown'; +type NextRouter = 'app' | 'pages' | 'proxy' | 'unknown'; +type NextRouteKind = + | 'page' + | 'layout' + | 'route' + | 'api' + | 'loading' + | 'error' + | 'not-found' + | 'template' + | 'default' + | 'forbidden' + | 'unauthorized' + | 'proxy' + | 'unknown'; + +const APP_ROUTE_FILE_KINDS: Readonly> = { + page: 'page', + layout: 'layout', + route: 'route', + loading: 'loading', + error: 'error', + 'global-error': 'error', + 'not-found': 'not-found', + template: 'template', + default: 'default', + forbidden: 'forbidden', + unauthorized: 'unauthorized' +}; interface NextRoutingInfo { router: NextRouter; @@ -44,7 +71,7 @@ export class NextJsAnalyzer implements FrameworkAnalyzer { return false; } - if (isInAppRouter(filePath) || isInPagesRouter(filePath)) { + if (isInAppRouter(filePath) || isInPagesRouter(filePath) || isProxyFile(filePath)) { return true; } @@ -83,6 +110,9 @@ export class NextJsAnalyzer implements FrameworkAnalyzer { if (routing.kind === 'api') { detectedPatterns.push({ category: 'routing', name: 'API Route' }); } + if (routing.kind === 'proxy') { + detectedPatterns.push({ category: 'routing', name: 'Proxy' }); + } if (routing.hasMetadata) { detectedPatterns.push({ category: 'metadata', name: 'Next.js metadata' }); } @@ -276,21 +306,22 @@ export class NextJsAnalyzer implements FrameworkAnalyzer { function analyzeRouting(filePath: string, content: string): NextRoutingInfo { const normalizedPath = filePath.replace(/\\/g, '/'); const baseName = path.basename(normalizedPath, path.extname(normalizedPath)); - const router: NextRouter = isInAppRouter(filePath) - ? 'app' - : isInPagesRouter(filePath) - ? 'pages' - : 'unknown'; + const router: NextRouter = isProxyFile(filePath) + ? 'proxy' + : isInAppRouter(filePath) + ? 'app' + : isInPagesRouter(filePath) + ? 'pages' + : 'unknown'; let kind: NextRouteKind = 'unknown'; - if (router === 'app') { + if (router === 'proxy') { + kind = 'proxy'; + } else if (router === 'app') { const fileName = path.basename(normalizedPath); - if (fileName.startsWith('page.')) { - kind = 'page'; - } else if (fileName.startsWith('layout.')) { - kind = 'layout'; - } else if (fileName.startsWith('route.')) { - kind = 'route'; + const appFileKind = getAppRouteFileKind(fileName); + if (appFileKind) { + kind = appFileKind; } } else if (router === 'pages') { if (normalizedPath.includes('/pages/api/') || normalizedPath.includes('/src/pages/api/')) { @@ -303,9 +334,11 @@ function analyzeRouting(filePath: string, content: string): NextRoutingInfo { return { router, kind, - routePath: router === 'unknown' ? null : computeRoutePath(router, normalizedPath), + routePath: + router === 'app' || router === 'pages' ? computeRoutePath(router, normalizedPath) : null, isClientComponent: hasUseClientDirective(content), - hasMetadata: /\bexport\s+(?:const|function)\s+(metadata|generateMetadata)\b/.test(content) + hasMetadata: + /\bexport\s+(?:const|async\s+function|function)\s+(metadata|generateMetadata)\b/.test(content) }; } @@ -320,7 +353,7 @@ function isInPagesRouter(filePath: string): boolean { } function computeRoutePath( - router: Exclude, + router: Extract, normalizedFilePath: string ): string | null { const pathSegments = normalizedFilePath.split('/').filter(Boolean); @@ -361,6 +394,16 @@ function isPagesSystemFile(baseName: string): boolean { return ['_app', '_document', '_error', '_middleware'].includes(baseName); } +function getAppRouteFileKind(fileName: string): NextRouteKind | null { + const baseName = path.basename(fileName, path.extname(fileName)); + return APP_ROUTE_FILE_KINDS[baseName] || null; +} + +function isProxyFile(filePath: string): boolean { + const normalizedPath = filePath.replace(/\\/g, '/'); + return /(^|\/)(src\/)?proxy\.(?:js|jsx|ts|tsx|mjs|cjs|mts|cts)$/.test(normalizedPath); +} + async function detectRouterPresence( rootPath: string ): Promise<{ hasAppRouter: boolean; hasPagesRouter: boolean }> { diff --git a/src/analyzers/react/index.ts b/src/analyzers/react/index.ts index 317d296..2ebd29d 100644 --- a/src/analyzers/react/index.ts +++ b/src/analyzers/react/index.ts @@ -37,7 +37,11 @@ const BUILTIN_HOOKS = new Set([ 'useTransition', 'useId', 'useSyncExternalStore', - 'useInsertionEffect' + 'useInsertionEffect', + 'use', + 'useActionState', + 'useOptimistic', + 'useFormStatus' ]); const REACT_LIBRARY_SIGNALS: ReadonlyArray<{ @@ -59,6 +63,9 @@ interface ReactAstSummary { usesContext: boolean; usesMemoization: boolean; usesSuspense: boolean; + usesActionState: boolean; + usesOptimistic: boolean; + usesFormStatus: boolean; } export class ReactAnalyzer implements FrameworkAnalyzer { @@ -144,6 +151,15 @@ export class ReactAnalyzer implements FrameworkAnalyzer { if (summary.usesSuspense) { detectedPatterns.push({ category: 'reactivity', name: 'Suspense' }); } + if (summary.usesActionState) { + detectedPatterns.push({ category: 'reactivity', name: 'Actions' }); + } + if (summary.usesOptimistic) { + detectedPatterns.push({ category: 'reactivity', name: 'Optimistic UI' }); + } + if (summary.usesFormStatus) { + detectedPatterns.push({ category: 'forms', name: 'Form status' }); + } if (summary.usesMemoization) { detectedPatterns.push({ category: 'reactivity', name: 'Memoization' }); } @@ -309,6 +325,9 @@ function summarizeReactProgram(program: TSESTree.Program): ReactAstSummary { let usesContext = false; let usesMemoization = false; let usesSuspense = false; + let usesActionState = false; + let usesOptimistic = false; + let usesFormStatus = false; walkAst(program, (node, parent) => { if (node.type === 'CallExpression') { @@ -325,6 +344,18 @@ function summarizeReactProgram(program: TSESTree.Program): ReactAstSummary { if (calleeName === 'lazy') { usesSuspense = true; } + if (calleeName === 'use') { + usesSuspense = true; + } + if (calleeName === 'useActionState') { + usesActionState = true; + } + if (calleeName === 'useOptimistic') { + usesOptimistic = true; + } + if (calleeName === 'useFormStatus') { + usesFormStatus = true; + } if ( calleeName === 'createContext' && parent?.type === 'VariableDeclarator' && @@ -390,7 +421,10 @@ function summarizeReactProgram(program: TSESTree.Program): ReactAstSummary { customHooks: Array.from(customHooks).sort(), usesContext, usesMemoization, - usesSuspense + usesSuspense, + usesActionState, + usesOptimistic, + usesFormStatus }; } diff --git a/tests/nextjs-analyzer.test.ts b/tests/nextjs-analyzer.test.ts index 1882dbb..4f6d9e6 100644 --- a/tests/nextjs-analyzer.test.ts +++ b/tests/nextjs-analyzer.test.ts @@ -59,6 +59,40 @@ export default function Page() { return
; } }); }); + it('classifies current App Router file conventions and proxy files', async () => { + const analyzer = new NextJsAnalyzer(); + + const loading = await analyzer.analyze( + path.join(process.cwd(), 'app', 'dashboard', 'loading.tsx'), + 'export default function Loading() { return
; }' + ); + expect(loading.metadata.nextjs).toMatchObject({ + router: 'app', + kind: 'loading', + routePath: '/dashboard' + }); + + const notFound = await analyzer.analyze( + path.join(process.cwd(), 'src', 'app', 'shop', 'not-found.tsx'), + 'export default function NotFound() { return
; }' + ); + expect(notFound.metadata.nextjs).toMatchObject({ + router: 'app', + kind: 'not-found', + routePath: '/shop' + }); + + const proxy = await analyzer.analyze( + path.join(process.cwd(), 'src', 'proxy.ts'), + 'export function proxy() { return null; }' + ); + expect(proxy.metadata.nextjs).toMatchObject({ + router: 'proxy', + kind: 'proxy', + routePath: null + }); + }); + it('does not treat _app as a pages route and infers metadata variants from disk', async () => { const analyzer = new NextJsAnalyzer(); const appShell = await analyzer.analyze( diff --git a/tests/react-analyzer.test.ts b/tests/react-analyzer.test.ts index b3b9ef4..5728637 100644 --- a/tests/react-analyzer.test.ts +++ b/tests/react-analyzer.test.ts @@ -80,6 +80,34 @@ export class LegacyWidget extends Component { expect(patterns).toContainEqual({ category: 'styling', name: 'tailwind' }); }); + it('detects React 19 action and optimistic hooks', async () => { + const analyzer = new ReactAnalyzer(); + const filePath = path.join(process.cwd(), 'src', 'components', 'CheckoutForm.tsx'); + + const code = ` +import { use, useActionState, useOptimistic } from "react"; +import { useFormStatus } from "react-dom"; + +export function CheckoutForm({ cartPromise }: { cartPromise: Promise }) { + const cart = use(cartPromise); + const [state, submitAction] = useActionState(async () => ({ ok: true }), { ok: false }); + const [optimisticCart] = useOptimistic(cart); + const status = useFormStatus(); + + return
{state.ok && optimisticCart.length && status.pending}
; +} +`; + + const result = await analyzer.analyze(filePath, code); + const patterns = (result.metadata.detectedPatterns || []) as Array<{ category: string; name: string }>; + + expect(patterns).toContainEqual({ category: 'reactHooks', name: 'Built-in hooks' }); + expect(patterns).toContainEqual({ category: 'reactivity', name: 'Actions' }); + expect(patterns).toContainEqual({ category: 'reactivity', name: 'Optimistic UI' }); + expect(patterns).toContainEqual({ category: 'forms', name: 'Form status' }); + expect(patterns).toContainEqual({ category: 'reactivity', name: 'Suspense' }); + }); + it('does not claim React framework when react dependency is absent', async () => { const analyzer = new ReactAnalyzer(); const tempRoot = path.join(process.cwd(), 'tests', '.tmp', `react-${randomUUID()}`); From 754130dda398b463024de7ea6668c4c849e40d2e Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Sun, 5 Jul 2026 09:17:55 -0500 Subject: [PATCH 085/100] feat(analyzers): Recognize NestJS projects We work in NestJS code often enough that treating it as generic TypeScript loses useful context. Add a dedicated analyzer for controllers, modules, providers, routes, DI, and common Nest packages so backend code gets the same first-class treatment as React and Next.js. --- README.md | 2 +- docs/capabilities.md | 2 +- package.json | 5 + src/analyzers/nestjs/index.ts | 736 ++++++++++++++++++++++++++++++++ src/cli.ts | 2 + src/core/indexer.ts | 1 + src/index.ts | 2 + src/lib.ts | 7 + src/tools/search-codebase.ts | 2 +- src/types/index.ts | 3 +- tests/analyzer-registry.test.ts | 12 +- tests/indexer-metadata.test.ts | 28 ++ tests/nestjs-analyzer.test.ts | 214 ++++++++++ 13 files changed, 1010 insertions(+), 6 deletions(-) create mode 100644 src/analyzers/nestjs/index.ts create mode 100644 tests/nestjs-analyzer.test.ts diff --git a/README.md b/README.md index 92453b4..293de8e 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ If you get `selection_required`, retry with one of the paths from `availableProj ## Language Support -10 languages with full symbol extraction via Tree-sitter: TypeScript, JavaScript, Python, Java, Kotlin, C, C++, C#, Go, Rust. 30+ languages with indexing and retrieval coverage, including PHP, Ruby, Swift, Scala, Shell, and config formats. Angular, React, and Next.js have dedicated analyzers; everything else uses the Generic analyzer with AST-aligned chunking when a grammar is available. +10 languages with full symbol extraction via Tree-sitter: TypeScript, JavaScript, Python, Java, Kotlin, C, C++, C#, Go, Rust. 30+ languages with indexing and retrieval coverage, including PHP, Ruby, Swift, Scala, Shell, and config formats. Angular, React, Next.js, and NestJS have dedicated analyzers; everything else uses the Generic analyzer with AST-aligned chunking when a grammar is available. ## Configuration diff --git a/docs/capabilities.md b/docs/capabilities.md index 9085a07..7545148 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -305,6 +305,6 @@ Reproducible evaluation is shipped as a CLI entrypoint backed by shared scoring/ - **Symbol refs are not a call-graph.** `get_symbol_references` counts identifier-node occurrences in the AST (comments/strings excluded via Tree-sitter). It does not distinguish call sites from type annotations, variable assignments, or imports. Full call-site-specific analysis (`call_expression` nodes only) is a roadmap item. - **Impact is 2-hop max.** `computeImpactCandidates` walks direct importers then their importers. Full BFS reachability is on the roadmap. -- **Angular, React, and Next.js have dedicated analyzers.** All other languages go through the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). +- **Angular, React, Next.js, and NestJS have dedicated analyzers.** All other languages go through the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). - **Default embedding model is `bge-small-en-v1.5` (512-token context).** Granite (8192 context) is opt-in via `EMBEDDING_MODEL`. OpenAI is opt-in via `EMBEDDING_PROVIDER=openai` — sends code externally. - **Patterns are file-level frequency counts.** Not semantic clustering. Rising/Declining trend is derived from git commit recency for files using each pattern, not from usage semantics. diff --git a/package.json b/package.json index 3ec44d1..a1fb5a0 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,10 @@ "import": "./dist/analyzers/nextjs/index.js", "types": "./dist/analyzers/nextjs/index.d.ts" }, + "./analyzers/nestjs": { + "import": "./dist/analyzers/nestjs/index.js", + "types": "./dist/analyzers/nestjs/index.d.ts" + }, "./analyzers/generic": { "import": "./dist/analyzers/generic/index.js", "types": "./dist/analyzers/generic/index.d.ts" @@ -87,6 +91,7 @@ "windsurf", "angular", "react", + "nestjs", "typescript", "developer-tools", "static-analysis", diff --git a/src/analyzers/nestjs/index.ts b/src/analyzers/nestjs/index.ts new file mode 100644 index 0000000..5912e34 --- /dev/null +++ b/src/analyzers/nestjs/index.ts @@ -0,0 +1,736 @@ +import { promises as fs } from 'fs'; +import path from 'path'; +import { parse, type TSESTree } from '@typescript-eslint/typescript-estree'; +import type { + AnalysisResult, + ArchitecturalLayer, + CodeChunk, + CodeComponent, + CodebaseMetadata, + ExportStatement, + FrameworkAnalyzer, + ImportStatement +} from '../../types/index.js'; +import { createChunksFromCode } from '../../utils/chunking.js'; +import { categorizeDependency } from '../../utils/dependency-detection.js'; +import { + createEmptyStatistics, + isFileNotFoundError, + loadAnalyzerIndexStatistics, + normalizeAnalyzerVersion, + readAnalyzerPackageInfo +} from '../shared/metadata.js'; + +type DetectedPattern = { category: string; name: string }; +type NestComponentType = + | 'module' + | 'controller' + | 'service' + | 'repository' + | 'provider' + | 'guard' + | 'interceptor' + | 'pipe' + | 'filter' + | 'gateway' + | 'resolver' + | 'processor' + | 'schema'; + +interface NestRouteInfo { + method: string; + path: string; + handler: string; + line?: number; +} + +interface NestModuleMetadata { + imports?: string[]; + providers?: string[]; + controllers?: string[]; + exports?: string[]; +} + +interface NestClassSummary { + component: CodeComponent; + routes: NestRouteInfo[]; +} + +const ROUTE_DECORATORS: Readonly> = { + Get: 'GET', + Post: 'POST', + Put: 'PUT', + Patch: 'PATCH', + Delete: 'DELETE', + Options: 'OPTIONS', + Head: 'HEAD', + All: 'ALL' +}; + +const NEST_LIBRARY_SIGNALS: ReadonlyArray<{ + source: string; + category: string; + name: string; +}> = [ + { source: '@nestjs/swagger', category: 'documentation', name: 'Swagger' }, + { source: '@nestjs/graphql', category: 'data', name: 'GraphQL' }, + { source: '@nestjs/mongoose', category: 'data', name: 'Mongoose' }, + { source: '@nestjs/bull', category: 'queues', name: 'Bull' }, + { source: '@nestjs/bullmq', category: 'queues', name: 'BullMQ' }, + { source: '@nestjs/schedule', category: 'scheduling', name: 'Schedule' }, + { source: '@nestjs/websockets', category: 'realtime', name: 'WebSockets' }, + { source: '@nestjs/passport', category: 'security', name: 'Passport' }, + { source: '@nestjs/terminus', category: 'health', name: 'Terminus' } +]; + +const TESTING_DEPENDENCIES: ReadonlyArray = [ + ['@nestjs/testing', 'Nest Testing'], + ['vitest', 'Vitest'], + ['jest', 'Jest'] +]; + +export class NestJsAnalyzer implements FrameworkAnalyzer { + readonly name = 'nestjs'; + readonly version = '1.0.0'; + readonly supportedExtensions = ['.ts', '.js', '.mjs', '.cjs', '.mts', '.cts']; + readonly priority = 85; + + canAnalyze(filePath: string, content?: string): boolean { + const extension = path.extname(filePath).toLowerCase(); + if (!this.supportedExtensions.includes(extension) || !content) { + return false; + } + + return /\bfrom\s+['"]@nestjs\//.test(content) || /\bNestFactory\b/.test(content); + } + + async analyze(filePath: string, content: string): Promise { + const extension = path.extname(filePath).toLowerCase(); + const language = + extension === '.ts' || extension === '.mts' || extension === '.cts' + ? 'typescript' + : 'javascript'; + const relativePath = path.relative(process.cwd(), filePath); + const imports: ImportStatement[] = []; + const exports: ExportStatement[] = []; + const dependencyNames = new Set(); + const importSources = new Set(); + const detectedPatterns: DetectedPattern[] = []; + let components: CodeComponent[] = []; + + try { + const program = parse(content, { + loc: true, + range: true, + comment: true, + sourceType: 'module', + experimentalDecorators: true + }); + + for (const statement of program.body) { + if (statement.type === 'ImportDeclaration' && typeof statement.source.value === 'string') { + const source = statement.source.value; + const packageName = getPackageName(source); + importSources.add(packageName); + imports.push({ + source, + imports: statement.specifiers.map(getImportSpecifierName), + isDefault: statement.specifiers.some( + (specifier) => specifier.type === 'ImportDefaultSpecifier' + ), + isDynamic: false, + line: statement.loc?.start.line + }); + + if (!source.startsWith('.') && !source.startsWith('/')) { + dependencyNames.add(packageName); + } + } + + appendExports(exports, statement); + } + + const summaries = summarizeNestProgram(program); + components = summaries.map((summary) => summary.component); + addProgramPatterns(detectedPatterns, summaries, importSources); + } catch (error) { + console.warn(`Failed to parse NestJS file ${filePath}:`, error); + } + + const chunks = await createChunksFromCode( + content, + filePath, + relativePath, + language, + components, + { + framework: 'nestjs', + detectedPatterns + } + ); + + return { + filePath, + language, + framework: 'nestjs', + components, + imports, + exports, + dependencies: Array.from(dependencyNames) + .sort() + .map((name) => ({ + name, + category: categorizeDependency(name) + })), + metadata: { + analyzer: this.name, + detectedPatterns + }, + chunks + }; + } + + async detectCodebaseMetadata(rootPath: string): Promise { + const metadata: CodebaseMetadata = { + name: path.basename(rootPath), + rootPath, + languages: [], + dependencies: [], + architecture: { + type: 'feature-based', + layers: createEmptyStatistics().componentsByLayer, + patterns: [] + }, + styleGuides: [], + documentation: [], + projectStructure: { + type: 'single-app' + }, + statistics: createEmptyStatistics(), + customMetadata: {} + }; + + try { + const packageInfo = await readAnalyzerPackageInfo(rootPath); + metadata.name = packageInfo.projectName; + metadata.dependencies = Object.entries(packageInfo.allDependencies).map( + ([name, version]) => ({ + name, + version, + category: categorizeDependency(name) + }) + ); + + const indicators = await detectNestIndicators(rootPath, packageInfo.allDependencies); + if (indicators.includes('dep:@nestjs/core') || indicators.includes('dep:@nestjs/common')) { + metadata.framework = { + name: 'NestJS', + version: normalizeAnalyzerVersion( + packageInfo.allDependencies['@nestjs/core'] || + packageInfo.allDependencies['@nestjs/common'] + ), + type: 'nestjs', + variant: getFrameworkVariant(packageInfo.allDependencies), + testingFrameworks: detectDependencyList( + packageInfo.allDependencies, + TESTING_DEPENDENCIES + ), + indicators + }; + } + + metadata.customMetadata = { + nestjs: { + packages: Object.keys(packageInfo.allDependencies) + .filter((name) => name.startsWith('@nestjs/')) + .sort() + } + }; + } catch (error) { + if (!isFileNotFoundError(error)) { + console.warn('Failed to read NestJS project metadata:', error); + } + } + + metadata.statistics = await loadAnalyzerIndexStatistics(rootPath); + return metadata; + } + + summarize(chunk: CodeChunk): string { + const nestMetadata = + typeof chunk.metadata.nestjs === 'object' && chunk.metadata.nestjs + ? (chunk.metadata.nestjs as { type?: unknown; routes?: unknown }) + : undefined; + + if (nestMetadata?.type === 'controller' && Array.isArray(nestMetadata.routes)) { + return `NestJS controller in ${path.basename(chunk.filePath)} with ${nestMetadata.routes.length} route handlers.`; + } + if (typeof nestMetadata?.type === 'string') { + return `NestJS ${nestMetadata.type} in ${path.basename(chunk.filePath)}: lines ${chunk.startLine}-${chunk.endLine}.`; + } + + return `NestJS code in ${path.basename(chunk.filePath)}: lines ${chunk.startLine}-${chunk.endLine}.`; + } +} + +function summarizeNestProgram(program: TSESTree.Program): NestClassSummary[] { + const summaries: NestClassSummary[] = []; + + walkAst(program, (node) => { + if (node.type !== 'ClassDeclaration' || !node.id?.name) { + return; + } + + const decorators = getDecoratorNames(node); + const type = getNestComponentType(node.id.name, decorators); + if (!type) { + return; + } + + const dependencies = extractConstructorDependencies(node); + const routes = type === 'controller' ? extractRoutes(node) : []; + const moduleMetadata = type === 'module' ? extractModuleMetadata(node) : undefined; + const component: CodeComponent = { + name: node.id.name, + type: 'class', + componentType: type, + startLine: node.loc?.start.line || 1, + endLine: node.loc?.end.line || node.loc?.start.line || 1, + layer: getComponentLayer(type), + decorators: decorators.map((name) => ({ name })), + dependencies, + metadata: { + nestjs: { + type, + decorators, + routes, + dependencies, + ...(moduleMetadata ? { moduleMetadata } : {}) + } + } + }; + + summaries.push({ component, routes }); + }); + + return dedupeSummaries(summaries); +} + +function addProgramPatterns( + patterns: DetectedPattern[], + summaries: NestClassSummary[], + importSources: Set +): void { + const patternKeys = new Set(); + const addPattern = (pattern: DetectedPattern): void => { + const key = `${pattern.category}:${pattern.name}`; + if (!patternKeys.has(key)) { + patterns.push(pattern); + patternKeys.add(key); + } + }; + + for (const summary of summaries) { + const type = summary.component.componentType; + if (type === 'module') addPattern({ category: 'modules', name: 'Nest modules' }); + if (type === 'controller') addPattern({ category: 'routing', name: 'Controllers' }); + if (type === 'gateway') addPattern({ category: 'realtime', name: 'WebSockets' }); + if (type === 'resolver') addPattern({ category: 'data', name: 'GraphQL' }); + if (summary.routes.length > 0) addPattern({ category: 'routing', name: 'REST routes' }); + if ((summary.component.dependencies || []).length > 0) { + addPattern({ category: 'dependencyInjection', name: 'Constructor injection' }); + } + const decorators = getNestMetadataDecorators(summary.component); + if (decorators.includes('UseGuards')) addPattern({ category: 'security', name: 'Guards' }); + if (decorators.some((decorator) => decorator.startsWith('Api'))) { + addPattern({ category: 'documentation', name: 'Swagger' }); + } + } + + for (const signal of NEST_LIBRARY_SIGNALS) { + if (importSources.has(signal.source)) { + addPattern({ category: signal.category, name: signal.name }); + } + } +} + +function getNestComponentType( + className: string, + decorators: readonly string[] +): NestComponentType | null { + if (decorators.includes('Module')) return 'module'; + if (decorators.includes('Controller')) return 'controller'; + if (decorators.includes('WebSocketGateway')) return 'gateway'; + if (decorators.includes('Resolver')) return 'resolver'; + if (decorators.includes('Processor')) return 'processor'; + if (decorators.includes('Schema')) return 'schema'; + if (decorators.includes('Catch')) return 'filter'; + if (!decorators.includes('Injectable')) return null; + + if (/Repository$/.test(className)) return 'repository'; + if (/Guard$/.test(className)) return 'guard'; + if (/Interceptor$/.test(className)) return 'interceptor'; + if (/Pipe$/.test(className)) return 'pipe'; + if (/Filter$/.test(className)) return 'filter'; + if (/Service$/.test(className)) return 'service'; + return 'provider'; +} + +function getComponentLayer(type: NestComponentType): ArchitecturalLayer { + if (type === 'controller' || type === 'resolver') return 'presentation'; + if (type === 'service') return 'business'; + if (type === 'repository' || type === 'schema') return 'data'; + if (type === 'module') return 'feature'; + if (type === 'gateway' || type === 'processor') return 'infrastructure'; + return 'core'; +} + +function extractRoutes(node: TSESTree.ClassDeclaration): NestRouteInfo[] { + const controllerPath = normalizeRouteSegment( + getDecoratorStringArgument(getDecorator(node, 'Controller')) || '' + ); + const routes: NestRouteInfo[] = []; + + for (const member of node.body.body) { + if (member.type !== 'MethodDefinition') continue; + const methodName = getPropertyName(member.key); + if (!methodName) continue; + + for (const decorator of getDecorators(member)) { + const decoratorName = getDecoratorName(decorator); + if (!decoratorName || !ROUTE_DECORATORS[decoratorName]) continue; + const routePath = normalizeRouteSegment(getDecoratorStringArgument(decorator) || ''); + routes.push({ + method: ROUTE_DECORATORS[decoratorName], + path: joinRoutePaths(controllerPath, routePath), + handler: methodName, + line: member.loc?.start.line + }); + } + } + + return routes; +} + +function extractConstructorDependencies(node: TSESTree.ClassDeclaration): string[] { + const dependencies = new Set(); + + for (const member of node.body.body) { + if (member.type !== 'MethodDefinition' || member.kind !== 'constructor') continue; + for (const parameter of member.value.params) { + const injectToken = getInjectToken(parameter); + if (injectToken) { + dependencies.add(injectToken); + continue; + } + + const typeName = getParameterTypeName(parameter); + if (typeName) dependencies.add(typeName); + } + } + + return Array.from(dependencies).sort(); +} + +function extractModuleMetadata(node: TSESTree.ClassDeclaration): NestModuleMetadata | undefined { + const moduleDecorator = getDecorator(node, 'Module'); + if (!moduleDecorator || moduleDecorator.expression.type !== 'CallExpression') { + return undefined; + } + + const firstArgument = moduleDecorator.expression.arguments[0]; + if (!firstArgument || firstArgument.type !== 'ObjectExpression') { + return undefined; + } + + const metadata: NestModuleMetadata = {}; + for (const property of firstArgument.properties) { + if (property.type !== 'Property') continue; + const key = getPropertyName(property.key); + if (!isModuleMetadataKey(key)) continue; + const values = extractExpressionNames(property.value); + if (values.length > 0) metadata[key] = values; + } + + return Object.keys(metadata).length > 0 ? metadata : undefined; +} + +function extractExpressionNames(expression: TSESTree.Node): string[] { + if (expression.type === 'ArrayExpression') { + return expression.elements.flatMap((element) => + element && element.type !== 'SpreadElement' ? extractExpressionNames(element) : [] + ); + } + if (expression.type === 'Identifier') return [expression.name]; + if (expression.type === 'Literal') return [String(expression.value)]; + if (expression.type === 'CallExpression') return [getCalleeName(expression.callee) || 'factory']; + if (expression.type === 'MemberExpression') { + const memberName = getMemberExpressionName(expression); + return memberName ? [memberName] : []; + } + return []; +} + +function getInjectToken(parameter: TSESTree.Parameter): string | null { + const decorators = getDecorators(parameter); + const injectDecorator = decorators.find((decorator) => getDecoratorName(decorator) === 'Inject'); + if (!injectDecorator || injectDecorator.expression.type !== 'CallExpression') { + return null; + } + + const token = injectDecorator.expression.arguments[0]; + if (!token || token.type === 'SpreadElement') return null; + const names = extractExpressionNames(token); + return names[0] || null; +} + +function getParameterTypeName(parameter: TSESTree.Parameter): string | null { + const target = parameter.type === 'TSParameterProperty' ? parameter.parameter : parameter; + if (target.type !== 'Identifier') { + return null; + } + + const annotation = target.typeAnnotation?.typeAnnotation; + if (!annotation) return null; + if (annotation.type === 'TSTypeReference') return getTypeName(annotation.typeName); + return null; +} + +function getTypeName(typeName: TSESTree.EntityName): string | null { + if (typeName.type === 'Identifier') return typeName.name; + if (typeName.type === 'TSQualifiedName') { + const left = getTypeName(typeName.left); + return left ? `${left}.${typeName.right.name}` : typeName.right.name; + } + return null; +} + +function getDecorator(node: TSESTree.Node, name: string): TSESTree.Decorator | undefined { + return getDecorators(node).find((decorator) => getDecoratorName(decorator) === name); +} + +function getDecoratorNames(node: TSESTree.Node): string[] { + return getDecorators(node) + .map(getDecoratorName) + .filter((name): name is string => Boolean(name)); +} + +function getDecorators(node: TSESTree.Node): TSESTree.Decorator[] { + const candidate = node as { decorators?: unknown }; + return Array.isArray(candidate.decorators) + ? candidate.decorators.filter((decorator): decorator is TSESTree.Decorator => + isDecoratorNode(decorator) + ) + : []; +} + +function isDecoratorNode(value: unknown): value is TSESTree.Decorator { + return Boolean( + value && typeof value === 'object' && (value as { type?: unknown }).type === 'Decorator' + ); +} + +function getDecoratorName(decorator: TSESTree.Decorator): string | null { + const expression = decorator.expression; + return expression.type === 'CallExpression' + ? getCalleeName(expression.callee) + : getCalleeName(expression); +} + +function getDecoratorStringArgument(decorator: TSESTree.Decorator | undefined): string | null { + if (!decorator || decorator.expression.type !== 'CallExpression') return null; + const firstArgument = decorator.expression.arguments[0]; + if (!firstArgument || firstArgument.type === 'SpreadElement') return null; + if (firstArgument.type === 'Literal' && typeof firstArgument.value === 'string') { + return firstArgument.value; + } + if (firstArgument.type === 'TemplateLiteral' && firstArgument.expressions.length === 0) { + return firstArgument.quasis[0]?.value.cooked || null; + } + return null; +} + +function getCalleeName(node: TSESTree.Node): string | null { + if (node.type === 'Identifier') return node.name; + if (node.type === 'MemberExpression') return getMemberExpressionName(node); + return null; +} + +function getMemberExpressionName(node: TSESTree.MemberExpression): string | null { + const objectName = + node.object.type === 'Identifier' + ? node.object.name + : node.object.type === 'MemberExpression' + ? getMemberExpressionName(node.object) + : null; + const propertyName = getPropertyName(node.property); + return objectName && propertyName ? `${objectName}.${propertyName}` : propertyName; +} + +function getPropertyName(node: TSESTree.Expression | TSESTree.PrivateIdentifier): string | null { + if (node.type === 'Identifier') return node.name; + if (node.type === 'Literal') return String(node.value); + return null; +} + +function normalizeRouteSegment(segment: string): string { + return segment.replace(/^\/+|\/+$/g, '').replace(/:$/, ''); +} + +function joinRoutePaths(base: string, child: string): string { + const joined = [base, child] + .filter((segment) => segment.length > 0) + .join('/') + .replace(/\/+/g, '/'); + return `/${joined}`.replace(/\/$/, '') || '/'; +} + +function isModuleMetadataKey(key: string | null): key is keyof NestModuleMetadata { + return key === 'imports' || key === 'providers' || key === 'controllers' || key === 'exports'; +} + +function getNestMetadataDecorators(component: CodeComponent): string[] { + const metadata = component.metadata.nestjs as { decorators?: unknown } | undefined; + return Array.isArray(metadata?.decorators) + ? metadata.decorators.filter((decorator): decorator is string => typeof decorator === 'string') + : []; +} + +async function detectNestIndicators( + rootPath: string, + allDependencies: Record +): Promise { + const indicators: string[] = []; + for (const name of Object.keys(allDependencies).sort()) { + if (name.startsWith('@nestjs/')) indicators.push(`dep:${name}`); + } + + try { + await fs.stat(path.join(rootPath, 'nest-cli.json')); + indicators.push('disk:nest-cli-json'); + } catch { + // absent + } + if (await anyExists([path.join(rootPath, 'src', 'main.ts'), path.join(rootPath, 'main.ts')])) { + indicators.push('disk:main-ts'); + } + + return indicators; +} + +async function anyExists(paths: string[]): Promise { + for (const candidatePath of paths) { + try { + await fs.stat(candidatePath); + return true; + } catch { + // Continue checking remaining candidates. + } + } + return false; +} + +function getFrameworkVariant(allDependencies: Record): string { + const hasPlatform = Boolean( + allDependencies['@nestjs/platform-express'] || allDependencies['@nestjs/platform-fastify'] + ); + const hasGraphql = Boolean(allDependencies['@nestjs/graphql']); + const hasMicroservices = Boolean(allDependencies['@nestjs/microservices']); + const hasRealtime = Boolean(allDependencies['@nestjs/websockets']); + + if (hasPlatform && (hasGraphql || hasMicroservices || hasRealtime)) return 'mixed'; + if (hasPlatform) return 'http-api'; + if (hasMicroservices) return 'microservice'; + return 'library'; +} + +function detectDependencyList( + allDependencies: Record, + candidates: ReadonlyArray +): string[] { + return candidates + .filter(([dependencyName]) => Boolean(allDependencies[dependencyName])) + .map(([, label]) => label); +} + +function appendExports(exports: ExportStatement[], statement: TSESTree.Statement): void { + if (statement.type === 'ExportNamedDeclaration') { + const declaration = statement.declaration; + if (declaration?.type === 'FunctionDeclaration' && declaration.id) { + exports.push({ name: declaration.id.name, isDefault: false, type: 'function' }); + return; + } + if (declaration?.type === 'ClassDeclaration' && declaration.id) { + exports.push({ name: declaration.id.name, isDefault: false, type: 'class' }); + return; + } + if (declaration?.type === 'VariableDeclaration') { + for (const declarator of declaration.declarations) { + if (declarator.id.type === 'Identifier') { + exports.push({ name: declarator.id.name, isDefault: false, type: 'variable' }); + } + } + return; + } + for (const specifier of statement.specifiers) { + if (specifier.exported.type === 'Identifier') { + exports.push({ name: specifier.exported.name, isDefault: false, type: 're-export' }); + } + } + return; + } + + if (statement.type === 'ExportDefaultDeclaration') { + exports.push({ name: 'default', isDefault: true, type: 'default' }); + } +} + +function getImportSpecifierName(specifier: TSESTree.ImportClause): string { + if (specifier.type === 'ImportDefaultSpecifier') return 'default'; + if (specifier.type === 'ImportNamespaceSpecifier') return '*'; + return 'value' in specifier.imported ? String(specifier.imported.value) : specifier.imported.name; +} + +function getPackageName(importSource: string): string { + if (importSource.startsWith('@')) { + const [scope, name] = importSource.split('/'); + return name ? `${scope}/${name}` : importSource; + } + return importSource.split('/')[0] || importSource; +} + +function dedupeSummaries(summaries: NestClassSummary[]): NestClassSummary[] { + const seen = new Set(); + return summaries.filter((summary) => { + const key = `${summary.component.name}:${summary.component.startLine}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function walkAst( + node: TSESTree.Node, + visit: (node: TSESTree.Node, parent: TSESTree.Node | null) => void, + parent: TSESTree.Node | null = null +): void { + visit(node, parent); + for (const key of Object.keys(node) as Array) { + if (key === 'parent') continue; + const value = node[key]; + if (!value) continue; + if (Array.isArray(value)) { + for (const child of value) { + if (isAstNode(child)) walkAst(child, visit, node); + } + } else if (isAstNode(value)) { + walkAst(value, visit, node); + } + } +} + +function isAstNode(value: unknown): value is TSESTree.Node { + return Boolean( + value && typeof value === 'object' && typeof (value as { type?: unknown }).type === 'string' + ); +} diff --git a/src/cli.ts b/src/cli.ts index 1bc89f4..d2de83a 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -21,6 +21,7 @@ import type { IndexState } from './tools/types.js'; import { analyzerRegistry } from './core/analyzer-registry.js'; import { AngularAnalyzer } from './analyzers/angular/index.js'; import { NextJsAnalyzer } from './analyzers/nextjs/index.js'; +import { NestJsAnalyzer } from './analyzers/nestjs/index.js'; import { ReactAnalyzer } from './analyzers/react/index.js'; import { GenericAnalyzer } from './analyzers/generic/index.js'; import { formatJson } from './cli-formatters.js'; @@ -31,6 +32,7 @@ import { handleMapCli } from './cli-map.js'; analyzerRegistry.register(new AngularAnalyzer()); analyzerRegistry.register(new NextJsAnalyzer()); +analyzerRegistry.register(new NestJsAnalyzer()); analyzerRegistry.register(new ReactAnalyzer()); analyzerRegistry.register(new GenericAnalyzer()); diff --git a/src/core/indexer.ts b/src/core/indexer.ts index 9530576..3d04bd9 100644 --- a/src/core/indexer.ts +++ b/src/core/indexer.ts @@ -282,6 +282,7 @@ export class CodebaseIndexer { analyzers: { angular: { enabled: true, priority: 100 }, nextjs: { enabled: false, priority: 90 }, + nestjs: { enabled: false, priority: 85 }, react: { enabled: false, priority: 90 }, vue: { enabled: false, priority: 90 }, generic: { enabled: true, priority: 10 } diff --git a/src/index.ts b/src/index.ts index 8d7386b..3f82078 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,6 +17,7 @@ import { CodebaseIndexer } from './core/indexer.js'; import { analyzerRegistry } from './core/analyzer-registry.js'; import { AngularAnalyzer } from './analyzers/angular/index.js'; import { NextJsAnalyzer } from './analyzers/nextjs/index.js'; +import { NestJsAnalyzer } from './analyzers/nestjs/index.js'; import { ReactAnalyzer } from './analyzers/react/index.js'; import { GenericAnalyzer } from './analyzers/generic/index.js'; import { IndexCorruptedError } from './errors/index.js'; @@ -62,6 +63,7 @@ import { analyzerRegistry.register(new AngularAnalyzer()); analyzerRegistry.register(new NextJsAnalyzer()); +analyzerRegistry.register(new NestJsAnalyzer()); analyzerRegistry.register(new ReactAnalyzer()); analyzerRegistry.register(new GenericAnalyzer()); diff --git a/src/lib.ts b/src/lib.ts index 28f4d90..01ed318 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -12,6 +12,7 @@ * analyzerRegistry, * AngularAnalyzer, * NextJsAnalyzer, + * NestJsAnalyzer, * ReactAnalyzer, * GenericAnalyzer * } from 'codebase-context'; @@ -19,6 +20,7 @@ * // Register analyzers * analyzerRegistry.register(new AngularAnalyzer()); * analyzerRegistry.register(new NextJsAnalyzer()); + * analyzerRegistry.register(new NestJsAnalyzer()); * analyzerRegistry.register(new ReactAnalyzer()); * analyzerRegistry.register(new GenericAnalyzer()); * @@ -67,6 +69,8 @@ export { AngularAnalyzer } from './analyzers/angular/index.js'; import { AngularAnalyzer } from './analyzers/angular/index.js'; export { NextJsAnalyzer } from './analyzers/nextjs/index.js'; import { NextJsAnalyzer } from './analyzers/nextjs/index.js'; +export { NestJsAnalyzer } from './analyzers/nestjs/index.js'; +import { NestJsAnalyzer } from './analyzers/nestjs/index.js'; export { ReactAnalyzer } from './analyzers/react/index.js'; import { ReactAnalyzer } from './analyzers/react/index.js'; export { GenericAnalyzer } from './analyzers/generic/index.js'; @@ -164,6 +168,9 @@ export function createIndexer( if (!analyzerRegistry.get('nextjs')) { analyzerRegistry.register(new NextJsAnalyzer()); } + if (!analyzerRegistry.get('nestjs')) { + analyzerRegistry.register(new NestJsAnalyzer()); + } if (!analyzerRegistry.get('react')) { analyzerRegistry.register(new ReactAnalyzer()); } diff --git a/src/tools/search-codebase.ts b/src/tools/search-codebase.ts index ab2ccc4..7a4f822 100644 --- a/src/tools/search-codebase.ts +++ b/src/tools/search-codebase.ts @@ -113,7 +113,7 @@ export const definition: Tool = { properties: { framework: { type: 'string', - description: 'Filter by framework (angular, react, nextjs, vue)' + description: 'Filter by framework (angular, react, nextjs, nestjs, vue)' }, language: { type: 'string', diff --git a/src/types/index.ts b/src/types/index.ts index a792517..54309c8 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -206,7 +206,7 @@ export interface CodebaseMetadata { export interface FrameworkInfo { name: string; version: string; - type: 'angular' | 'react' | 'nextjs' | 'vue' | 'svelte' | 'solid' | 'other'; + type: 'angular' | 'react' | 'nextjs' | 'nestjs' | 'vue' | 'svelte' | 'solid' | 'other'; variant?: string; // 'standalone', 'module-based', 'class-components', etc. stateManagement?: string[]; // 'ngrx', 'redux', 'zustand', 'pinia', etc. uiLibraries?: string[]; @@ -461,6 +461,7 @@ export interface CodebaseConfig { angular?: AnalyzerConfig; react?: AnalyzerConfig; nextjs?: AnalyzerConfig; + nestjs?: AnalyzerConfig; vue?: AnalyzerConfig; generic?: AnalyzerConfig; [key: string]: AnalyzerConfig | undefined; diff --git a/tests/analyzer-registry.test.ts b/tests/analyzer-registry.test.ts index 73d0712..ae7013e 100644 --- a/tests/analyzer-registry.test.ts +++ b/tests/analyzer-registry.test.ts @@ -2,12 +2,14 @@ import { describe, it, expect, vi } from 'vitest'; import { analyzerRegistry, AnalyzerRegistry } from '../src/core/analyzer-registry'; import { AngularAnalyzer } from '../src/analyzers/angular/index'; import { NextJsAnalyzer } from '../src/analyzers/nextjs/index'; +import { NestJsAnalyzer } from '../src/analyzers/nestjs/index'; import { ReactAnalyzer } from '../src/analyzers/react/index'; import { GenericAnalyzer } from '../src/analyzers/generic/index'; // Register default analyzers analyzerRegistry.register(new AngularAnalyzer()); analyzerRegistry.register(new NextJsAnalyzer()); +analyzerRegistry.register(new NestJsAnalyzer()); analyzerRegistry.register(new ReactAnalyzer()); analyzerRegistry.register(new GenericAnalyzer()); @@ -47,12 +49,13 @@ describe('AnalyzerRegistry', () => { } }); - it('should include default analyzers (Angular, Next.js, React, Generic)', () => { + it('should include default analyzers (Angular, Next.js, NestJS, React, Generic)', () => { const analyzers = analyzerRegistry.getAll(); const names = analyzers.map((a) => a.name); expect(names).toContain('angular'); expect(names).toContain('nextjs'); + expect(names).toContain('nestjs'); expect(names).toContain('react'); expect(names).toContain('generic'); }); @@ -65,8 +68,9 @@ describe('AnalyzerRegistry', () => { expect(angular?.name).toBe('angular'); }); - it('should return nextjs and react analyzers by name', () => { + it('should return nextjs, nestjs, and react analyzers by name', () => { expect(analyzerRegistry.get('nextjs')?.name).toBe('nextjs'); + expect(analyzerRegistry.get('nestjs')?.name).toBe('nestjs'); expect(analyzerRegistry.get('react')?.name).toBe('react'); }); @@ -80,14 +84,18 @@ describe('AnalyzerRegistry', () => { it('should have Angular higher priority than Generic', () => { const angular = analyzerRegistry.get('angular'); const nextjs = analyzerRegistry.get('nextjs'); + const nestjs = analyzerRegistry.get('nestjs'); const react = analyzerRegistry.get('react'); const generic = analyzerRegistry.get('generic'); expect(angular).toBeDefined(); expect(nextjs).toBeDefined(); + expect(nestjs).toBeDefined(); expect(react).toBeDefined(); expect(generic).toBeDefined(); expect(angular!.priority).toBeGreaterThan(generic!.priority); + expect(nextjs!.priority).toBeGreaterThan(nestjs!.priority); + expect(nestjs!.priority).toBeGreaterThan(react!.priority); expect(nextjs!.priority).toBeGreaterThan(react!.priority); expect(react!.priority).toBeGreaterThan(generic!.priority); }); diff --git a/tests/indexer-metadata.test.ts b/tests/indexer-metadata.test.ts index 8705447..b4c7ade 100644 --- a/tests/indexer-metadata.test.ts +++ b/tests/indexer-metadata.test.ts @@ -6,6 +6,7 @@ import { CodebaseIndexer } from '../src/core/indexer'; import { analyzerRegistry } from '../src/core/analyzer-registry'; import { AngularAnalyzer } from '../src/analyzers/angular/index'; import { NextJsAnalyzer } from '../src/analyzers/nextjs/index'; +import { NestJsAnalyzer } from '../src/analyzers/nestjs/index'; import { ReactAnalyzer } from '../src/analyzers/react/index'; import { GenericAnalyzer } from '../src/analyzers/generic/index'; @@ -15,6 +16,9 @@ if (!analyzerRegistry.get('angular')) { if (!analyzerRegistry.get('nextjs')) { analyzerRegistry.register(new NextJsAnalyzer()); } +if (!analyzerRegistry.get('nestjs')) { + analyzerRegistry.register(new NestJsAnalyzer()); +} if (!analyzerRegistry.get('react')) { analyzerRegistry.register(new ReactAnalyzer()); } @@ -230,5 +234,29 @@ describe('CodebaseIndexer.detectMetadata', () => { expect(metadata.framework?.indicators).toContain('dep:@angular/core'); expect(metadata.framework?.indicators).toContain('disk:ng-package-json'); }); + + it('detects NestJS project metadata from @nestjs dependencies', async () => { + await fs.writeFile( + path.join(tempDir, 'package.json'), + JSON.stringify({ + name: 'api', + dependencies: { + '@nestjs/common': '11.1.19', + '@nestjs/core': '11.1.19', + '@nestjs/platform-express': '11.1.19', + }, + devDependencies: { + '@nestjs/testing': '11.1.19', + }, + }) + ); + + const indexer = new CodebaseIndexer({ rootPath: tempDir }); + const metadata = await indexer.detectMetadata(); + + expect(metadata.framework?.type).toBe('nestjs'); + expect(metadata.framework?.name).toBe('NestJS'); + expect(metadata.framework?.indicators).toContain('dep:@nestjs/core'); + }); }); }); diff --git a/tests/nestjs-analyzer.test.ts b/tests/nestjs-analyzer.test.ts new file mode 100644 index 0000000..bd08d8e --- /dev/null +++ b/tests/nestjs-analyzer.test.ts @@ -0,0 +1,214 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { promises as fs } from 'fs'; +import path from 'path'; +import { randomUUID } from 'crypto'; +import { NestJsAnalyzer } from '../src/analyzers/nestjs/index'; + +const analyzer = new NestJsAnalyzer(); +const tempRoots: string[] = []; + +afterEach(async () => { + await Promise.all( + tempRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })) + ); +}); + +describe('NestJsAnalyzer', () => { + it('detects NestJS files without claiming generic decorator files', () => { + expect( + analyzer.canAnalyze( + '/tmp/app.controller.ts', + 'import { Controller } from "@nestjs/common";\n@Controller("health") export class HealthController {}' + ) + ).toBe(true); + expect( + analyzer.canAnalyze( + '/tmp/main.ts', + 'import { NestFactory } from "@nestjs/core";\nawait NestFactory.create(AppModule);' + ) + ).toBe(true); + expect( + analyzer.canAnalyze( + '/tmp/angular.service.ts', + 'import { Injectable } from "@angular/core";\n@Injectable() export class AngularService {}' + ) + ).toBe(false); + }); + + it('extracts controller routes, DI dependencies, guards, and Swagger patterns', async () => { + const content = ` +import { Body, Controller, Delete, Get, Inject, Param, Post, UseGuards } from "@nestjs/common"; +import { AuthGuard } from "@nestjs/passport"; +import { ApiOAuth2, ApiTags } from "@nestjs/swagger"; +import { PermissionsGuard } from "./permissions.guard"; +import { PaymentsService } from "./payments.service"; + +@ApiTags("Payments") +@Controller("payments") +@UseGuards(AuthGuard("jwt"), PermissionsGuard) +export class PaymentsController { + constructor(@Inject(PaymentsService) private readonly paymentsService: PaymentsService) {} + + @Get(":id") + findOne(@Param("id") id: string) { + return this.paymentsService.findOne(id); + } + + @Post() + create(@Body() dto: CreatePaymentDto) { + return this.paymentsService.create(dto); + } + + @Delete(":id") + remove(@Param("id") id: string) { + return this.paymentsService.remove(id); + } +} +`; + + const result = await analyzer.analyze('/tmp/payments.controller.ts', content); + const controller = result.components.find( + (component) => component.name === 'PaymentsController' + ); + + expect(result.framework).toBe('nestjs'); + expect(controller).toMatchObject({ + componentType: 'controller', + layer: 'presentation', + dependencies: ['PaymentsService'] + }); + expect(controller?.metadata.nestjs).toMatchObject({ + type: 'controller', + decorators: ['ApiTags', 'Controller', 'UseGuards'], + routes: [ + { method: 'GET', path: '/payments/:id', handler: 'findOne' }, + { method: 'POST', path: '/payments', handler: 'create' }, + { method: 'DELETE', path: '/payments/:id', handler: 'remove' } + ] + }); + expect(result.metadata.detectedPatterns).toContainEqual({ + category: 'routing', + name: 'REST routes' + }); + expect(result.metadata.detectedPatterns).toContainEqual({ + category: 'security', + name: 'Guards' + }); + expect(result.metadata.detectedPatterns).toContainEqual({ + category: 'documentation', + name: 'Swagger' + }); + }); + + it('extracts modules, services, gateways, and provider metadata', async () => { + const content = ` +import { Inject, Injectable, Module } from "@nestjs/common"; +import { ConfigModule } from "@nestjs/config"; +import { WebSocketGateway } from "@nestjs/websockets"; + +@Injectable() +export class WidgetBootstrapService { + constructor(@Inject("API_CONFIG") private readonly config: ApiConfigService) {} +} + +@WebSocketGateway({ cors: true }) +export class RealtimeGateway {} + +@Module({ + imports: [ConfigModule], + providers: [WidgetBootstrapService, RealtimeGateway], + exports: [WidgetBootstrapService], +}) +export class WidgetBootstrapModule {} +`; + + const result = await analyzer.analyze('/tmp/widget-bootstrap.module.ts', content); + + expect(result.components.map((component) => component.componentType)).toEqual([ + 'service', + 'gateway', + 'module' + ]); + expect( + result.components.find((component) => component.name === 'WidgetBootstrapModule')?.metadata + .nestjs + ).toMatchObject({ + type: 'module', + moduleMetadata: { + imports: ['ConfigModule'], + providers: ['WidgetBootstrapService', 'RealtimeGateway'], + exports: ['WidgetBootstrapService'] + } + }); + expect(result.metadata.detectedPatterns).toContainEqual({ + category: 'realtime', + name: 'WebSockets' + }); + expect(result.metadata.detectedPatterns).toContainEqual({ + category: 'modules', + name: 'Nest modules' + }); + }); + + it('detects NestJS 11 metadata from package dependencies', async () => { + const root = await createTempProject({ + name: 'nestjs-api', + dependencies: { + '@nestjs/common': '11.1.19', + '@nestjs/core': '11.1.19', + '@nestjs/platform-express': '11.1.19', + '@nestjs/swagger': '11.4.2', + '@nestjs/graphql': '13.3.0', + '@nestjs/bullmq': '^11.0.4' + }, + devDependencies: { + '@nestjs/testing': '11.1.19', + vitest: '^3.0.0' + } + }); + + const metadata = await analyzer.detectCodebaseMetadata(root); + + expect(metadata.framework).toMatchObject({ + name: 'NestJS', + version: '11.1.19', + type: 'nestjs', + variant: 'mixed' + }); + expect(metadata.framework?.indicators).toEqual( + expect.arrayContaining([ + 'dep:@nestjs/common', + 'dep:@nestjs/core', + 'dep:@nestjs/platform-express' + ]) + ); + expect(metadata.framework?.testingFrameworks).toEqual(['Nest Testing', 'Vitest']); + expect(metadata.customMetadata.nestjs).toMatchObject({ + packages: expect.arrayContaining(['@nestjs/common', '@nestjs/core', '@nestjs/graphql']) + }); + }); + + it('detects older NestJS versions without version-specific assumptions', async () => { + const root = await createTempProject({ + name: 'legacy-nest-api', + dependencies: { + '@nestjs/common': '^8.4.0', + '@nestjs/core': '^8.4.0', + '@nestjs/platform-express': '^8.4.0' + } + }); + + const metadata = await analyzer.detectCodebaseMetadata(root); + + expect(metadata.framework?.type).toBe('nestjs'); + expect(metadata.framework?.version).toBe('8.4.0'); + }); +}); + +async function createTempProject(packageJson: Record): Promise { + const root = path.join(process.cwd(), 'tests', '.tmp', `nestjs-${randomUUID()}`); + tempRoots.push(root); + await fs.mkdir(root, { recursive: true }); + await fs.writeFile(path.join(root, 'package.json'), JSON.stringify(packageJson)); + return root; +} From 161241480e08ff6e9794f91da1cff28a541edd3b Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Wed, 15 Jul 2026 15:52:54 -0500 Subject: [PATCH 086/100] feat(indexing): Make chunk limits configurable per project Large repositories were silently limited to the first 5,000 searchable chunks. Let each configured project raise that safety limit while keeping 5,000 as the default for smaller codebases. Use the same setting in MCP server and CLI reindex paths so both keyword and semantic indexes cover the same files. --- README.md | 13 +++++++ src/cli.ts | 3 ++ src/core/indexer.ts | 12 ++++--- src/index.ts | 13 +++++-- src/project-state.ts | 2 ++ src/server/config.ts | 23 ++++++++++++ src/types/index.ts | 1 + tests/indexer-large-file-skip.test.ts | 25 +++++++++++++ tests/server-config.test.ts | 52 ++++++++++++++++++++++++++- 9 files changed, 136 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 293de8e..ed333e2 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,19 @@ If you get `selection_required`, retry with one of the paths from `availableProj | `CODEBASE_CONTEXT_PORT` | `3100` | HTTP server port override (same as `--port`; ignored in stdio mode) | | `CODEBASE_CONTEXT_CONFIG_PATH` | `~/.codebase-context/config.json` | Override the server config file path | +Large projects can override the default 5,000 searchable-chunk safety limit per project: + +```json +{ + "projects": [ + { + "root": "/path/to/large-project", + "parsing": { "maxChunks": 25000 } + } + ] +} +``` + ## Performance - **First indexing**: 2-5 minutes for ~30k files (embedding computation). diff --git a/src/cli.ts b/src/cli.ts index d2de83a..c7a0ed4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -29,6 +29,7 @@ import { handleMemoryCli } from './cli-memory.js'; export { handleMemoryCli } from './cli-memory.js'; import { handleInitCli } from './cli-init.js'; import { handleMapCli } from './cli-map.js'; +import { loadProjectConfig } from './server/config.js'; analyzerRegistry.register(new AngularAnalyzer()); analyzerRegistry.register(new NextJsAnalyzer()); @@ -105,6 +106,7 @@ function resolveCliRootPath(): string { async function initToolContext(): Promise { const rootPath = resolveCliRootPath(); + const projectConfig = await loadProjectConfig(rootPath); const paths = { baseDir: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME), @@ -136,6 +138,7 @@ async function initToolContext(): Promise { let lastLoggedProgress = { phase: '', percentage: -1 }; const indexer = new CodebaseIndexer({ rootPath, + ...(projectConfig?.parsing ? { config: { parsing: projectConfig.parsing } } : {}), incrementalOnly, onProgress: (progress) => { const shouldLog = diff --git a/src/core/indexer.ts b/src/core/indexer.ts index 3d04bd9..b4e68d2 100644 --- a/src/core/indexer.ts +++ b/src/core/indexer.ts @@ -57,6 +57,7 @@ import { deriveCodebaseHealth } from '../health/derive.js'; const STAGING_DIRNAME = '.staging'; const PREVIOUS_DIRNAME = '.previous'; +const DEFAULT_MAX_CHUNKS = 5000; import { computeFileHashes, @@ -302,6 +303,7 @@ export class CodebaseIndexer { respectGitignore: true, parsing: { maxFileSize: 1048576, + maxChunks: DEFAULT_MAX_CHUNKS, chunkSize: 50, chunkOverlap: 0, parseTests: true, @@ -749,13 +751,13 @@ export class CodebaseIndexer { const chunksForEmbedding = diff ? changedChunks : allChunks; // Memory safety: limit chunks to prevent embedding memory issues - const MAX_CHUNKS = 5000; + const maxChunks = this.config.parsing.maxChunks ?? DEFAULT_MAX_CHUNKS; let chunksToEmbed = chunksForEmbedding; - if (chunksForEmbedding.length > MAX_CHUNKS) { + if (chunksForEmbedding.length > maxChunks) { console.warn( - `WARNING: ${chunksForEmbedding.length} chunks exceed limit. Indexing first ${MAX_CHUNKS} chunks.` + `WARNING: ${chunksForEmbedding.length} chunks exceed limit. Indexing first ${maxChunks} chunks.` ); - chunksToEmbed = chunksForEmbedding.slice(0, MAX_CHUNKS); + chunksToEmbed = chunksForEmbedding.slice(0, maxChunks); } // Phase 3: Embedding (only changed/added chunks in incremental mode) @@ -895,7 +897,7 @@ export class CodebaseIndexer { const indexPath = path.join(activeContextDir, KEYWORD_INDEX_FILENAME); // Memory safety: cap keyword index too const keywordChunks = - allChunks.length > MAX_CHUNKS ? allChunks.slice(0, MAX_CHUNKS) : allChunks; + allChunks.length > maxChunks ? allChunks.slice(0, maxChunks) : allChunks; await fs.writeFile( indexPath, JSON.stringify({ diff --git a/src/index.ts b/src/index.ts index 3f82078..5f3e551 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1251,10 +1251,15 @@ async function performIndexingOnce( ...(project.runtimeOverrides.extraExcludePatterns?.length ? { config: { - exclude: [...EXCLUDED_GLOB_PATTERNS, ...project.runtimeOverrides.extraExcludePatterns] + exclude: [...EXCLUDED_GLOB_PATTERNS, ...project.runtimeOverrides.extraExcludePatterns], + ...(project.runtimeOverrides.maxChunks + ? { parsing: { maxChunks: project.runtimeOverrides.maxChunks } } + : {}) } } - : {}), + : project.runtimeOverrides.maxChunks + ? { config: { parsing: { maxChunks: project.runtimeOverrides.maxChunks } } } + : {}), ...(project.runtimeOverrides.preferredAnalyzer || project.runtimeOverrides.extraSourceExtensions?.length ? { @@ -1638,6 +1643,10 @@ function buildProjectRuntimeOverrides(projectConfig: ProjectConfig): ProjectRunt runtimeOverrides.extraExcludePatterns = [...projectConfig.excludePatterns]; } + if (projectConfig.parsing?.maxChunks) { + runtimeOverrides.maxChunks = projectConfig.parsing.maxChunks; + } + if (projectConfig.analyzerHints?.analyzer) { runtimeOverrides.preferredAnalyzer = projectConfig.analyzerHints.analyzer.trim(); } diff --git a/src/project-state.ts b/src/project-state.ts index 4155017..deaa5a6 100644 --- a/src/project-state.ts +++ b/src/project-state.ts @@ -14,6 +14,8 @@ import type { ToolPaths, IndexState } from './tools/types.js'; export interface ProjectRuntimeOverrides { /** Extra glob exclusion patterns merged with the default index-time exclusions. */ extraExcludePatterns?: string[]; + /** Maximum chunks retained in semantic and keyword indexes for this project. */ + maxChunks?: number; /** Analyzer name to prefer for this project without mutating global registry order. */ preferredAnalyzer?: string; /** Additional source extensions treated as code for this project only. */ diff --git a/src/server/config.ts b/src/server/config.ts index b74ed6d..ff6c1d6 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -5,6 +5,9 @@ import path from 'node:path'; export interface ProjectConfig { root: string; excludePatterns?: string[]; + parsing?: { + maxChunks?: number; + }; analyzerHints?: { extensions?: string[]; analyzer?: string; @@ -87,6 +90,20 @@ export async function loadServerConfig(): Promise { parsedProject.excludePatterns = excludePatterns; } + if ( + typeof project.parsing === 'object' && + project.parsing !== null && + !Array.isArray(project.parsing) + ) { + const parsing = project.parsing as Record; + const maxChunks = parsing.maxChunks; + if (typeof maxChunks === 'number' && Number.isInteger(maxChunks) && maxChunks > 0) { + parsedProject.parsing = { maxChunks }; + } else if (maxChunks !== undefined) { + console.error(`[config] Ignoring invalid project parsing.maxChunks: ${maxChunks}`); + } + } + if ( typeof project.analyzerHints === 'object' && project.analyzerHints !== null && @@ -137,3 +154,9 @@ export async function loadServerConfig(): Promise { return result; } + +export async function loadProjectConfig(rootPath: string): Promise { + const serverConfig = await loadServerConfig(); + const resolvedRoot = path.resolve(rootPath); + return serverConfig?.projects?.find((project) => project.root === resolvedRoot); +} diff --git a/src/types/index.ts b/src/types/index.ts index 54309c8..9010843 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -475,6 +475,7 @@ export interface CodebaseConfig { // Parsing options parsing: { maxFileSize?: number; // bytes + maxChunks?: number; chunkSize?: number; // lines chunkOverlap?: number; // lines parseTests?: boolean; diff --git a/tests/indexer-large-file-skip.test.ts b/tests/indexer-large-file-skip.test.ts index 741fb68..e54e513 100644 --- a/tests/indexer-large-file-skip.test.ts +++ b/tests/indexer-large-file-skip.test.ts @@ -57,4 +57,29 @@ describe('Indexer large file skip regression', () => { expect(indexedFiles.has('big.ts')).toBe(false); expect(indexedFiles.has('big.generated.ts')).toBe(false); }); + + it('limits the keyword index to the configured maximum chunk count', async () => { + await Promise.all( + ['one.ts', 'two.ts', 'three.ts'].map((fileName, index) => + fs.writeFile(path.join(tempDir, fileName), `export const value${index} = ${index};\n`) + ) + ); + + const indexer = new CodebaseIndexer({ + rootPath: tempDir, + config: { + skipEmbedding: true, + parsing: { maxChunks: 2 } + } + }); + + await indexer.index(); + + const indexPath = path.join(tempDir, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexRaw = JSON.parse(await fs.readFile(indexPath, 'utf-8')) as { + chunks?: unknown[]; + }; + + expect(indexRaw.chunks).toHaveLength(2); + }); }); diff --git a/tests/server-config.test.ts b/tests/server-config.test.ts index de44feb..eb7f6c6 100644 --- a/tests/server-config.test.ts +++ b/tests/server-config.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, afterEach, vi } from 'vitest'; import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { loadServerConfig } from '../src/server/config.js'; +import { loadProjectConfig, loadServerConfig } from '../src/server/config.js'; // Helper: write a temp config file and set CODEBASE_CONTEXT_CONFIG_PATH async function withTempConfig(content: string, fn: (filePath: string) => Promise) { @@ -158,6 +158,56 @@ describe('loadServerConfig', () => { }); }); + it('parses a positive per-project maxChunks value', async () => { + const config = JSON.stringify({ + projects: [{ root: '~/large-repo', parsing: { maxChunks: 25000 } }] + }); + + await withTempConfig(config, async (filePath) => { + process.env.CODEBASE_CONTEXT_CONFIG_PATH = filePath; + const result = await loadServerConfig(); + + expect(result?.projects?.[0].parsing).toEqual({ maxChunks: 25000 }); + }); + }); + + it('loads parsing configuration for the requested CLI project root', async () => { + const projectRoot = path.join(os.tmpdir(), 'ccc-large-cli-project'); + const config = JSON.stringify({ + projects: [ + { root: path.join(os.tmpdir(), 'ccc-other-project') }, + { root: projectRoot, parsing: { maxChunks: 25000 } } + ] + }); + + await withTempConfig(config, async (filePath) => { + process.env.CODEBASE_CONTEXT_CONFIG_PATH = filePath; + const projectConfig = await loadProjectConfig(projectRoot); + + expect(projectConfig).toEqual({ + root: projectRoot, + parsing: { maxChunks: 25000 } + }); + }); + }); + + it('ignores an invalid per-project maxChunks value', async () => { + const errorSpy = vi.spyOn(console, 'error'); + const config = JSON.stringify({ + projects: [{ root: '~/large-repo', parsing: { maxChunks: 0 } }] + }); + + await withTempConfig(config, async (filePath) => { + process.env.CODEBASE_CONTEXT_CONFIG_PATH = filePath; + const result = await loadServerConfig(); + + expect(result?.projects?.[0].parsing).toBeUndefined(); + expect(errorSpy).toHaveBeenCalledWith( + '[config] Ignoring invalid project parsing.maxChunks: 0' + ); + }); + }); + it('drops empty analyzerHints objects after parsing', async () => { const config = JSON.stringify({ projects: [ From 96c6d91f518820ff957e19889ca8f694a4601de2 Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Wed, 15 Jul 2026 16:01:32 -0500 Subject: [PATCH 087/100] chore(memory): Record recent repository fixes Keep the auto-extracted Git history in project memory so future searches can surface the fixes and test constraints already handled in this repository. --- .codebase-context/memory.json | 180 ++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/.codebase-context/memory.json b/.codebase-context/memory.json index f0eee24..868ef15 100644 --- a/.codebase-context/memory.json +++ b/.codebase-context/memory.json @@ -569,5 +569,185 @@ "scope": { "kind": "global" } + }, + { + "id": "cbe8bd1377ba", + "type": "gotcha", + "category": "conventions", + "memory": "fix(analyzers): Recognize current React and Next patterns", + "reason": "Auto-extracted from git commit history", + "date": "2026-07-05T07:33:25.000Z", + "source": "git" + }, + { + "id": "e3b009c312e8", + "type": "gotcha", + "category": "conventions", + "memory": "fix(reranker): Score passages with the supported tokenizer pair API", + "reason": "Auto-extracted from git commit history", + "date": "2026-07-05T14:43:29.000Z", + "source": "git" + }, + { + "id": "449b60830c9b", + "type": "gotcha", + "category": "conventions", + "memory": "fix(eval): deduplicate blocked ContextBench rows", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-30T15:45:12.000Z", + "source": "git" + }, + { + "id": "9dfaf21a0bc8", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): harden ContextBench schema cleanup", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T19:14:49.000Z", + "source": "git" + }, + { + "id": "b8776a500ef5", + "type": "gotcha", + "category": "conventions", + "memory": "fix(eval): preserve ContextBench executor model provenance", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T19:04:04.000Z", + "source": "git" + }, + { + "id": "a8b0f8664253", + "type": "gotcha", + "category": "conventions", + "memory": "fix(eval): harden ContextBench manifest checks", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T19:03:51.000Z", + "source": "git" + }, + { + "id": "f84080651a19", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): relax zombie guard timeout jitter", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T18:24:56.000Z", + "source": "git" + }, + { + "id": "3d7776af8a23", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): tolerate ContextBench runner cleanup races", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T18:21:18.000Z", + "source": "git" + }, + { + "id": "6428cb02706e", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): tolerate ContextBench schema cleanup races", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T18:13:07.000Z", + "source": "git" + }, + { + "id": "c1bd14e88de2", + "type": "gotcha", + "category": "conventions", + "memory": "fix(eval): align ContextBench harness evidence contracts", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T18:08:08.000Z", + "source": "git" + }, + { + "id": "2f3db0023789", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): relax slow Windows integration timeouts", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T17:59:20.000Z", + "source": "git" + }, + { + "id": "da039b1d0ff8", + "type": "gotcha", + "category": "conventions", + "memory": "fix(eval): harden ContextBench fixture verification", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T17:55:12.000Z", + "source": "git" + }, + { + "id": "5e3bdcaa984c", + "type": "gotcha", + "category": "conventions", + "memory": "fix(git): scope local artifact ignores", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T17:50:47.000Z", + "source": "git" + }, + { + "id": "861eb0bfb69f", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): relax slow Windows search timeouts", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T16:47:38.000Z", + "source": "git" + }, + { + "id": "3614711e00a4", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): tolerate ContextBench temp cleanup races", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T16:29:24.000Z", + "source": "git" + }, + { + "id": "c0cfb6599bcc", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): isolate ContextBench baseline Git env", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T16:25:08.000Z", + "source": "git" + }, + { + "id": "084a0540558c", + "type": "gotcha", + "category": "conventions", + "memory": "fix(format): format ContextBench harness sources", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T16:06:53.000Z", + "source": "git" + }, + { + "id": "0153743292c4", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): isolate ContextBench git fixtures", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T15:56:57.000Z", + "source": "git" + }, + { + "id": "a5397b2ee20d", + "type": "gotcha", + "category": "conventions", + "memory": "fix(cli): lazy-load mcp runtime for direct commands", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-17T14:34:22.000Z", + "source": "git" + }, + { + "id": "377e7c7c37b1", + "type": "gotcha", + "category": "conventions", + "memory": "fix(ci): unblock functional tests", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-17T20:05:19.000Z", + "source": "git" } ] \ No newline at end of file From 9d62f8c7933cc58b6556d58a1a719354b53c2aba Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Wed, 15 Jul 2026 16:11:17 -0500 Subject: [PATCH 088/100] style(indexing): Format project index overrides --- src/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 5f3e551..ab539b8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1251,7 +1251,10 @@ async function performIndexingOnce( ...(project.runtimeOverrides.extraExcludePatterns?.length ? { config: { - exclude: [...EXCLUDED_GLOB_PATTERNS, ...project.runtimeOverrides.extraExcludePatterns], + exclude: [ + ...EXCLUDED_GLOB_PATTERNS, + ...project.runtimeOverrides.extraExcludePatterns + ], ...(project.runtimeOverrides.maxChunks ? { parsing: { maxChunks: project.runtimeOverrides.maxChunks } } : {}) From 6093099e02e39d2babff58d26a4f0fff634e0ce5 Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Sun, 26 Jul 2026 00:59:52 -0500 Subject: [PATCH 089/100] fix(analyzers): Prefer NestJS provider analysis NestJS imports are a stronger framework signal than Angular's shared decorator names. Give the NestJS analyzer precedence so providers keep their Nest-specific classification and metadata. --- src/analyzers/nestjs/index.ts | 2 +- src/core/indexer.ts | 2 +- tests/analyzer-registry.test.ts | 18 +++++++++++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/analyzers/nestjs/index.ts b/src/analyzers/nestjs/index.ts index 5912e34..844194d 100644 --- a/src/analyzers/nestjs/index.ts +++ b/src/analyzers/nestjs/index.ts @@ -93,7 +93,7 @@ export class NestJsAnalyzer implements FrameworkAnalyzer { readonly name = 'nestjs'; readonly version = '1.0.0'; readonly supportedExtensions = ['.ts', '.js', '.mjs', '.cjs', '.mts', '.cts']; - readonly priority = 85; + readonly priority = 105; canAnalyze(filePath: string, content?: string): boolean { const extension = path.extname(filePath).toLowerCase(); diff --git a/src/core/indexer.ts b/src/core/indexer.ts index b4e68d2..93aebbe 100644 --- a/src/core/indexer.ts +++ b/src/core/indexer.ts @@ -283,7 +283,7 @@ export class CodebaseIndexer { analyzers: { angular: { enabled: true, priority: 100 }, nextjs: { enabled: false, priority: 90 }, - nestjs: { enabled: false, priority: 85 }, + nestjs: { enabled: false, priority: 105 }, react: { enabled: false, priority: 90 }, vue: { enabled: false, priority: 90 }, generic: { enabled: true, priority: 10 } diff --git a/tests/analyzer-registry.test.ts b/tests/analyzer-registry.test.ts index ae7013e..703ceb7 100644 --- a/tests/analyzer-registry.test.ts +++ b/tests/analyzer-registry.test.ts @@ -94,10 +94,26 @@ describe('AnalyzerRegistry', () => { expect(react).toBeDefined(); expect(generic).toBeDefined(); expect(angular!.priority).toBeGreaterThan(generic!.priority); - expect(nextjs!.priority).toBeGreaterThan(nestjs!.priority); + expect(nestjs!.priority).toBeGreaterThan(angular!.priority); expect(nestjs!.priority).toBeGreaterThan(react!.priority); expect(nextjs!.priority).toBeGreaterThan(react!.priority); expect(react!.priority).toBeGreaterThan(generic!.priority); }); + + it('should prefer NestJS when framework decorators overlap with Angular', () => { + const content = ` +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class PaymentsService {} +`; + + expect( + analyzerRegistry.findAllAnalyzers('/tmp/payments.service.ts', content).map((a) => a.name) + ).toEqual(['nestjs', 'angular', 'generic']); + expect(analyzerRegistry.findAnalyzer('/tmp/payments.service.ts', content)?.name).toBe( + 'nestjs' + ); + }); }); }); From 0154f45d0fd5053c6215f6d540b557bc7f2ccefd Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Sun, 26 Jul 2026 01:01:28 -0500 Subject: [PATCH 090/100] fix(analyzers): Tie React use metadata to imports A function named use is not necessarily React's API. Track the actual React import binding so local and unrelated use calls do not add hook or Suspense metadata. --- src/analyzers/react/index.ts | 34 ++++++++++++++++++---- tests/react-analyzer.test.ts | 56 ++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/src/analyzers/react/index.ts b/src/analyzers/react/index.ts index 2ebd29d..5838c09 100644 --- a/src/analyzers/react/index.ts +++ b/src/analyzers/react/index.ts @@ -38,7 +38,6 @@ const BUILTIN_HOOKS = new Set([ 'useId', 'useSyncExternalStore', 'useInsertionEffect', - 'use', 'useActionState', 'useOptimistic', 'useFormStatus' @@ -108,6 +107,7 @@ export class ReactAnalyzer implements FrameworkAnalyzer { const exports: ExportStatement[] = []; const dependencyNames = new Set(); const importSources = new Set(); + const reactUseBindings = new Set(); const detectedPatterns: DetectedPattern[] = []; let components: CodeComponent[] = []; @@ -124,6 +124,9 @@ export class ReactAnalyzer implements FrameworkAnalyzer { if (statement.type === 'ImportDeclaration' && typeof statement.source.value === 'string') { const source = statement.source.value; importSources.add(getPackageName(source)); + if (source === 'react') { + addReactUseBindings(reactUseBindings, statement.specifiers); + } imports.push({ source, imports: statement.specifiers.map(getImportSpecifierName), @@ -142,7 +145,7 @@ export class ReactAnalyzer implements FrameworkAnalyzer { appendExports(exports, statement); } - const summary = summarizeReactProgram(program); + const summary = summarizeReactProgram(program, reactUseBindings); components = summary.components; if (summary.usesContext) { @@ -318,7 +321,10 @@ export class ReactAnalyzer implements FrameworkAnalyzer { } } -function summarizeReactProgram(program: TSESTree.Program): ReactAstSummary { +function summarizeReactProgram( + program: TSESTree.Program, + reactUseBindings: ReadonlySet +): ReactAstSummary { const components: CodeComponent[] = []; const builtinHooksUsed = new Set(); const customHooks = new Set(); @@ -332,7 +338,8 @@ function summarizeReactProgram(program: TSESTree.Program): ReactAstSummary { walkAst(program, (node, parent) => { if (node.type === 'CallExpression') { const calleeName = getCalleeName(node.callee); - if (calleeName && BUILTIN_HOOKS.has(calleeName)) { + const usesReactUse = calleeName !== null && reactUseBindings.has(calleeName); + if (calleeName && (BUILTIN_HOOKS.has(calleeName) || usesReactUse)) { builtinHooksUsed.add(calleeName); } if (calleeName === 'createContext' || calleeName === 'useContext') { @@ -344,7 +351,7 @@ function summarizeReactProgram(program: TSESTree.Program): ReactAstSummary { if (calleeName === 'lazy') { usesSuspense = true; } - if (calleeName === 'use') { + if (usesReactUse) { usesSuspense = true; } if (calleeName === 'useActionState') { @@ -428,6 +435,23 @@ function summarizeReactProgram(program: TSESTree.Program): ReactAstSummary { }; } +function addReactUseBindings(bindings: Set, specifiers: TSESTree.ImportClause[]): void { + for (const specifier of specifiers) { + if ( + specifier.type === 'ImportSpecifier' && + getModuleExportedName(specifier.imported) === 'use' + ) { + bindings.add(specifier.local.name); + } + if ( + specifier.type === 'ImportDefaultSpecifier' || + specifier.type === 'ImportNamespaceSpecifier' + ) { + bindings.add(`${specifier.local.name}.use`); + } + } +} + function appendExports(exports: ExportStatement[], statement: TSESTree.Statement): void { if (statement.type === 'ExportNamedDeclaration') { const declaration = statement.declaration; diff --git a/tests/react-analyzer.test.ts b/tests/react-analyzer.test.ts index 5728637..2cb7939 100644 --- a/tests/react-analyzer.test.ts +++ b/tests/react-analyzer.test.ts @@ -108,6 +108,62 @@ export function CheckoutForm({ cartPromise }: { cartPromise: Promise } expect(patterns).toContainEqual({ category: 'reactivity', name: 'Suspense' }); }); + it('does not treat unrelated functions named use as React hooks or Suspense', async () => { + const analyzer = new ReactAnalyzer(); + const filePath = path.join(process.cwd(), 'src', 'components', 'UtilityWidget.tsx'); + const samples = [ + ` +function use(value: string) { + return value; +} + +export function UtilityWidget() { + return
{use("local value")}
; +} +`, + ` +import { use } from "other-library"; + +export function UtilityWidget() { + const value = use("plain value"); + return
{value}
; +} +` + ]; + + for (const code of samples) { + const result = await analyzer.analyze(filePath, code); + const patterns = (result.metadata.detectedPatterns || []) as Array<{ + category: string; + name: string; + }>; + + expect(patterns).not.toContainEqual({ category: 'reactHooks', name: 'Built-in hooks' }); + expect(patterns).not.toContainEqual({ category: 'reactivity', name: 'Suspense' }); + } + }); + + it('recognizes aliased use imports from React', async () => { + const analyzer = new ReactAnalyzer(); + const filePath = path.join(process.cwd(), 'src', 'components', 'AliasedUseWidget.tsx'); + const code = ` +import { use as readPromise } from "react"; + +export function AliasedUseWidget({ value }: { value: Promise }) { + return
{readPromise(value)}
; +} +`; + + const result = await analyzer.analyze(filePath, code); + const patterns = (result.metadata.detectedPatterns || []) as Array<{ + category: string; + name: string; + }>; + + expect(patterns).toContainEqual({ category: 'reactHooks', name: 'Built-in hooks' }); + expect(patterns).toContainEqual({ category: 'reactivity', name: 'Suspense' }); + }); + it('does not claim React framework when react dependency is absent', async () => { const analyzer = new ReactAnalyzer(); const tempRoot = path.join(process.cwd(), 'tests', '.tmp', `react-${randomUUID()}`); From 2c56b10ff32d588d9d660a586c0defee5a3d76d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:47:31 +0200 Subject: [PATCH 091/100] feat(review): add bounded review-context packets Add the first public review-context vertical slice to codebase-context: exact committed diff envelopes, bounded deterministic change signals, repository retrieval, versioned review-context-v1 output, runtime tests, documentation, and the autonomous product spec/roadmap. No reviewer-quality claims are made; the next gate is a frozen public evaluation. --- .planning/ROADMAP.md | 519 ++++++++++++++++++++ .planning/SPEC.md | 493 +++++++++++++++++++ docs/review-context.md | 125 +++++ package.json | 4 +- src/review-bin.ts | 8 + src/review-cli.ts | 401 +++++++++++++++ src/review-context.ts | 452 +++++++++++++++++ tests/review-cli-entrypoint-runtime.test.ts | 21 + tests/review-context.test.ts | 174 +++++++ 9 files changed, 2196 insertions(+), 1 deletion(-) create mode 100644 .planning/ROADMAP.md create mode 100644 .planning/SPEC.md create mode 100644 docs/review-context.md create mode 100644 src/review-bin.ts create mode 100644 src/review-cli.ts create mode 100644 src/review-context.ts create mode 100644 tests/review-cli-entrypoint-runtime.test.ts create mode 100644 tests/review-context.test.ts diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md new file mode 100644 index 0000000..2183355 --- /dev/null +++ b/.planning/ROADMAP.md @@ -0,0 +1,519 @@ +# Flagship Roadmap + +Status: **active** +Spec: `.planning/SPEC.md` +Operating rule: advance only when the current gate is explicitly PASS. + +## How To Use This Roadmap + +This file is an executable state machine, not a wishlist. + +At the start of a session: + +1. find the first phase whose gate is not `PASS`, `KILL`, or permanently `BLOCKED` +2. fix blockers inside that phase before starting anything later +3. make the smallest change needed to answer the phase question +4. run the listed verification +5. record `PASS`, `FIX`, `KILL`, or `BLOCKED` +6. only then move to the next phase + +New ideas go to **Backlog** unless they materially change the current gate. + +--- + +# Current State + +## Phase 0 — Review-context v1 functional contract + +Status: **IN PROGRESS** + +Current implementation PR: `#128 feat(review): add bounded review-context packets` + +Question: + +> Can the public project reliably compile a bounded, versioned review-context packet from an exact committed git change without an LLM? + +### Deliverables + +- [x] review-context core packet builder +- [x] executable `codebase-context-review` package binary +- [x] exact base/head commit identity +- [x] exact diff fingerprint +- [x] clean-worktree / checked-out-head invariants +- [x] rename and binary handling +- [x] NUL-safe filename parsing +- [x] bounded identifier extraction +- [x] bounded related-context retrieval +- [x] convention/pattern attachment +- [x] warnings / quality metadata +- [x] unit tests for the packet core +- [x] executable-wrapper smoke test +- [x] public review-context documentation +- [x] autonomous product spec and gated roadmap +- [ ] CI quality gate green +- [ ] CI functional test gate green +- [ ] squash merge to `master` + +### Gate G0 — Functional publishability + +PASS only if all are true: + +- [ ] `pnpm lint` has zero errors +- [ ] `pnpm format:check` passes +- [ ] `pnpm type-check` passes +- [ ] `pnpm build` passes +- [ ] full test suite passes +- [ ] executable wrapper smoke passes +- [ ] PR remains mergeable +- [ ] docs describe actual behavior and limitations +- [ ] no performance/reviewer-quality claim has been added + +On PASS: + +- squash merge PR #128 +- evidence level becomes **E1** for the review-context surface +- move immediately to Phase 1 + +On FAIL: + +- fix only the failing G0 requirement +- do not add features + +--- + +# Phase 1 — Freeze the first honest review-context evaluation + +Status: **BLOCKED BY G0** + +Question: + +> What frozen public task set can distinguish useful review context from plausible-looking retrieval noise? + +This phase freezes evidence **before** retrieval tuning. + +## Required task set + +Minimum target: + +- 20 public tasks +- at least 5 public repositories +- at least 3 language ecosystems +- mix of small and non-trivial changes +- real bug-fix / review-relevant changes, not only synthetic retrieval queries +- exact repository + base/head commit identity +- public reproducibility + +Prefer tasks where ground truth can identify: + +- changed files +- causally relevant non-changed files +- relevant symbols/spans when defensible +- actual bug/finding when the task supports reviewer evaluation + +## Freeze artifacts + +Before looking at treatment results, commit: + +- [ ] task manifest +- [ ] repo/commit identities +- [ ] task inclusion/exclusion rationale +- [ ] ground-truth format +- [ ] scorer +- [ ] retrieval limits +- [ ] setup/index accounting policy +- [ ] failure-row policy +- [ ] baseline definitions +- [ ] report schema + +## Required retrieval baselines + +At minimum: + +1. changed files/path signals only +2. repository-native/basic text navigation +3. current `review-context-v1` + +If an additional competitor is cheap and reproducible, add it **before** running treatment results. + +## Gate G1 — Eval freeze integrity + +PASS only if: + +- [ ] all tasks are frozen before treatment inspection +- [ ] task set meets diversity minimums +- [ ] scorer does not depend on treatment output +- [ ] baseline budgets are explicit +- [ ] failures stay in denominator +- [ ] no private/employer data is used +- [ ] another engineer could materialize the set from public information + +On PASS: + +- tag/fingerprint the frozen manifest +- move to Phase 2 + +On FAIL: + +- repair the evaluation design before running any claim-bearing treatment + +--- + +# Phase 2 — Prove or kill repository-context retrieval value + +Status: **BLOCKED BY G1** + +Question: + +> Does the review-context retrieval lane find materially more relevant context than simple baselines at acceptable precision and cost? + +No reviewer LLM is needed yet. First isolate retrieval quality. + +## Metrics + +Record separately: + +- relevant-file recall@k +- relevant-span recall@k where ground truth permits +- precision@k +- characters/tokens returned +- setup/index time +- retrieval wall time +- failures / abstentions +- peak memory where practical + +## Comparison discipline + +- same task set +- same frozen commits +- same k/budget where comparable +- setup/index costs reported, not hidden +- no best-of-N +- no per-task hand tuning + +## Gate G2 — Retrieval value + +### PASS + +Pass if the evidence shows a useful and reasonably general trade-off versus simple baselines. + +A PASS does not require winning every task. It requires a defensible aggregate improvement that is not explained by massively larger context budgets. + +Then: + +- evidence level becomes **E2** +- publish honest result table + failure analysis +- move to Phase 3 + +### FIX + +One bounded repair cycle is allowed if: + +- failure mode is coherent and general +- fix can be justified without task-specific branching +- frozen tasks/ground truth remain untouched + +After that repair, rerun the full frozen set once. + +### KILL + +Kill or substantially simplify this retrieval lane if, after the bounded repair: + +- simple baselines match/beat it at lower complexity/cost +- precision is too low +- gains are isolated to development repos/frameworks +- setup/index cost overwhelms value + +If killed: + +- publish the negative result honestly +- retain the deterministic git/review packet pieces that still have value +- redesign from the simplest winning baseline, not from sunk cost + +--- + +# Phase 3 — Measure actual reviewer impact + +Status: **BLOCKED BY G2 PASS** + +Question: + +> Does the bounded context packet improve real review findings when reasoning is held constant? + +## Frozen experimental arms + +### Control + +Same reviewer model + prompt + budget with raw diff/repository-native tools only. + +### Treatment + +Same reviewer model + prompt + budget plus `review-context-v1`. + +Do not give the treatment hidden extra turns/tokens unless that cost difference is explicitly part of the reported trade-off. + +## Reviewer harness requirements + +- [ ] model/provider/version recorded +- [ ] prompt recorded +- [ ] tool set recorded +- [ ] token/turn budget recorded +- [ ] timeout recorded +- [ ] repeated runs where nondeterminism matters +- [ ] append-only attempt records +- [ ] invalid output / tool failure is terminal evidence +- [ ] no best-of-N selection + +## Metrics + +Report separately: + +- real finding recall +- precision / false positives +- evidence correctness +- unsupported claims +- token/context usage +- wall time +- tool/setup failure rate + +## Gate G3 — Reviewer value + +### PASS + +Pass only if reviewer quality improves enough to justify the added context cost and the effect is not confined to one repo/language. + +Then: + +- evidence level becomes **E3** +- publish methodology + failure cases +- move to Phase 4 + +### FIX + +One bounded context-compilation repair cycle is allowed if the failure analysis clearly implicates retrieval/context rather than reviewer reasoning. + +Do not tune the reviewer prompt per task to rescue the treatment. + +### KILL + +Kill the claim that this context strategy helps review if the treatment does not produce a meaningful trade-off after the bounded repair cycle. + +Do not respond by adding agent orchestration, more models, or a dashboard. + +--- + +# Phase 4 — External reproduction and usefulness + +Status: **BLOCKED BY G3 PASS** + +Question: + +> Can someone outside the development loop reproduce the value and use the product without bespoke help? + +## Deliverables + +- [ ] stable npm release containing the review command +- [ ] concise README entrypoint +- [ ] one public end-to-end example +- [ ] reproducible evaluation command +- [ ] machine-readable result artifact +- [ ] install/runtime limitations documented +- [ ] external feedback channel + +Seek at least one of: + +- external engineer reproduces evaluation +- maintainer/user runs it on another repo +- useful issue/PR from an external user +- independent adoption/integration signal + +## Gate G4 — External evidence + +PASS when there is credible external reproduction/use rather than only owner/agent self-evaluation. + +Then evidence level becomes **E4**. + +If external users cannot successfully run or understand it, fix usability before expanding scope. + +--- + +# Phase 5 — Real PR workflow integration + +Status: **BLOCKED BY G3 PASS; PREFER G4** + +Question: + +> Is the proven context capability useful enough to integrate into real pull-request workflows? + +Only now consider: + +- GitHub Actions integration +- GitHub App +- Azure DevOps adapter +- model-agnostic reviewer consumer +- comment drafting +- review result persistence + +## Mandatory safety boundary + +The first integration must default to **read-only / draft output**. + +No autonomous public review comments until: + +- reviewer precision is measured +- false-positive policy exists +- explicit opt-in exists +- dry-run evidence exists + +## Gate G5 — Operational usefulness + +PASS requires: + +- real PR input +- reproducible context packet +- reviewer output trace +- explicit failure state +- no silent posting +- measurable latency/cost + +--- + +# Phase 6 — Optional platform/full-stack expansion + +Status: **BLOCKED BY PRODUCT EVIDENCE** + +This phase exists for product utility and engineering breadth, not portfolio theater. + +Promote only if real usage creates a need for shared state, concurrency, observability, or hosted exploration. + +Possible components: + +- TypeScript/Node API boundary +- async worker/queue for evaluations +- persistent run metadata +- Next.js benchmark/result explorer +- optional authentication +- observability +- one minimal cloud deployment path + +The default local-first CLI/MCP path must remain usable without cloud infrastructure. + +## Gate G6 — Platform justification + +Do not build this phase unless at least one is true: + +- external users need shared runs/results +- evaluations require durable async execution +- a real integration needs a service boundary +- operating evidence manually is now a material bottleneck + +A desire to demonstrate AWS/Next.js is not sufficient product justification by itself. + +--- + +# Autonomous Routing Rules + +When new information arrives, route it as follows. + +## CI or test failure + +→ current phase blocker +→ fix before all other work + +## Bug in published contract + +→ hotfix current stable surface +→ add regression test +→ return to current roadmap gate + +## Benchmark result + +→ update current evidence gate +→ choose PASS/FIX/KILL/BLOCKED +→ do not brainstorm around an unfavorable result + +## New paper/tool/competitor + +Ask: + +1. does it change a frozen baseline or invalidate current methodology? +2. does it expose a clear general failure mode? +3. is it cheap enough to add before treatment observation? + +If no → Backlog. + +## External user/maintainer interest + +→ high-priority opportunity +→ fix adoption blockers ahead of speculative features + +## New project idea + +→ default Backlog/reject +→ it must beat finishing the current gate, not merely sound interesting + +--- + +# Work-In-Progress Limits + +At any time: + +- maximum 1 active flagship product PR +- maximum 1 active claim-bearing evaluation +- maximum 1 bounded repair cycle per failed evidence gate before a kill/rethink decision + +Do not maintain parallel speculative implementations of the same hypothesis. + +--- + +# Merge Rules + +For roadmap product work: + +1. branch from current `master` +2. one primary scope per PR +3. CI must be green +4. relevant evidence gate must be satisfied +5. squash merge unless history itself is useful evidence +6. update this roadmap when a gate changes state + +Never merge because the code "looks right" while the required gate is red. + +--- + +# Backlog — Not Active + +These are intentionally **not** roadmap commitments. + +- AST-aware diff signal extraction +- richer graph expansion +- hosted dashboard +- autonomous PR commenter +- Azure DevOps review adapter +- GitHub App +- queue/worker infrastructure +- Next.js result explorer +- AWS deployment +- multi-user auth +- paid hosted embeddings +- framework-specific reviewer rules +- standalone replacement for existing reviewer products + +Promotion rule: + +> A backlog item enters Track A only when evidence or external pull shows it is the smallest next step for the current product bottleneck. + +--- + +# Immediate Next Action + +**Do not start Phase 1 yet.** + +Finish G0 on PR #128: + +1. make `src/review-context.ts` exactly Prettier-compliant +2. get quality checks green +3. get full functional tests green, including the executable wrapper smoke +4. remove any temporary verification-only workflow/artifact before merge +5. squash merge PR #128 +6. mark G0 PASS +7. only then freeze the Phase 1 evaluation diff --git a/.planning/SPEC.md b/.planning/SPEC.md new file mode 100644 index 0000000..ee6a294 --- /dev/null +++ b/.planning/SPEC.md @@ -0,0 +1,493 @@ +# Flagship Product & Autonomous Operating Spec + +Status: **active** +Owner: PatrickSys +Scope: `codebase-context` as a measurable repository-context system for AI-assisted code review and coding agents +Primary execution plan: `.planning/ROADMAP.md` + +## 1. Mission + +Build a public, local-first, vendor-neutral context system that can answer one hard question with evidence: + +> Does bounded repository context materially improve an AI reviewer's ability to understand a change and find real issues, at acceptable precision, latency, and context cost? + +The project is not successful because it has many analyzers, tools, workflows, or agent integrations. It is successful when an external engineer can reproduce the system, inspect its decisions, run the evidence harness, and see honest results. + +## 2. Product Wedge + +The flagship wedge is **review context compilation**. + +Given a clean committed git change, compile a bounded packet containing: + +1. exact base/head identity and diff fingerprint +2. changed files and change statistics +3. deterministic signals extracted from changed lines +4. bounded related repository context +5. relevant repository conventions/patterns +6. retrieval/preflight quality metadata +7. explicit warnings and limits + +That packet can then be consumed by any reviewer model or human evaluation harness. + +This separates: + +- **context compilation** from +- **review reasoning** from +- **evaluation** + +That separation is the core architectural principle for the flagship. + +## 3. Current Product Contract + +The current public review surface is `review-context-v1`. + +### Inputs + +- local git repository +- clean checked-out `HEAD` +- base git ref +- optional explicit head ref that must resolve to checked-out `HEAD` +- existing or refreshable `codebase-context` index +- explicit query/result/identifier limits + +### Outputs + +A versioned JSON packet containing: + +- exact resolved base/head SHAs +- exact git diff SHA-256 +- changed file descriptors +- addition/deletion counts +- rename/binary metadata +- bounded identifiers derived from changed lines +- bounded retrieval queries +- bounded related search results +- search quality / edit preflight metadata when available +- convention/pattern snapshot +- warnings +- all operative limits + +### Invariants + +The review path MUST: + +- use exact resolved commits for diff generation +- fail closed if the working tree is dirty +- fail closed if requested head is not checked-out `HEAD` +- parse git paths safely, including unusual valid filenames +- never silently truncate the diff used for fingerprinting +- bound queries, results, identifiers, and snippets +- never call an LLM during context compilation +- never post review comments +- never represent retrieval confidence as code correctness +- preserve failures/warnings rather than papering over them + +## 4. What This Product Is Not + +Until evidence passes the relevant gates, this project is **not**: + +- a proven AI reviewer +- a replacement for CodeRabbit, Greptile, Copilot review, or similar products +- a claim that semantic/vector retrieval improves review quality +- a cloud platform +- an autonomous pull-request commenter +- a dashboard product +- a framework-specific reviewer +- a place to port abandoned private experiments + +`context-kit` is not an authority, benchmark, source of claims, or required dependency. Its historical existence does not validate any design here. + +## 5. Evidence Levels + +Every public statement must map to one of these levels. + +### E0 — Code exists + +Evidence: + +- implementation present +- unit tests present + +Allowed claim: + +- "The project implements X." + +Not allowed: + +- "X works better" +- "X improves review" +- "X is efficient" + +### E1 — Reproducible functional behavior + +Evidence: + +- build passes +- full tests pass +- executable smoke passes +- deterministic envelope invariants verified +- public reproduction instructions exist + +Allowed claim: + +- "The project can compile a bounded review-context packet from a committed git range." + +### E2 — Retrieval value + +Evidence: + +- frozen public task set before algorithm tuning +- multiple repositories +- multiple languages/frameworks +- fixed limits +- baseline and treatment run under identical scoring +- relevant-file/span retrieval metrics reported with failures + +Allowed claim: + +- bounded claims about retrieval performance on the frozen evaluation only + +### E3 — Reviewer impact + +Evidence: + +- same reviewer model/harness +- same prompt/budget/timeout +- raw-diff/tool baseline vs review-context treatment +- repeated runs where model nondeterminism matters +- finding recall, precision/false positives, evidence quality, token/context cost, wall time reported separately +- no best-of-N selection + +Allowed claim: + +- bounded claim that review-context changed measured reviewer performance on the frozen evaluation + +### E4 — External usefulness + +Evidence can include: + +- external users +- independent reproduction +- maintainer adoption +- accepted integrations +- third-party benchmark replication + +Only at E4 should broad product-positioning claims become credible. + +## 6. Core Hypotheses + +### H1 — Changed-line signals can cheaply focus retrieval + +A small deterministic signal extractor can produce useful queries from a diff without an LLM. + +Kill/fix trigger: + +- relevant-file/span coverage is materially worse than simpler path/symbol baselines + +### H2 — Bounded repository context can outperform raw diff/repository navigation alone + +Adding a bounded `review-context-v1` packet improves review outcomes without unacceptable false positives or context cost. + +Kill/fix trigger: + +- no meaningful improvement across a frozen multi-repo review set +- improvement is explained only by substantially larger token budgets +- gains disappear outside the development repositories + +### H3 — The existing `codebase-context` retrieval stack is good enough to support the wedge + +The current search/indexing system can supply relevant review context reliably enough to justify reuse. + +Kill/fix trigger: + +- retrieval remains the dominant failure mode after a frozen diagnostic evaluation +- simpler baselines match or beat it at lower setup/runtime cost + +If H3 fails, do not build more reviewer orchestration. Fix, simplify, or replace the retrieval lane first. + +## 7. Architecture + +```text +Git repository + | + v +Exact committed change envelope +(base/head SHAs, clean tree, exact diff hash) + | + v +Deterministic change-signal extraction +(files, stats, identifiers, path fallbacks) + | + v +Bounded query planner +(max queries / identifiers) + | + v +Repository retrieval +(codebase-context search + quality/preflight metadata) + | + +--> conventions / patterns + | + v +review-context-v1 packet + | + +--> human inspection + +--> frozen evaluation harness + +--> model-agnostic reviewer consumer (only after gate) +``` + +### Architectural boundaries + +- Git/diff mechanics must remain independent of model vendors. +- Review-context packet construction must remain usable without an LLM. +- Evaluation must be able to swap reviewer models without rewriting context compilation. +- Framework-specific logic stays in analyzers. +- Core/shared types stay framework-neutral. +- No cloud infrastructure is required for the default product path. + +## 8. Metrics + +Do not collapse metrics into one vanity score. + +### Retrieval metrics + +At minimum: + +- relevant file recall@k +- relevant span recall@k when ground truth supports it +- precision@k +- context characters/tokens returned +- setup/index time +- retrieval wall time +- failure/abstention rate + +### Review metrics + +At minimum: + +- real finding recall +- false positives / precision +- evidence correctness +- unsupported-claim rate +- reviewer token/context usage +- wall time +- tool/runtime failure rate + +### Operational metrics + +Track when meaningful: + +- cold install size +- cold index time +- incremental refresh time +- peak memory +- packet size + +Performance claims require recorded measurements, not intuition. + +## 9. Evaluation Integrity + +The rules in root `AGENTS.md` remain binding. This spec adds the following review-specific requirements. + +### Freeze before tuning + +Before changing retrieval/ranking for a claim-bearing evaluation: + +1. freeze task IDs +2. freeze repositories and commits +3. freeze ground truth / scorer +4. freeze budgets and limits +5. commit the manifest +6. only then inspect treatment results + +### Required baselines + +Use the cheapest meaningful baselines first: + +1. raw diff only +2. raw diff + repository-native/basic text tools +3. simple changed-file/path/symbol context +4. current `codebase-context` retrieval packet + +Do not claim value against an intentionally weak baseline. + +### Failure accounting + +A setup failure, index failure, timeout, invalid output, missing evidence, or judge failure remains a failure row. It is never silently removed from the denominator. + +## 10. Autonomous Agent Operating Model + +An agent may manage routine progress without asking for permission for every implementation detail, but it MUST follow this state machine. + +### State A — Observe + +Read: + +1. `AGENTS.md` +2. `.planning/SPEC.md` +3. `.planning/ROADMAP.md` +4. relevant code/tests/docs +5. current PR/CI state + +Do not code before identifying the current open gate. + +### State B — Classify new information + +Every new input is one of: + +- **blocker** — CI/test/security/IP/reproducibility failure +- **evidence** — benchmark or external usage result +- **bug** — shipped behavior violates contract +- **hypothesis signal** — suggests a current hypothesis may be right/wrong +- **opportunity** — integration/user/recruiter/maintainer pull +- **idea** — unvalidated feature suggestion +- **noise** — does not materially affect current gate + +Only blockers, evidence, bugs, and strong external opportunities can pre-empt the current milestone. + +### State C — Route + +Priority order: + +1. protect privacy/IP/security/reproducibility +2. restore green build/tests +3. satisfy the current roadmap gate +4. collect evidence +5. package/document proven behavior +6. only then expand capability + +A new idea never outranks a failing gate. + +### State D — Change + +Rules: + +- one primary hypothesis or defect per PR +- smallest change capable of crossing the current gate +- prefer deleting/simplifying over layering heuristics +- no hidden benchmark-specific branches +- no fixture edits to make output pass +- no broad refactors while evidence is blocked + +### State E — Verify + +Minimum before merge: + +- lint/format +- type-check +- build +- full relevant test suite +- executable smoke for new public binaries +- docs match behavior +- no new unsupported public claim + +Evidence-changing work also requires the relevant frozen evaluation. + +### State F — Decide + +For each milestone, the agent must choose exactly one: + +- **PASS** — gate met; advance +- **FIX** — hypothesis still plausible; one bounded repair cycle justified +- **KILL** — evidence says this lane is not worth further work +- **BLOCKED** — external dependency prevents honest conclusion + +Do not use "mostly done" as a state. + +### State G — Record + +After a gate decision: + +- update `.planning/ROADMAP.md` +- update claim-bearing docs only if evidence level changed +- record material limitations +- preserve benchmark artifacts/manifests needed for reproduction + +## 11. Agent Authority + +### May do autonomously + +- create branches and PRs +- fix CI failures in current scope +- add tests for current contract +- harden parsing/error handling/reproducibility +- simplify implementation +- add frozen evaluation fixtures before implementation +- run public benchmark/eval infrastructure +- update docs to become more conservative/accurate +- revert its own failing changes +- close/kill a roadmap lane when predefined kill criteria are met + +### Must not do autonomously + +- publish employer/private code or data +- import private `context-kit` history as validation +- publish broad performance/superiority claims without evidence gate +- add paid/cloud dependencies as required defaults +- weaken frozen ground truth after seeing results +- silently increase treatment budgets relative to baseline +- add an autonomous PR-commenting bot before reviewer-impact evidence passes +- create a dashboard merely to make the project look productized +- change project licensing or ownership policy +- spend money or enable paid external services without explicit owner approval + +### Escalate to owner when + +- evidence is genuinely ambiguous after one bounded repair cycle +- a decision requires money or credentials +- a privacy/IP boundary is unclear +- two viable architectures have materially different long-term product directions +- external opportunity requires a commitment or public statement + +## 12. Scope Control + +### Track A — Current + +Only work needed to pass the next open gate in `.planning/ROADMAP.md`. + +### Track B — Backlog + +Potential future work that cannot pre-empt Track A without new evidence. + +Examples: + +- hosted dashboard +- GitHub App +- Azure DevOps adapter +- autonomous reviewer comments +- queue/worker cloud architecture +- large framework-specific rule packs +- general code-review agent orchestration + +Track B items become Track A only when the roadmap explicitly promotes them. + +## 13. Product Kill Rules + +Stop expanding the review-context wedge if any of these holds after a fair frozen evaluation and one bounded repair cycle: + +- simple baselines equal or beat it at materially lower complexity/cost +- relevant-context precision is too low to be useful +- reviewer false positives rise enough to negate recall gains +- benefits require dramatically larger token/context budgets +- gains do not generalize across repositories/languages +- setup/index cost makes realistic use unattractive + +A killed hypothesis is a successful engineering result if the evidence is credible and published honestly. + +## 14. Definition of Publishable + +A milestone is publishable only when a stranger can: + +1. understand the problem from public docs +2. install/build the project +3. run the relevant command +4. inspect the versioned output contract +5. run the evidence harness or reproduce the stated functional behavior +6. see limitations and failures +7. distinguish shipped capability from experimental hypothesis + +## 15. Current Strategic Decision + +Do **not** build a second overlapping project. + +Use `codebase-context` as the public flagship substrate. Prove or kill the review-context wedge inside it. If the existing retrieval engine fails the evidence gates, simplify or replace that layer rather than inventing another repository. diff --git a/docs/review-context.md b/docs/review-context.md new file mode 100644 index 0000000..e35df6a --- /dev/null +++ b/docs/review-context.md @@ -0,0 +1,125 @@ +# Review context + +`codebase-context-review` turns a committed git diff into a bounded review-context packet. + +It is deliberately **not** an AI reviewer. It does not call an LLM, decide whether code is correct, or post comments. Its job is narrower and testable: compile the changed surface, stable diff signals, related codebase context, and current conventions into a versioned input for a reviewer or evaluation harness. + +## Why this exists + +A PR reviewer needs more than the patch, but dumping an entire repository into a model is expensive and hard to reproduce. The review-context path creates a clean boundary between: + +1. **context compilation** — git + local repository analysis +2. **review reasoning** — any model or human reviewer consuming the packet +3. **evaluation** — measuring whether the context actually helps find the right files/spans/issues + +Keeping those layers separate makes failures diagnosable. A bad review can be attributed to missing context, bad reasoning, or both instead of hiding every variable behind one agent loop. + +## Usage + +Build and run from the repository: + +```bash +pnpm build +node dist/review-bin.js --base origin/main --head HEAD +``` + +After package publication, the package also exposes: + +```bash +npx codebase-context-review --base origin/main --head HEAD +``` + +Use `--json` for the complete machine-readable packet: + +```bash +npx codebase-context-review \ + --base origin/main \ + --head HEAD \ + --max-queries 8 \ + --max-results 3 \ + --json > review-context.json +``` + +The command uses merge-base diff semantics (`base...head`), which matches the normal pull-request question: what changed on this branch since it diverged from the base branch? + +`review-context-v1` requires `--head` to resolve to the checked-out `HEAD` and requires a clean working tree. The diff itself is then generated from the already-resolved commit SHAs rather than mutable symbolic refs. + +## Packet contract + +The current schema is `review-context-v1`. + +The packet contains: + +- exact resolved base/head commit SHAs +- SHA-256 fingerprint of Git's exact raw diff output +- changed file status, additions/deletions, rename source, and binary flag +- NUL-safe filename parsing for unusual valid git paths +- identifiers extracted deterministically from changed lines +- bounded search queries derived from those diff signals +- bounded related-context results from the existing `search_codebase` engine +- edit-preflight/search-quality metadata when the search engine provides it +- current team-pattern/convention output +- explicit warnings when context could not be produced + +Absolute local repository paths are not part of the packet contract. + +The git envelope, identifier extraction, bounds, and query derivation are deterministic. Related-context ranking is only reproducible under the same `codebase-context` version, index contents, embedding/reranking configuration, and runtime dependencies; benchmark manifests should freeze those inputs rather than pretending the entire retrieval stack is environment-independent. + +## Bounds + +Defaults: + +| Bound | Default | +| --- | ---: | +| Changed-file search queries | 8 | +| Related results per query | 3 | +| Identifier candidates per file | 10 | +| Snippet characters per related result | 1,200 | + +These are explicit because an unbounded context compiler is not useful evidence. Increasing them is allowed, but benchmark runs should record the values and compare like-for-like. + +## Index behavior + +By default, the command creates an index when one is missing and runs an incremental refresh when one already exists. This keeps normal use aligned with the checked-out clean `HEAD`. + +Pass `--no-index` to prohibit both creation and refresh. It fails if no index exists. This is useful in controlled benchmark runs where setup/index work is captured separately; the caller is then responsible for proving that the supplied index matches the frozen source state. + +The command is local-first. It invokes git and the existing local index/search pipeline; it does not introduce an LLM or external review API. + +## What this proves + +Shipping this command proves only that the project can compile a bounded, versioned review-oriented context packet from an exact committed git range. + +It **does not** prove that the packet improves review quality, catches more bugs, reduces false positives, or beats another context strategy. Those are benchmark claims and remain blocked until measured. + +## Evaluation path + +The intended evaluation is an ablation, not a marketing benchmark: + +1. freeze a set of public PR/bug-fix tasks before observing lane outputs +2. run the same reviewer/model with **raw diff only** +3. run the same reviewer/model with **raw diff + `review-context-v1` packet** +4. keep model, prompt, turn/token budget, timeout, and scoring fixed +5. score bug/finding recall and precision, evidence quality, false positives, token cost, and wall time separately +6. report setup/index failures as failures, not as competitor losses +7. do not publish a superiority claim from a one-task pilot + +The existing ContextBench protocol already follows the same evidence discipline for repository-context retrieval. Review-specific claims should meet at least the same standard rather than creating a weaker parallel benchmark. + +## Current limitations + +- Only committed git refs are supported in v1; working-tree/staged review is intentionally deferred. +- `--head` must be the checked-out commit and the worktree must be clean. +- Query generation is lexical and deterministic. It extracts identifiers from changed lines and falls back to path signals; it is not AST-aware yet. +- `--no-index` deliberately skips freshness work, so benchmark callers must attest the index/source match themselves. +- Large diffs are bounded by the CLI git-buffer limit and fail rather than silently truncating the raw fingerprint input. +- Binary files are recorded but do not generate identifier-based queries. +- The packet is context, not a verdict. A consumer should never turn `preflight.ready` into "the change is correct." + +## Next gate + +Do not add reviewer-agent features until a small frozen public evaluation can answer this first question: + +> Does `review-context-v1` improve relevant-file/span coverage at an acceptable precision and context cost compared with the same reviewer using the raw diff/repository tools alone? + +If the answer is no, fix or kill this lane before adding more orchestration. diff --git a/package.json b/package.json index a1fb5a0..2b75047 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ } }, "bin": { - "codebase-context": "./dist/index.js" + "codebase-context": "./dist/index.js", + "codebase-context-review": "./dist/review-bin.js" }, "files": [ "dist", @@ -54,6 +55,7 @@ "docs/cli.md", "docs/capabilities.md", "docs/client-setup.md", + "docs/review-context.md", "templates" ], "packageManager": "pnpm@10.27.0", diff --git a/src/review-bin.ts b/src/review-bin.ts new file mode 100644 index 0000000..1591998 --- /dev/null +++ b/src/review-bin.ts @@ -0,0 +1,8 @@ +#!/usr/bin/env node + +import { runReviewCli } from './review-cli.js'; + +runReviewCli(process.argv.slice(2)).catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/src/review-cli.ts b/src/review-cli.ts new file mode 100644 index 0000000..e4e08f7 --- /dev/null +++ b/src/review-cli.ts @@ -0,0 +1,401 @@ +#!/usr/bin/env node + +import { execFileSync } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { AngularAnalyzer } from './analyzers/angular/index.js'; +import { GenericAnalyzer } from './analyzers/generic/index.js'; +import { NestJsAnalyzer } from './analyzers/nestjs/index.js'; +import { NextJsAnalyzer } from './analyzers/nextjs/index.js'; +import { ReactAnalyzer } from './analyzers/react/index.js'; +import { + CODEBASE_CONTEXT_DIRNAME, + HEALTH_FILENAME, + INTELLIGENCE_FILENAME, + KEYWORD_INDEX_FILENAME, + MEMORY_FILENAME, + VECTOR_DB_DIRNAME +} from './constants/codebase-context.js'; +import { analyzerRegistry } from './core/analyzer-registry.js'; +import { CodebaseIndexer } from './core/indexer.js'; +import { buildReviewContextPacket, parseNameStatus } from './review-context.js'; +import { loadProjectConfig } from './server/config.js'; +import { dispatchTool } from './tools/index.js'; +import type { PatternResponse, SearchResponse, ToolContext, ToolResponse } from './tools/types.js'; + +analyzerRegistry.register(new AngularAnalyzer()); +analyzerRegistry.register(new NextJsAnalyzer()); +analyzerRegistry.register(new NestJsAnalyzer()); +analyzerRegistry.register(new ReactAnalyzer()); +analyzerRegistry.register(new GenericAnalyzer()); + +interface ReviewCliOptions { + base: string; + head: string; + rootPath: string; + maxQueries: number; + maxResults: number; + maxIdentifiers: number; + json: boolean; + noIndex: boolean; +} + +const DEFAULT_MAX_QUERIES = 8; +const DEFAULT_MAX_RESULTS = 3; +const DEFAULT_MAX_IDENTIFIERS = 10; +const MAX_GIT_BUFFER = 16 * 1024 * 1024; + +function printUsage(): void { + console.log(`codebase-context-review --base [options] + +Build a bounded review-context packet from a committed git diff. +It does not call an LLM and it does not post review comments. + +Required: + --base PR/base ref. Compared with merge-base semantics. + +Options: + --head Head ref; must resolve to checked-out HEAD (default: HEAD) + --root Repository root (default: CODEBASE_ROOT or cwd) + --max-queries Maximum changed-file search queries (default: 8) + --max-results Maximum related results per query (default: 3) + --max-identifiers Identifier candidates retained per file (default: 10) + --no-index Do not create/refresh the index; fail if it is missing + --json Emit the complete JSON packet + --help Show this help + +The v1 command requires a clean working tree so repository search and the git diff +refer to the same committed source state. + +Example: + codebase-context-review --base origin/main --head HEAD --json +`); +} + +function parsePositiveInt(value: string | undefined, flag: string, fallback: number): number { + if (value === undefined) return fallback; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${flag} must be a positive integer`); + } + return parsed; +} + +function parseArgs(argv: string[]): ReviewCliOptions | undefined { + if (argv.includes('--help') || argv.includes('-h')) { + printUsage(); + return undefined; + } + + const values = new Map(); + const booleans = new Set(); + const valueFlags = new Set([ + '--base', + '--head', + '--root', + '--max-queries', + '--max-results', + '--max-identifiers' + ]); + const booleanFlags = new Set(['--json', '--no-index']); + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (booleanFlags.has(arg)) { + booleans.add(arg); + continue; + } + if (!valueFlags.has(arg)) { + throw new Error(`Unknown argument: ${arg}`); + } + const value = argv[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error(`${arg} requires a value`); + } + values.set(arg, value); + index += 1; + } + + const base = values.get('--base'); + if (!base) throw new Error('--base is required'); + + return { + base, + head: values.get('--head') ?? 'HEAD', + rootPath: path.resolve(values.get('--root') ?? process.env.CODEBASE_ROOT ?? process.cwd()), + maxQueries: parsePositiveInt(values.get('--max-queries'), '--max-queries', DEFAULT_MAX_QUERIES), + maxResults: parsePositiveInt(values.get('--max-results'), '--max-results', DEFAULT_MAX_RESULTS), + maxIdentifiers: parsePositiveInt( + values.get('--max-identifiers'), + '--max-identifiers', + DEFAULT_MAX_IDENTIFIERS + ), + json: booleans.has('--json'), + noIndex: booleans.has('--no-index') + }; +} + +function runGit(rootPath: string, args: string[], maxBuffer = MAX_GIT_BUFFER): string { + try { + return execFileSync('git', ['-c', 'core.quotepath=false', ...args], { + cwd: rootPath, + encoding: 'utf8', + input: '', + stdio: ['pipe', 'pipe', 'pipe'], + maxBuffer + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`git ${args.join(' ')} failed: ${message}`); + } +} + +function resolveCommit(rootPath: string, ref: string): string { + const commit = runGit(rootPath, ['rev-parse', '--verify', `${ref}^{commit}`]).trim(); + if (!/^[0-9a-f]{40}$/i.test(commit)) { + throw new Error(`Could not resolve ${ref} to a commit`); + } + return commit; +} + +function assertReviewSourceState(rootPath: string, headCommit: string): void { + const checkoutHead = resolveCommit(rootPath, 'HEAD'); + if (checkoutHead !== headCommit) { + throw new Error( + `--head must resolve to the checked-out HEAD in review-context-v1 (${checkoutHead.slice(0, 12)} != ${headCommit.slice(0, 12)})` + ); + } + + const dirty = runGit(rootPath, ['status', '--porcelain=v1', '--untracked-files=all']); + if (dirty.length > 0) { + throw new Error( + 'review-context-v1 requires a clean working tree so the repository index and diff describe the same committed source state' + ); + } +} + +async function createToolContext(rootPath: string, noIndex: boolean): Promise { + const projectConfig = await loadProjectConfig(rootPath); + const paths = { + baseDir: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME), + memory: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, MEMORY_FILENAME), + intelligence: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, INTELLIGENCE_FILENAME), + health: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, HEALTH_FILENAME), + keywordIndex: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME), + vectorDb: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, VECTOR_DB_DIRNAME) + }; + + let indexExists = false; + try { + await fs.access(paths.keywordIndex); + indexExists = true; + } catch { + indexExists = false; + } + + const indexState: ToolContext['indexState'] = { + status: indexExists ? 'ready' : 'idle' + }; + + const performIndexing = async (incrementalOnly?: boolean, reason?: string): Promise => { + indexState.status = 'indexing'; + console.error( + `Indexing review target (${incrementalOnly ? 'incremental' : 'full'})${reason ? ` — ${reason}` : ''}` + ); + + try { + const indexer = new CodebaseIndexer({ + rootPath, + ...(projectConfig?.parsing ? { config: { parsing: projectConfig.parsing } } : {}), + incrementalOnly + }); + indexState.indexer = indexer; + const stats = await indexer.index(); + indexState.status = 'ready'; + indexState.lastIndexed = new Date(); + indexState.stats = stats; + } catch (error) { + indexState.status = 'error'; + indexState.error = error instanceof Error ? error.message : String(error); + throw error; + } + }; + + const context: ToolContext = { indexState, paths, rootPath, performIndexing }; + + if (!indexExists) { + if (noIndex) { + throw new Error( + 'No codebase-context index exists. Run `codebase-context reindex` or omit --no-index.' + ); + } + await performIndexing(false, 'review-context'); + } else if (!noIndex) { + await performIndexing(true, 'review-context-refresh'); + } + + return context; +} + +function parseToolJson(response: ToolResponse, operation: string): T { + const text = response.content?.find((item) => item.type === 'text')?.text; + if (!text) throw new Error(`${operation} returned no JSON payload`); + + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + throw new Error(`${operation} returned invalid JSON`); + } + + if (response.isError) { + const message = + typeof parsed === 'object' && parsed !== null && 'error' in parsed + ? String((parsed as { error: unknown }).error) + : `${operation} failed`; + throw new Error(message); + } + return parsed as T; +} + +function printHuman(packet: Awaited>): void { + console.log(`Review context: ${packet.refs.base}...${packet.refs.head}`); + console.log( + `Commits: ${packet.refs.baseCommit.slice(0, 12)} -> ${packet.refs.headCommit.slice(0, 12)}` + ); + console.log(`Diff: sha256:${packet.diffSha256}`); + console.log( + `Changed: ${packet.summary.filesChanged} files, +${packet.summary.additions}/-${packet.summary.deletions}` + ); + console.log( + `Searches: ${packet.summary.queryCount}, related results: ${packet.summary.relatedResultCount}` + ); + console.log(''); + + for (const file of packet.changedFiles) { + const rename = file.previousPath ? ` (from ${file.previousPath})` : ''; + const identifiers = file.identifiers.slice(0, 5).join(', '); + console.log( + `${file.rawStatus.padEnd(4)} ${file.path}${rename} +${file.additions}/-${file.deletions}${file.binary ? ' [binary]' : ''}` + ); + if (identifiers) console.log(` signals: ${identifiers}`); + } + + if (packet.searches.length > 0) { + console.log('\nRelated context'); + for (const search of packet.searches) { + console.log(`- ${search.query} <- ${search.sourceFiles.join(', ')}`); + if (search.error) { + console.log(` error: ${search.error}`); + continue; + } + for (const result of search.results) { + console.log(` ${result.file} score=${result.score.toFixed(3)} ${result.summary}`); + } + } + } + + if (packet.warnings.length > 0) { + console.log('\nWarnings'); + for (const warning of packet.warnings) console.log(`- ${warning}`); + } + + console.log('\nUse --json for the machine-readable packet including conventions and snippets.'); +} + +export async function runReviewCli(argv: string[]): Promise { + const options = parseArgs(argv); + if (!options) return 0; + + const gitRoot = runGit(options.rootPath, ['rev-parse', '--show-toplevel']).trim(); + const rootPath = path.resolve(gitRoot); + const baseCommit = resolveCommit(rootPath, options.base); + const headCommit = resolveCommit(rootPath, options.head); + assertReviewSourceState(rootPath, headCommit); + + const range = `${baseCommit}...${headCommit}`; + const nameStatus = runGit(rootPath, [ + 'diff', + '--name-status', + '-z', + '--find-renames', + range, + '--' + ]); + const changedFiles = parseNameStatus(nameStatus); + const rawDiff = runGit(rootPath, [ + 'diff', + '--no-ext-diff', + '--binary', + '--no-color', + '--find-renames', + range, + '--' + ]); + + const patchesByPath = new Map(); + for (const file of changedFiles) { + const patch = runGit(rootPath, [ + 'diff', + '--no-ext-diff', + '--unified=0', + '--no-color', + '--find-renames', + range, + '--', + file.path + ]); + patchesByPath.set(file.path, patch); + } + + const toolContext = await createToolContext(rootPath, options.noIndex); + const packet = await buildReviewContextPacket({ + base: options.base, + head: options.head, + baseCommit, + headCommit, + rawDiff, + changedFiles, + patchesByPath, + maxQueries: options.maxQueries, + maxResultsPerQuery: options.maxResults, + maxIdentifiersPerFile: options.maxIdentifiers, + search: async (query, limit) => + parseToolJson( + await dispatchTool( + 'search_codebase', + { query, includeSnippets: true, intent: 'edit', limit }, + toolContext + ), + `search_codebase(${query})` + ), + loadConventions: async () => + parseToolJson( + await dispatchTool('get_team_patterns', { category: 'all' }, toolContext), + 'get_team_patterns' + ) + }); + + if (options.json) console.log(JSON.stringify(packet, null, 2)); + else printHuman(packet); + + return 0; +} + +const isDirectRun = (() => { + const entry = process.argv[1]; + if (!entry) return false; + try { + return path.resolve(entry) === path.resolve(fileURLToPath(import.meta.url)); + } catch { + return entry.replace(/\\/g, '/').endsWith('/review-cli.js'); + } +})(); + +if (isDirectRun) { + runReviewCli(process.argv.slice(2)).catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/src/review-context.ts b/src/review-context.ts new file mode 100644 index 0000000..f3b76a7 --- /dev/null +++ b/src/review-context.ts @@ -0,0 +1,452 @@ +import { createHash } from 'node:crypto'; +import path from 'node:path'; +import type { + PatternResponse, + SearchQuality, + SearchResponse, + SearchResultItem +} from './tools/types.js'; + +const IDENTIFIER_RE = /[A-Za-z_$][A-Za-z0-9_$]*/g; + +const IDENTIFIER_STOPWORDS = new Set([ + 'abstract', + 'any', + 'array', + 'async', + 'await', + 'boolean', + 'break', + 'case', + 'catch', + 'class', + 'const', + 'continue', + 'debugger', + 'declare', + 'default', + 'delete', + 'else', + 'enum', + 'export', + 'extends', + 'false', + 'finally', + 'for', + 'from', + 'func', + 'function', + 'implements', + 'import', + 'interface', + 'let', + 'map', + 'namespace', + 'never', + 'new', + 'null', + 'number', + 'object', + 'package', + 'private', + 'protected', + 'public', + 'readonly', + 'return', + 'self', + 'static', + 'string', + 'struct', + 'super', + 'switch', + 'this', + 'throw', + 'true', + 'try', + 'type', + 'typeof', + 'undefined', + 'unknown', + 'using', + 'var', + 'void', + 'while', + 'with', + 'yield' +]); + +export interface ChangedFileDescriptor { + path: string; + previousPath?: string; + status: string; + rawStatus: string; +} + +export interface ChangedFileContext extends ChangedFileDescriptor { + additions: number; + deletions: number; + binary: boolean; + identifiers: string[]; +} + +export interface ReviewQuery { + query: string; + sourceFiles: string[]; + identifiers: string[]; +} + +export interface ReviewRelatedResult { + file: string; + summary: string; + score: number; + relevanceReason?: string; + type?: string; + trend?: 'Rising' | 'Declining'; + patternWarning?: string; + symbol?: string; + scope?: string; + signaturePreview?: string; + snippet?: string; + importedByCount?: number; + hasTests?: boolean; +} + +export interface ReviewSearchResult { + query: string; + sourceFiles: string[]; + searchQuality?: SearchQuality; + preflight?: SearchResponse['preflight']; + results: ReviewRelatedResult[]; + error?: string; +} + +export interface ReviewContextPacket { + schemaVersion: 'review-context-v1'; + refs: { + base: string; + head: string; + baseCommit: string; + headCommit: string; + range: string; + }; + diffSha256: string; + limits: { + maxQueries: number; + maxResultsPerQuery: number; + maxIdentifiersPerFile: number; + }; + summary: { + filesChanged: number; + additions: number; + deletions: number; + queryCount: number; + relatedResultCount: number; + }; + changedFiles: ChangedFileContext[]; + searches: ReviewSearchResult[]; + conventions?: PatternResponse; + warnings: string[]; +} + +export interface PatchStats { + additions: number; + deletions: number; + binary: boolean; +} + +export interface BuildReviewContextOptions { + base: string; + head: string; + baseCommit: string; + headCommit: string; + rawDiff: string; + changedFiles: ChangedFileDescriptor[]; + patchesByPath: ReadonlyMap; + maxQueries?: number; + maxResultsPerQuery?: number; + maxIdentifiersPerFile?: number; + search: (query: string, limit: number) => Promise; + loadConventions?: () => Promise; +} + +function descriptorFromStatus(rawStatus: string, filePath: string): ChangedFileDescriptor { + return { + path: filePath, + status: rawStatus.charAt(0), + rawStatus + }; +} + +export function parseNameStatus(output: string): ChangedFileDescriptor[] { + if (output.includes('\0')) { + const fields = output.split('\0'); + const files: ChangedFileDescriptor[] = []; + let index = 0; + + while (index < fields.length) { + const rawStatus = fields[index++]; + if (!rawStatus) continue; + + const status = rawStatus.charAt(0); + if (status === 'R' || status === 'C') { + const previousPath = fields[index++]; + const currentPath = fields[index++]; + if (previousPath && currentPath) { + files.push({ path: currentPath, previousPath, status, rawStatus }); + } + continue; + } + + const filePath = fields[index++]; + if (filePath) files.push(descriptorFromStatus(rawStatus, filePath)); + } + + return files; + } + + const files: ChangedFileDescriptor[] = []; + for (const rawLine of output.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + + const parts = line.split('\t'); + const rawStatus = parts[0] ?? ''; + const status = rawStatus.charAt(0); + if (!status) continue; + + if ((status === 'R' || status === 'C') && parts.length >= 3) { + const previousPath = parts[1]; + const currentPath = parts[2]; + if (previousPath && currentPath) { + files.push({ path: currentPath, previousPath, status, rawStatus }); + } + continue; + } + + const filePath = parts[1]; + if (filePath) files.push(descriptorFromStatus(rawStatus, filePath)); + } + + return files; +} + +export function inspectPatch(patch: string): PatchStats { + let additions = 0; + let deletions = 0; + let binary = false; + + for (const line of patch.split(/\r?\n/)) { + if (line.startsWith('Binary files ') || line === 'GIT binary patch') { + binary = true; + continue; + } + if (line.startsWith('+++') || line.startsWith('---')) continue; + if (line.startsWith('+')) additions += 1; + else if (line.startsWith('-')) deletions += 1; + } + + return { additions, deletions, binary }; +} + +export function extractChangedIdentifiers(patch: string, limit = 10): string[] { + const scores = new Map(); + + for (const line of patch.split(/\r?\n/)) { + const isAddition = line.startsWith('+') && !line.startsWith('+++'); + const isDeletion = line.startsWith('-') && !line.startsWith('---'); + if (!isAddition && !isDeletion) continue; + + for (const match of line.slice(1).matchAll(IDENTIFIER_RE)) { + const identifier = match[0]; + if (identifier.length < 3) continue; + if (IDENTIFIER_STOPWORDS.has(identifier.toLowerCase())) continue; + + let score = isAddition ? 3 : 2; + if (/^[A-Z][A-Za-z0-9_$]+$/.test(identifier)) score += 2; + if (/[a-z][A-Z]/.test(identifier)) score += 1; + if (identifier.length >= 8) score += 1; + + scores.set(identifier, (scores.get(identifier) ?? 0) + score); + } + } + + return Array.from(scores.entries()) + .sort(([nameA, scoreA], [nameB, scoreB]) => scoreB - scoreA || nameA.localeCompare(nameB)) + .slice(0, Math.max(0, limit)) + .map(([identifier]) => identifier); +} + +function fallbackIdentifiersForPath(filePath: string): string[] { + const withoutExtension = filePath.replace(/\.[^./]+$/, ''); + const basename = path.basename(withoutExtension); + const parentSegments = path + .dirname(withoutExtension) + .split(/[\\/._-]+/) + .filter(Boolean) + .reverse(); + const ordered = [basename, ...parentSegments]; + const seen = new Set(); + + return ordered.filter((part) => { + const normalized = part.toLowerCase(); + if (part.length < 3 || IDENTIFIER_STOPWORDS.has(normalized) || seen.has(normalized)) { + return false; + } + seen.add(normalized); + return true; + }); +} + +export function buildReviewQueries(files: ChangedFileContext[], maxQueries = 8): ReviewQuery[] { + const selected = [...files] + .sort((a, b) => { + const identifierDelta = b.identifiers.length - a.identifiers.length; + if (identifierDelta !== 0) return identifierDelta; + + const changeDelta = b.additions + b.deletions - (a.additions + a.deletions); + if (changeDelta !== 0) return changeDelta; + + return a.path.localeCompare(b.path); + }) + .slice(0, Math.max(0, maxQueries)); + + const queries: ReviewQuery[] = []; + const seen = new Set(); + + for (const file of selected) { + const identifiers = ( + file.identifiers.length > 0 ? file.identifiers : fallbackIdentifiersForPath(file.path) + ).slice(0, 3); + if (identifiers.length === 0) continue; + + const query = identifiers.join(' '); + const key = query.toLowerCase(); + const existing = queries.find((entry) => entry.query.toLowerCase() === key); + if (existing) { + if (!existing.sourceFiles.includes(file.path)) existing.sourceFiles.push(file.path); + continue; + } + if (seen.has(key)) continue; + + seen.add(key); + queries.push({ query, sourceFiles: [file.path], identifiers }); + } + + return queries; +} + +export function fingerprintDiff(rawDiff: string): string { + return createHash('sha256').update(rawDiff, 'utf8').digest('hex'); +} + +function compactSearchResult(result: SearchResultItem): ReviewRelatedResult { + const importedByCount = result.relationships?.importedByCount ?? result.importedByCount; + + return { + file: result.file, + summary: result.summary, + score: result.score, + ...(result.relevanceReason ? { relevanceReason: result.relevanceReason } : {}), + ...(result.type ? { type: result.type } : {}), + ...(result.trend ? { trend: result.trend } : {}), + ...(result.patternWarning ? { patternWarning: result.patternWarning } : {}), + ...(result.symbol ? { symbol: result.symbol } : {}), + ...(result.scope ? { scope: result.scope } : {}), + ...(result.signaturePreview ? { signaturePreview: result.signaturePreview } : {}), + ...(result.snippet ? { snippet: result.snippet.slice(0, 1200) } : {}), + ...(importedByCount != null ? { importedByCount } : {}), + ...(result.relationships?.hasTests != null ? { hasTests: result.relationships.hasTests } : {}) + }; +} + +export async function buildReviewContextPacket( + options: BuildReviewContextOptions +): Promise { + const maxQueries = Math.max(1, Math.floor(options.maxQueries ?? 8)); + const maxResultsPerQuery = Math.max(1, Math.floor(options.maxResultsPerQuery ?? 3)); + const maxIdentifiersPerFile = Math.max(1, Math.floor(options.maxIdentifiersPerFile ?? 10)); + const warnings: string[] = []; + + const changedFiles = options.changedFiles.map((file) => { + const patch = options.patchesByPath.get(file.path) ?? ''; + if (!patch) warnings.push(`No textual patch captured for ${file.path}`); + + const stats = inspectPatch(patch); + return { + ...file, + ...stats, + identifiers: stats.binary ? [] : extractChangedIdentifiers(patch, maxIdentifiersPerFile) + }; + }); + + const queries = buildReviewQueries(changedFiles, maxQueries); + if (queries.length === 0 && changedFiles.length > 0) { + warnings.push('No review search queries could be derived from the changed diff'); + } + if (changedFiles.length > maxQueries) { + warnings.push( + `Search was bounded to ${maxQueries} changed-file queries for ${changedFiles.length} changed files` + ); + } + + const searches: ReviewSearchResult[] = []; + for (const reviewQuery of queries) { + try { + const response = await options.search(reviewQuery.query, maxResultsPerQuery); + searches.push({ + query: reviewQuery.query, + sourceFiles: reviewQuery.sourceFiles, + searchQuality: response.searchQuality, + ...(response.preflight ? { preflight: response.preflight } : {}), + results: response.results.slice(0, maxResultsPerQuery).map(compactSearchResult) + }); + } catch (error) { + searches.push({ + query: reviewQuery.query, + sourceFiles: reviewQuery.sourceFiles, + results: [], + error: error instanceof Error ? error.message : String(error) + }); + warnings.push(`Search failed for query: ${reviewQuery.query}`); + } + } + + let conventions: PatternResponse | undefined; + if (options.loadConventions) { + try { + conventions = await options.loadConventions(); + } catch (error) { + warnings.push( + `Convention lookup failed: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + const additions = changedFiles.reduce((sum, file) => sum + file.additions, 0); + const deletions = changedFiles.reduce((sum, file) => sum + file.deletions, 0); + const relatedResultCount = searches.reduce((sum, search) => sum + search.results.length, 0); + + return { + schemaVersion: 'review-context-v1', + refs: { + base: options.base, + head: options.head, + baseCommit: options.baseCommit, + headCommit: options.headCommit, + range: `${options.base}...${options.head}` + }, + diffSha256: fingerprintDiff(options.rawDiff), + limits: { maxQueries, maxResultsPerQuery, maxIdentifiersPerFile }, + summary: { + filesChanged: changedFiles.length, + additions, + deletions, + queryCount: searches.length, + relatedResultCount + }, + changedFiles, + searches, + ...(conventions ? { conventions } : {}), + warnings + }; +} diff --git a/tests/review-cli-entrypoint-runtime.test.ts b/tests/review-cli-entrypoint-runtime.test.ts new file mode 100644 index 0000000..1e3c131 --- /dev/null +++ b/tests/review-cli-entrypoint-runtime.test.ts @@ -0,0 +1,21 @@ +import { spawnSync } from 'node:child_process'; +import { resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(import.meta.dirname, '..'); +const entrypoint = resolve(root, 'src', 'review-bin.ts'); + +describe('review CLI entrypoint runtime', () => { + it('executes the package wrapper and prints help without loading a review target', () => { + const result = spawnSync(process.execPath, ['--import', 'tsx', entrypoint, '--help'], { + cwd: root, + encoding: 'utf8', + timeout: 30_000 + }); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(result.stdout).toContain('codebase-context-review --base '); + expect(result.stdout).toContain('It does not call an LLM'); + }, 30_000); +}); diff --git a/tests/review-context.test.ts b/tests/review-context.test.ts new file mode 100644 index 0000000..52dca41 --- /dev/null +++ b/tests/review-context.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from 'vitest'; +import { + buildReviewContextPacket, + buildReviewQueries, + extractChangedIdentifiers, + fingerprintDiff, + inspectPatch, + parseNameStatus, + type ChangedFileContext +} from '../src/review-context.js'; +import type { SearchResponse } from '../src/tools/types.js'; + +describe('review context', () => { + it('parses modified, added, deleted and renamed files', () => { + expect( + parseNameStatus(['M\tsrc/a.ts', 'A\tsrc/b.ts', 'D\tsrc/c.ts', 'R100\tsrc/old.ts\tsrc/new.ts'].join('\n')) + ).toEqual([ + { path: 'src/a.ts', status: 'M', rawStatus: 'M' }, + { path: 'src/b.ts', status: 'A', rawStatus: 'A' }, + { path: 'src/c.ts', status: 'D', rawStatus: 'D' }, + { + path: 'src/new.ts', + previousPath: 'src/old.ts', + status: 'R', + rawStatus: 'R100' + } + ]); + }); + + it('parses NUL-delimited git output without corrupting unusual paths', () => { + expect(parseNameStatus('M\0src/tab\tname.ts\0R100\0old\nname.ts\0new name.ts\0')).toEqual([ + { path: 'src/tab\tname.ts', status: 'M', rawStatus: 'M' }, + { + path: 'new name.ts', + previousPath: 'old\nname.ts', + status: 'R', + rawStatus: 'R100' + } + ]); + }); + + it('counts textual changes without treating diff headers as source', () => { + const patch = [ + 'diff --git a/src/auth.ts b/src/auth.ts', + '--- a/src/auth.ts', + '+++ b/src/auth.ts', + '@@ -1,2 +1,3 @@', + '-const oldToken = loadToken();', + '+const refreshToken = loadRefreshToken();', + '+return refreshToken;' + ].join('\n'); + + expect(inspectPatch(patch)).toEqual({ additions: 2, deletions: 1, binary: false }); + }); + + it('detects binary patches', () => { + expect(inspectPatch('diff --git a/a.png b/a.png\nBinary files a/a.png and b/a.png differ')).toEqual({ + additions: 0, + deletions: 0, + binary: true + }); + }); + + it('ranks semantic identifiers from changed lines and drops language noise', () => { + const patch = [ + '-const token = auth.load();', + '+const refreshToken = sessionService.loadRefreshToken();', + '+return sessionService.rotateRefreshToken(refreshToken);', + '+const SessionService = createSessionService();' + ].join('\n'); + + const identifiers = extractChangedIdentifiers(patch, 8); + + expect(identifiers).toContain('sessionService'); + expect(identifiers).toContain('refreshToken'); + expect(identifiers).toContain('rotateRefreshToken'); + expect(identifiers).not.toContain('const'); + expect(identifiers).not.toContain('return'); + }); + + it('bounds review queries and falls back to file path signals', () => { + const files: ChangedFileContext[] = [ + { + path: 'src/auth/session-service.ts', + status: 'M', + rawStatus: 'M', + additions: 10, + deletions: 2, + binary: false, + identifiers: ['SessionService', 'refreshToken', 'rotateToken'] + }, + { + path: 'src/http/request-guard.ts', + status: 'M', + rawStatus: 'M', + additions: 2, + deletions: 1, + binary: false, + identifiers: [] + } + ]; + + expect(buildReviewQueries(files, 1)).toEqual([ + { + query: 'SessionService refreshToken rotateToken', + sourceFiles: ['src/auth/session-service.ts'], + identifiers: ['SessionService', 'refreshToken', 'rotateToken'] + } + ]); + + expect(buildReviewQueries(files, 2)[1]).toEqual({ + query: 'request-guard http src', + sourceFiles: ['src/http/request-guard.ts'], + identifiers: ['request-guard', 'http', 'src'] + }); + }); + + it('fingerprints exact diff bytes, including the final newline', () => { + expect(fingerprintDiff('diff body\n')).not.toBe(fingerprintDiff('diff body')); + }); + + it('builds a bounded packet and keeps failed searches explicit', async () => { + const rawDiff = [ + 'diff --git a/src/auth.ts b/src/auth.ts', + '--- a/src/auth.ts', + '+++ b/src/auth.ts', + '@@ -1 +1 @@', + '-oldSession();', + '+rotateSessionToken();' + ].join('\n'); + + const searchCalls: string[] = []; + const search = async (query: string): Promise => { + searchCalls.push(query); + return { + status: 'ok', + searchQuality: { status: 'ok', confidence: 0.91 }, + results: [ + { + file: 'src/session/store.ts:10-24', + summary: 'Session token storage', + score: 0.88, + snippet: 'export function rotateSessionToken() { return true; }' + } + ] + }; + }; + + const packet = await buildReviewContextPacket({ + base: 'origin/main', + head: 'HEAD', + baseCommit: 'a'.repeat(40), + headCommit: 'b'.repeat(40), + rawDiff, + changedFiles: [{ path: 'src/auth.ts', status: 'M', rawStatus: 'M' }], + patchesByPath: new Map([['src/auth.ts', rawDiff]]), + maxQueries: 2, + maxResultsPerQuery: 1, + search + }); + + expect(packet.schemaVersion).toBe('review-context-v1'); + expect(packet.diffSha256).toBe(fingerprintDiff(rawDiff)); + expect(packet.summary).toEqual({ + filesChanged: 1, + additions: 1, + deletions: 1, + queryCount: 1, + relatedResultCount: 1 + }); + expect(searchCalls).toEqual(['rotateSessionToken oldSession']); + expect(packet.searches[0]?.results[0]?.file).toBe('src/session/store.ts:10-24'); + }); +}); From cc54fb5aee50ec177402e459b23be0fe8da7836e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:51:10 +0200 Subject: [PATCH 092/100] docs: mark review-context G0 passed Record the completed functional publishability gate, evidence level E1, and Phase 1 as the active benchmark-freeze work. --- .planning/ROADMAP.md | 66 +++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 2183355..4b114b2 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -25,14 +25,17 @@ New ideas go to **Backlog** unless they materially change the current gate. ## Phase 0 — Review-context v1 functional contract -Status: **IN PROGRESS** +Status: **PASS — E1** -Current implementation PR: `#128 feat(review): add bounded review-context packets` +Merged implementation: PR `#128 feat(review): add bounded review-context packets` +Squash commit: `2c56b10ff32d588d9d660a586c0defee5a3d76d1` Question: > Can the public project reliably compile a bounded, versioned review-context packet from an exact committed git change without an LLM? +Answer at E1: **yes, functionally**. This does not imply retrieval or reviewer-quality value. + ### Deliverables - [x] review-context core packet builder @@ -50,40 +53,37 @@ Question: - [x] executable-wrapper smoke test - [x] public review-context documentation - [x] autonomous product spec and gated roadmap -- [ ] CI quality gate green -- [ ] CI functional test gate green -- [ ] squash merge to `master` +- [x] CI quality gate green +- [x] CI functional test gate green +- [x] squash merge to `master` ### Gate G0 — Functional publishability -PASS only if all are true: +PASS evidence: -- [ ] `pnpm lint` has zero errors -- [ ] `pnpm format:check` passes -- [ ] `pnpm type-check` passes -- [ ] `pnpm build` passes -- [ ] full test suite passes -- [ ] executable wrapper smoke passes -- [ ] PR remains mergeable -- [ ] docs describe actual behavior and limitations -- [ ] no performance/reviewer-quality claim has been added +- [x] `pnpm lint` has zero errors +- [x] `pnpm format:check` passes +- [x] `pnpm type-check` passes +- [x] `pnpm build` passes +- [x] full test suite passes +- [x] executable wrapper smoke passes +- [x] PR was mergeable +- [x] docs describe actual behavior and limitations +- [x] no performance/reviewer-quality claim was added -On PASS: +CI evidence: workflow run `32585647263` — Quality Checks **success**, Functional Tests **success**. -- squash merge PR #128 -- evidence level becomes **E1** for the review-context surface -- move immediately to Phase 1 +Result: -On FAIL: - -- fix only the failing G0 requirement -- do not add features +- Gate G0: **PASS** +- Evidence level: **E1** +- Phase 1 is now the only active flagship phase --- # Phase 1 — Freeze the first honest review-context evaluation -Status: **BLOCKED BY G0** +Status: **ACTIVE** Question: @@ -506,14 +506,12 @@ Promotion rule: # Immediate Next Action -**Do not start Phase 1 yet.** - -Finish G0 on PR #128: +**Phase 1 only: freeze the evaluation before tuning anything.** -1. make `src/review-context.ts` exactly Prettier-compliant -2. get quality checks green -3. get full functional tests green, including the executable wrapper smoke -4. remove any temporary verification-only workflow/artifact before merge -5. squash merge PR #128 -6. mark G0 PASS -7. only then freeze the Phase 1 evaluation +1. source candidate public review/bug-fix tasks +2. select at least 20 tasks across at least 5 repos and 3 language ecosystems +3. define ground truth and inclusion/exclusion rules without treatment output +4. define the simple baselines and equalized budgets +5. commit the manifest, scorer, limits, failure policy, and report schema +6. fingerprint the frozen set +7. only after G1 PASS may any claim-bearing treatment run begin From 10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:06:55 +0200 Subject: [PATCH 093/100] Publish historical benchmark and released setup docs (#130) * docs: publish benchmark evidence and released setup guidance * fix(docs): preserve package links and evidence contracts --- README.md | 278 ++-- docs/benchmark-prior-public-report.md | 182 +++ docs/benchmark.md | 190 +-- docs/capabilities.md | 56 +- docs/cli.md | 57 +- docs/client-setup.md | 375 +++-- docs/comparison-table.md | 49 +- docs/demo.md | 29 +- results/benchmark-presentation-evidence.json | 1251 +++++++++++++++++ ...contextbench-engineering-note-summary.json | 213 +++ tests/mcp-client-templates.test.ts | 61 +- tests/proof-truth-surfaces.test.ts | 56 +- 12 files changed, 2232 insertions(+), 565 deletions(-) create mode 100644 docs/benchmark-prior-public-report.md create mode 100644 results/benchmark-presentation-evidence.json create mode 100644 results/contextbench-engineering-note-summary.json diff --git a/README.md b/README.md index ed333e2..6249c02 100644 --- a/README.md +++ b/README.md @@ -1,251 +1,163 @@ -# codebase-context - -## Map your team's conventions before your AI agent starts searching. +# Codebase Context [![npm version](https://img.shields.io/npm/v/codebase-context)](https://www.npmjs.com/package/codebase-context) [![license](https://img.shields.io/npm/l/codebase-context)](./LICENSE) [![node](https://img.shields.io/node/v/codebase-context)](https://github.com/PatrickSys/codebase-context/blob/master/package.json) -You're tired of AI agents writing code that "just works" but still misses how your team actually builds things. They search too broadly, pick generic examples, and spend tokens exploring before they understand the shape of the repo. - -`codebase-context` changes the first step. Start with a bounded conventions map that shows the architecture, dominant patterns, and strongest local examples. Then search for the exact file, symbol, or workflow you need. - -Here's what codebase-context does: - -**Starts with a bounded conventions map** - The first call shows architecture layers, active patterns, golden files, and next calls without dumping vendored repos, fixtures, generated output, or oversized entrypoint lists into the default surface. - -**Finds the right local example** - Search does not just return code. Each result comes back with pattern signals, file relationships, and quality indicators so the agent can move from the map to the most relevant local example instead of wandering through raw hits. - -**Knows what is current** - Conventions are detected from your code and git history, not only from rules you wrote. The map distinguishes what is common from what is rising or declining, and points at the files that best represent the current direction. - -**Adds support signals when you need them** - Team memory and edit-readiness checks stay available, but as supporting context after the map and search have already narrowed the work. - -Map first, search second, local-first throughout. Your code never leaves your machine by default. - -See the [current discovery benchmark](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark.md) for the checked-in discovery-only proof. The gate is still `pending_evidence`, and `claimAllowed` remains `false`. +## Your coding agent doesn't understand your codebase. -### What it looks like +Coding agents can read files, but they still have to discover how your repository is organized, which patterns your team follows, and which examples are worth copying. -Real CLI output against `angular-spotify`, the repo used for the launch screenshots. +Codebase Context gives an agent a local view of that information through code search, team patterns, strong examples, and project memory. It runs as an MCP server - a local tool that your editor or command-line agent can call while it works - and keeps the index on your machine by default. -**Lead signal: pattern drift and golden files** +## Set up your AI client -![codebase-context patterns screenshot](https://raw.githubusercontent.com/PatrickSys/codebase-context/master/docs/assets/patterns.png) - -This is the part most tools miss: what the team is doing now, what it is moving away from, and which files are the strongest examples to follow. - -**Before editing: preflight and impact** - -![codebase-context search preflight screenshot](https://raw.githubusercontent.com/PatrickSys/codebase-context/master/docs/assets/search-query.png) - -When the agent searches with edit intent, it gets a compact decision card: confidence, whether it's safe to proceed, which patterns apply, the best example, and which files are likely to be affected. - -More CLI examples in [`docs/cli.md`](./docs/cli.md). Full walkthrough: [demo.md on GitHub](https://github.com/PatrickSys/codebase-context/blob/master/docs/demo.md). - -## Quick Start +Choose your coding tool and run its command once. Use Node.js 22 or newer. These commands use published npm `2.2.0` and do not require a project folder in your configuration: ```bash -claude mcp add codebase-context -- npx -y codebase-context -``` - -The server runs in two modes. Use stdio unless you need multiple clients connected at once: +# Claude Code +claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.2.0 -| Mode | How it runs | When to use | -| ------------------- | ------------------------------------------------ | ------------------------------------------ | -| **stdio** (default) | Process spawned by the client | One AI client talking to one or more repos | -| **HTTP** | Long-lived server at `http://127.0.0.1:3100/mcp` | Multiple clients sharing one server | +# Codex CLI +codex mcp add codebase-context -- npx -y codebase-context@2.2.0 -Client support at a glance: - -| Client | stdio | HTTP | -| ----------------- | ----- | -------------------------------------------- | -| Claude Code | Yes | No (stdio only) | -| Claude Desktop | Yes | No | -| Cursor | Yes | Yes — `.cursor/mcp.json` with `type: "http"` | -| Windsurf | Yes | Not yet | -| Codex | Yes | Yes — `--mcp-config` flag | -| VS Code (Copilot) | Yes | No | -| OpenCode | Yes | Not documented yet | - -Copy-pasteable templates: [`templates/mcp/stdio/.mcp.json`](./templates/mcp/stdio/.mcp.json) and [`templates/mcp/http/.mcp.json`](./templates/mcp/http/.mcp.json). +# OpenCode 1.x (keep the quoted separator on Windows) +opencode mcp add codebase-context '--' npx -y codebase-context@2.2.0 +``` -Full per-client setup, HTTP server instructions, and local build testing: [`docs/client-setup.md`](./docs/client-setup.md). +Start a new agent session in your project, then ask: -## First Use +> Use Codebase Context to find [feature] in this repository. Pass this repository's absolute path as project when checking get_indexing_status and searching. Wait for indexing if needed, read codebase://context, then search_codebase and open a returned source file. Show me the relevant files. -Get a conventions map of your codebase before exploring or editing: +Replace `[feature]` with something you want to find. The agent supplies the repository path in its tool calls, so the registration can serve different projects. Initial indexing may need a local model download. The October 6 isolated checks proved these client registrations and published-package project selection/search; they did not establish a full native agent investigation. -```bash -# See your codebase conventions — architecture layers, patterns, golden files -npx -y codebase-context map +For Codex Desktop, create or merge `.codex/config.toml` in the project you want to search: -# Then search for what you need -npx -y codebase-context search --query "auth middleware" +```toml +[mcp_servers.codebase-context] +command = "npx" +args = ["-y", "codebase-context@2.2.0"] +startup_timeout_sec = 120 ``` -Your AI agent uses the same map via the `codebase://context` MCP resource on first call. +Trust the project if asked, restart Codex, and start a new task there. Use the prompt above. Config placement alone does not establish successful first use; this no-folder recipe has not been accepted in a fresh native Desktop task. -## Common First Commands +Other clients use their own setup commands: -Three commands to understand a repo before you edit it: +| Client | Shortest current setup | +| --------------------------- | ---------------------------------------------------------------------------- | +| Gemini CLI | `gemini mcp add --scope user codebase-context npx -y codebase-context@2.2.0` | +| Cursor | Add `.cursor/mcp.json` | +| VS Code with GitHub Copilot | Add `.vscode/mcp.json` | +| GitHub Copilot CLI | `copilot mcp add codebase-context -- npx -y codebase-context@2.2.0` | +| Windsurf | Add `~/.codeium/windsurf/mcp_config.json` | -```bash -# What are the main conventions and best examples? -npx -y codebase-context map - -# Then search for the local example you need -npx -y codebase-context search --query "auth middleware" +Check an existing same-name entry before replacing it. To give the server a default folder, append that folder's absolute path to the `npx` arguments. The [client setup guide](./docs/client-setup.md) covers scopes, optional fixed-folder configuration, verification limits and the unreleased installer. Published `2.2.0`'s interactive `init` has registration bugs; use the commands above. -# What patterns is the team actually using right now? -npx -y codebase-context patterns -``` +The [client setup guide](./docs/client-setup.md) has the exact commands and config for every client, plus what was checked locally and what still relies on official instructions. -This is also what your AI agent consumes automatically via MCP tools; the CLI is the human-readable version of the same map-plus-search flow. +The default connection is `stdio` (standard input/output): your client starts the server when it needs it. HTTP is an advanced, client-dependent option; verify the setup guide and client support before relying on it. -## What it does +## What your agent gets -### The Search Tool (`search_codebase`) +### Relevant code -One call returns ranked results with `file`, `summary`, `score`, compact type (`componentType:layer`), pattern trend signals, relationship hints, related team memories, a search quality assessment, and a preflight decision card when `intent="edit"`. The decision card shows `ready` (boolean), `nextAction` when not ready, `patterns` (do/avoid), `bestExample`, impact coverage (`"3/5 callers in results"`), and `whatWouldHelp`. +`search_codebase` ranks files and symbols for the task instead of returning an unstructured dump. The agent can ask for a compact result first, then read the code it needs. -Default output is lean — if the agent wants code, it calls `read_file`. Add `includeSnippets: true` for inline code with scope headers (e.g. `// AuthService.getToken()`). +### Team patterns and examples -See [`docs/capabilities.md`](./docs/capabilities.md) for the full field reference. +`get_team_patterns` shows the approaches used in the repository and points to representative files. Published `2.2.0` includes dedicated analyzers for Angular, React and Next.js, with a generic analyzer for other stacks. NestJS support belongs to the newer source candidate. -### Patterns & Conventions (`get_team_patterns`) +### Project memory -Detects what your team actually does by analyzing the codebase: adoption percentages for DI, state management, testing, and library patterns; trend direction (Rising / Stable / Declining) from git recency; golden files ranked by modern pattern density; conflicts when two approaches both exceed 20%. +`remember` stores a convention, decision, gotcha, or past failure for the project. `get_memory` retrieves relevant entries in later sessions, including when the agent or editor changes. -### Team Memory (`remember` + `get_memory`) +## How it works -Record a decision once. It surfaces automatically in search results and preflight cards from then on. Conventional commits (`refactor:`, `migrate:`, `fix:`, `revert:`) from the last 90 days auto-extract into memory during indexing — no setup required. +1. **Index locally.** Codebase Context scans the project, builds a keyword index, and creates local semantic embeddings - numeric representations used to match code by meaning as well as exact words. +2. **Understand the repository.** The agent can request a compact codebase map with structure, patterns, and representative files. +3. **Find the code for the task.** Search returns ranked files and symbols; the agent reads the selected files before editing. -Memory types: `convention`, `decision`, `gotcha`, `failure`. Confidence decay: conventions never decay, decisions 180-day half-life, gotchas/failures 90-day. Stale memories get flagged instead of blindly trusted. +The same information is available from the terminal. Run these commands from your project root. The first index can take a while because it scans the project and creates local embeddings; once it is ready, inspect the map and search for the code you need: -## Tools +```bash +# Build or refresh the local index +npx -y codebase-context@2.2.0 reindex -| Tool | What it does | -| ------------------------------ | ----------------------------------------------------- | -| `search_codebase` | Hybrid search + decision card when `intent="edit"` | -| `get_team_patterns` | Pattern frequencies, golden files, conflict detection | -| `get_symbol_references` | Concrete references to a symbol (count + snippets) | -| `remember` | Record a convention, decision, gotcha, or failure | -| `get_memory` | Query team memory with confidence decay scoring | -| `get_codebase_metadata` | Project structure, frameworks, dependencies | -| `get_style_guide` | Style guide rules for the current project | -| `detect_circular_dependencies` | Import cycles between files | -| `refresh_index` | Full or incremental re-index + git memory extraction | -| `get_indexing_status` | Progress and stats for the current index | +# Repository structure, patterns, and representative files +npx -y codebase-context@2.2.0 map -## Multi-project +# Ranked code search +npx -y codebase-context@2.2.0 search --query "auth middleware" -One server, multiple repos. Three cases: +# Current team patterns +npx -y codebase-context@2.2.0 patterns +``` -| Case | What happens | -| --------------------------------------------- | --------------------------------------------------- | -| One project | Routing is automatic | -| Multiple projects, active project already set | Routes to the active project | -| Multiple projects, ambiguous | Returns `selection_required` — retry with `project` | +One stdio server can route across several repositories. Supply `project` in tool calls to select the intended repository; a successful selection becomes the default for later calls in that process. Some clients also announce workspace roots: one root can auto-select, while an ambiguous selection asks for a project instead of guessing. MCP deprecated Roots in its July 2026 revision, so explicit project selection is the documented default rather than a dependency on client discovery. -`project` accepts a project root path, file path, `file://` URI, or relative subproject path (e.g. `apps/dashboard`). +## See it -```json -{ - "name": "search_codebase", - "arguments": { - "query": "auth interceptor", - "project": "apps/dashboard" - } -} -``` +These are real CLI results from the open-source `angular-spotify` repository. -If you get `selection_required`, retry with one of the paths from `availableProjects`. Full routing details and response shapes in [`docs/capabilities.md`](./docs/capabilities.md#project-routing). +**Patterns and representative files** -## Language Support +![Codebase Context showing repository patterns and representative files](https://raw.githubusercontent.com/PatrickSys/codebase-context/master/docs/assets/patterns.png) -10 languages with full symbol extraction via Tree-sitter: TypeScript, JavaScript, Python, Java, Kotlin, C, C++, C#, Go, Rust. 30+ languages with indexing and retrieval coverage, including PHP, Ruby, Swift, Scala, Shell, and config formats. Angular, React, Next.js, and NestJS have dedicated analyzers; everything else uses the Generic analyzer with AST-aligned chunking when a grammar is available. +The map shows the patterns found in the project, how common they are, and files that demonstrate them. -## Configuration +**Search before an edit** -| Variable | Default | Description | -| ------------------------------ | --------------------------------- | ------------------------------------------------------------------- | -| `EMBEDDING_PROVIDER` | `transformers` | `openai` (fast, cloud) or `transformers` (local, private) | -| `OPENAI_API_KEY` | — | Required only if using `openai` provider | -| `CODEBASE_ROOT` | — | Bootstrap root for CLI and single-project MCP clients | -| `CODEBASE_CONTEXT_DEBUG` | — | Set to `1` for verbose logging | -| `EMBEDDING_MODEL` | `Xenova/bge-small-en-v1.5` | Local embedding model override | -| `CODEBASE_CONTEXT_HTTP` | — | Set to `1` to start in HTTP mode (same as `--http` flag) | -| `CODEBASE_CONTEXT_PORT` | `3100` | HTTP server port override (same as `--port`; ignored in stdio mode) | -| `CODEBASE_CONTEXT_CONFIG_PATH` | `~/.codebase-context/config.json` | Override the server config file path | +![Codebase Context showing a ranked search and edit preflight](https://raw.githubusercontent.com/PatrickSys/codebase-context/master/docs/assets/search-query.png) -Large projects can override the default 5,000 searchable-chunk safety limit per project: +The search result shows ranked files, relevant project patterns, and what the agent should read before it changes code. -```json -{ - "projects": [ - { - "root": "/path/to/large-project", - "parsing": { "maxChunks": 25000 } - } - ] -} -``` +More examples are available in the [CLI gallery](./docs/cli.md) and [walkthrough](https://github.com/PatrickSys/codebase-context/blob/master/docs/demo.md). -## Performance +## Privacy -- **First indexing**: 2-5 minutes for ~30k files (embedding computation). -- **Subsequent queries**: milliseconds from cache. -- **Incremental updates**: `refresh_index` with `incrementalOnly: true` processes only changed files (SHA-256 manifest diffing). +Code and indexes stay on the machine with the default local embedding provider. Docker, a GPU, and an API key are not required. -## File Structure +Cloud embeddings are optional. If you select a cloud provider, code chunks are sent to that provider to create the search index. The provider, model, project root, and local HTTP port can be changed through environment variables or the project config; see the [capabilities reference](./docs/capabilities.md). -``` -.codebase-context/ - memory.json # Team knowledge (should be persisted in git) - index-meta.json # Index metadata and version (generated) - intelligence.json # Pattern analysis (generated) - relationships.json # File/symbol relationships (generated) - index.json # Keyword index (generated) - index/ # Vector database (generated) -``` +This is the privacy boundary of Codebase Context itself. Your AI client may send search results or file contents to the model provider configured in that client. Codebase Context does not control that connection. If you commit and push `.codebase-context/memory.json`, the recorded project memory also travels with the repository. -**Recommended `.gitignore`:** +Generated indexes belong in `.gitignore`. Project memory can be kept in version control when the team wants to share it: ```gitignore -# Codebase Context - ignore generated files, keep memory .codebase-context/* !.codebase-context/memory.json ``` -## What to add to your CLAUDE.md / AGENTS.md - -Paste this into `.cursorrules`, `CLAUDE.md`, `AGENTS.md`, or wherever your AI reads project instructions: +## Evidence -```markdown -## Codebase Context (MCP) +Codebase Context runs locally through the Model Context Protocol (MCP), with indexed code kept on your machine by default. -**Start of every task:** Call `get_memory` to load team conventions before writing any code. +Its ranked code search combines **Match words**, **Search by meaning**, and **Rank results** to give the agent ranked files, a best example, project patterns, relationships, and relevant memory before they edit. -**Before editing existing code:** Call `search_codebase` with `intent: "edit"`. If the preflight card says `ready: false`, read the listed files before touching anything. +The [benchmark](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark.md) reports a corrected 100-attempt retrieval family across five local code-context tools: 99 completed attempts and 1 failed attempt. Codebase Context recovered 25.7% of expected gold files with 11.5% file precision in that fixed adapter run; jCodeMunch recovered 27.1% with 11.0% file precision. Raw-native is a deterministic lexical adapter here, not a full normal coding-agent baseline. These are retrieval observations, not a winner or a coding-quality claim. The [sanitized evidence extract](https://github.com/PatrickSys/codebase-context/blob/master/results/benchmark-presentation-evidence.json) records the exact values and source hashes. -**Before writing new code:** Call `get_team_patterns` to check how the team handles DI, state, testing, and library wrappers — don't introduce a new pattern if one already exists. +The same report retains a separate 300-attempt repeatability history, a 30-run two-task full-agent pilot, and a metered replay whose tool-use validity remains unresolved. A historical Codebase Context indexing observation with embeddings enabled took about 12 minutes 31 seconds, but it does not establish warm-use speed, query latency, install cost, or a cross-tool index-speed ranking. The report does not combine these families into a pooled score and does not establish patch correctness or end-to-end coding quality. Earlier public reports are preserved separately in the [benchmark archive](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark-prior-public-report.md). -**When asked to "remember" or "record" something:** Call `remember` immediately, before doing anything else. - -**When adding imports that cross module boundaries:** Call `detect_circular_dependencies` with the relevant scope after adding the import. -``` +The method and failures are documented so the measurements can be inspected with their limits. -These are the behaviors that make the most difference day-to-day. Copy, trim what doesn't apply to your stack, and add it once. +## Limits -## Links +- Local semantic indexing does more work than a plain text index, especially on a fresh checkout and CPU-only machine. +- Retrieval measurements describe expected-file coverage, file precision, and reported `peakPrivateGb`, not patch correctness or end-to-end coding quality. +- The paired token observation covers two frozen investigation tasks and records observed agent behavior; it is not a universal token or time guarantee. +- Setup checks differ by client. The detailed guide distinguishes a written config, a config recognized by the client, a local connection, and instructions checked only against official docs. +- Published `2.2.0` has dedicated Angular, React and Next.js analyzers; NestJS support is in the newer source candidate. Other projects use the generic analyzer and the language parsers available for that stack. +- The default searchable-chunk limit is 5,000 per project. Larger repositories can raise it in `.codebase-context/config.json`. +- The agent must identify its repository in tool calls when no default or unambiguous client root is available. Concurrent HTTP client isolation is not established by the stdio routing checks. -- [Benchmark](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark.md) — current discovery suite results and gate truth -- [Demo](https://github.com/PatrickSys/codebase-context/blob/master/docs/demo.md) — real CLI walkthrough -- [Client Setup](./docs/client-setup.md) — per-client config, HTTP setup, local build testing -- [Capabilities Reference](./docs/capabilities.md) — tool API, retrieval pipeline, decision card schema -- [CLI Gallery](./docs/cli.md) — formatted command output examples -- [Motivation](https://github.com/PatrickSys/codebase-context/blob/master/MOTIVATION.md) — research and design rationale -- [Contributing](https://github.com/PatrickSys/codebase-context/blob/master/CONTRIBUTING.md) — dev setup and eval harness -- [Changelog](https://github.com/PatrickSys/codebase-context/blob/master/CHANGELOG.md) +## Reference -## License +- [Client setup](./docs/client-setup.md#client-setup) - commands, config, proof level, and client limits +- [Capabilities](./docs/capabilities.md) - tools, response fields, routing, and configuration +- [CLI](./docs/cli.md) - terminal commands and example output +- [Benchmark](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark.md) - method, measurements, and failures +- [Demo](https://github.com/PatrickSys/codebase-context/blob/master/docs/demo.md) - a complete repository walkthrough +- [Motivation](https://github.com/PatrickSys/codebase-context/blob/master/MOTIVATION.md) - the design problem and research background +- [Contributing](https://github.com/PatrickSys/codebase-context/blob/master/CONTRIBUTING.md) - local development and evaluation commands +- [Changelog](https://github.com/PatrickSys/codebase-context/blob/master/CHANGELOG.md) - release history -Elastic-2.0 +Elastic-2.0. See [LICENSE](./LICENSE). diff --git a/docs/benchmark-prior-public-report.md b/docs/benchmark-prior-public-report.md new file mode 100644 index 0000000..506360f --- /dev/null +++ b/docs/benchmark-prior-public-report.md @@ -0,0 +1,182 @@ +# Earlier public benchmark report (snapshot) + +Preserved from `docs/benchmark.md` at public master commit `cc54fb5aee50ec177402e459b23be0fe8da7836e`. The content below is historical; its gates and dated observations are not new measurements. In the one-task section, ContextBench scores measure code-context overlap, not patch correctness. The original external artifact availability was not reverified for this presentation. + +[Return to the current engineering note](./benchmark.md). + +--- + +# Benchmarks + +This page tracks two separate benchmark surfaces: + +- The ContextBench implementation pilot, which uses the official ContextBench evaluator on one frozen task and five scoreable lanes. +- The older discovery benchmark, which measures local discovery usefulness and payload cost only. + +Neither section currently supports a broad benchmark-win claim. + +## ContextBench Implementation Pilot + +This is the current implementation-quality pilot for ContextBench. It is real scoreable evidence, but it is still a pilot because it covers one frozen task rather than the full frozen 20-task slice. + +### Scope + +- Protocol: `tests/fixtures/contextbench-benchmark-protocol.json` +- Task manifest: `tests/fixtures/contextbench-task-manifest.json` +- Selection file: `scripts/contextbench-five-lane-selections.json` +- Workflow: `.github/workflows/contextbench-five-lane-score.yml` +- Required lanes: `raw-native`, `codebase-context`, `codebase-memory-mcp`, `grepai`, `ripgrep-lexical` +- Model used for selection: `gpt-5.4-mini-high` +- Target task: `SWE-Bench-Pro__go__maintenance__bugfix__4df06349` +- Repository under test: `navidrome/navidrome` +- Base commit: `537e2fc033b71a4a69190b74f755ebc352bb4196` + +`CodeGraphContext` is not counted in this five-lane pilot because its supported CLI path indexed successfully but returned zero task-relevant candidates during readiness. That remains a readiness blocker, not a quality result. + +### Current Audited Run + +- Run: `25663469903` +- Job: `75329796667` +- Commit: `bbd3a8348aaec15809fd09dd8fc729e64df6d878` +- Artifact: `6915576867` +- Artifact digest: `sha256:718fd32049a2d98ed62fb0c15189d7dc9f1b027c202f286923de91d9f8985def` +- Artifact size: `88.9 KB` +- Uploaded files: `42` +- Status: `success` + +The artifact contains `summary.json`, `publishable-summary.json`, `publishable-validation.json`, `humanized-summary.md`, logs, lane selections, lane predictions, and official evaluator score files. It intentionally excludes full cloned repos and evaluator caches so the evidence package is small enough to inspect. + +### Quality Results + +Only rows scored by the official ContextBench evaluator are included here. Setup failures, tool errors, empty predictions, and judge failures are reliability outcomes, not quality rows. + +| Lane | File cov | File prec | Span cov | Span prec | Line cov | Line prec | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `raw-native` | 0.222 | 0.667 | 0.370 | 0.391 | 0.365 | 0.365 | +| `codebase-context` | 0.889 | 1.000 | 0.899 | 0.356 | 0.887 | 0.323 | +| `codebase-memory-mcp` | 0.222 | 0.667 | 0.346 | 0.380 | 0.315 | 0.337 | +| `grepai` | 0.333 | 0.500 | 0.048 | 0.042 | 0.059 | 0.061 | +| `ripgrep-lexical` | 0.222 | 0.667 | 0.401 | 0.341 | 0.419 | 0.302 | + +### Cost And Telemetry + +The report separates setup, indexing, query, selector, evaluator, and row-wall timing from quality. It also reports candidate counts, candidate token estimates when available, prediction token estimates, and selector token telemetry fields. + +`n/a` means the measurement was explicitly unavailable, not zero and not a hidden failure. Current gaps are: + +- Selector wall-clock and provider token telemetry were not captured in this proof artifact. +- `raw-native`, `codebase-context`, and `codebase-memory-mcp` emitted candidate counts but not candidate-pack bytes. +- `codebase-context` readiness did not emit index/query duration in the source artifact. + +### Bias Controls + +The generated `publishable-validation.json` must pass these checks before the report is treated as evidence: + +- Quality rows come only from the official ContextBench evaluator. +- Failed or unscoreable rows stay out of the quality table. +- All required lanes are scoreable. +- Setup, index, and query costs are separate from quality. +- Timing and token fields exist or carry explicit unavailable reasons. +- The protocol is frozen and `claimAllowed` is `false`. +- The task manifest attests that lane outputs were not observed during task selection. + +### What This Supports + +- It supports saying that the benchmark harness can produce real official ContextBench scores across five lanes. +- It supports saying that setup/index/query cost and context/token cost are now tracked separately from quality. +- It supports saying that the one-task pilot found a strong `codebase-context` result on this specific task. + +### What This Does Not Support + +- It does not support claiming that `codebase-context` beats competitors overall. +- It does not support claiming patch correctness or productivity improvements. +- It does not replace the full frozen 20-task, repeated-run benchmark required for claim-bearing results. + +## Discovery Benchmark + +This section documents the current public discovery proof from the checked-in result artifacts on `master`. +It is a discovery benchmark, not an implementation-quality benchmark. + +## Discovery Scope + +- Frozen fixtures: + - `tests/fixtures/discovery-angular-spotify.json` + - `tests/fixtures/discovery-excalidraw.json` + - `tests/fixtures/discovery-benchmark-protocol.json` +- Frozen repos used in the current proof run: + - `repos/angular-spotify` + - `repos/excalidraw` +- Current gate artifact: + - `results/gate-evaluation.json` +- Comparator evidence: + - `results/comparator-evidence.json` + +## Discovery Reproduction + +Run the repo-local proof artifacts from the current `master` checkout: + +```bash +node scripts/run-eval.mjs repos/angular-spotify --mode=discovery --fixture-a=tests/fixtures/discovery-angular-spotify.json --skip-reindex --output=results/codebase-context-angular-spotify.json +node scripts/run-eval.mjs repos/excalidraw --mode=discovery --fixture-a=tests/fixtures/discovery-excalidraw.json --skip-reindex --output=results/codebase-context-excalidraw.json +node scripts/benchmark-comparators.mjs --repos repos/angular-spotify,repos/excalidraw --output results/comparator-evidence.json +node scripts/run-eval.mjs repos/angular-spotify repos/excalidraw --mode=discovery --fixture-a=tests/fixtures/discovery-angular-spotify.json --fixture-b=tests/fixtures/discovery-excalidraw.json --competitor-results=results/comparator-evidence.json --skip-reindex --output=results/gate-evaluation.json +``` + +## Discovery Current Result + +From `results/gate-evaluation.json`: + +- `status`: `pending_evidence` +- `suiteStatus`: `complete` +- `claimAllowed`: `false` +- `totalTasks`: `24` +- `averageUsefulness`: `0.75` +- `averageEstimatedTokens`: `1827.0833` +- `bestExampleUsefulnessRate`: `0.125` + +Repo-level outputs from the same rerun: + +| Repo | Tasks | Avg usefulness | Avg estimated tokens | Best-example usefulness | +| --- | ---: | ---: | ---: | ---: | +| `angular-spotify` | 12 | 0.8333 | 2138.4167 | 0.25 | +| `excalidraw` | 12 | 0.6667 | 1506.0833 | 0 | + +## Discovery Gate Truth + +The gate is intentionally still blocked. + +- The combined suite covers both public repos. +- `claimAllowed` remains `false` because comparator evidence still does not support a benchmark-win claim. +- Two comparator artifacts now return `status: "ok"`, but that does not yet close the gate: + - `raw Claude Code` still leaves the baseline `pending_evidence` because `averageFirstRelevantHit` is `null` + - `codebase-memory-mcp` now has real current metrics, but the gate still marks it `failed` on the frozen tolerance rule +- Three comparator lanes still fail setup entirely: `GrepAI`, `jCodeMunch`, and `CodeGraphContext`. + +## Discovery Comparator Reality + +The current comparator artifact records incomplete comparator evidence, not benchmark wins. + +| Comparator | Status | Current reason | +| --- | --- | --- | +| `codebase-memory-mcp` | comparator artifact: `ok`; gate: `failed` | Runs through the repaired graph-backed path and now records real metrics (`averageUsefulness: 0.1875`, `averageFirstRelevantHit: 1.2857`, `bestExampleUsefulnessRate: 0.5`), but the frozen gate still fails it on the required usefulness comparisons | +| `jCodeMunch` | `setup_failed` | `MCP error -32000: Connection closed` | +| `GrepAI` | `setup_failed` | Local Go binary and Ollama model path not present | +| `CodeGraphContext` | `setup_failed` | `MCP error -32000: Connection closed` | +| `raw Claude Code` | comparator artifact: `ok`; gate: `pending_evidence` | The explicit Haiku CLI runner now returns current metrics (`averageUsefulness: 0.0278`, `averageEstimatedTokens: 32.1667`), but the baseline still lacks `averageFirstRelevantHit`, so the gate keeps this lane as missing evidence | + +`CodeGraphContext` remains part of the frozen comparison frame. It is not omitted from the public story just because the lane still fails to start. + +## Discovery Important Limitations + +- This benchmark measures discovery usefulness and payload cost only. +- It does not measure implementation correctness, patch quality, or end-to-end task completion. +- Comparator setup remains environment-sensitive, and the checked-in comparator outputs still do not satisfy the frozen claim gate. +- The reranker cache is currently corrupted on this machine. During the proof rerun, search fell back to original ordering after `Protobuf parsing failed` while still completing the harness. +- `averageFirstRelevantHit` remains `null` in the current gate output, which is enough to keep the raw-Claude baseline in `pending_evidence`. + +## Discovery Claims Supported + +- It can support claims about the shipped discovery surfaces and their current measured outputs on the frozen public tasks. +- It can support claims that the proof gate is still blocked by comparator evidence. +- It cannot support claims that `codebase-context` beats the named comparators today. +- It cannot support claims about edit success, code quality, or implementation speed. diff --git a/docs/benchmark.md b/docs/benchmark.md index bf71928..73d073b 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -1,174 +1,80 @@ -# Benchmarks +# What our local code-context experiments show -This page tracks two separate benchmark surfaces: +This report describes local evidence from five ways of giving an AI coding agent context: Raw-native, Codebase Context, jCodeMunch, Repowise, and context-mode. The evidence is split into separate families because retrieval, repeated runs, full-agent observations, and a metered replay do not answer the same question. The families are not pooled into one score. -- The ContextBench implementation pilot, which uses the official ContextBench evaluator on one frozen task and five scoreable lanes. -- The older discovery benchmark, which measures local discovery usefulness and payload cost only. +The clearest current finding is narrow: in this fixed retrieval setup, Codebase Context recovered 25.7% of expected gold files at 11.5% file precision, while fresh indexing had material local cost. These measurements do not show better completed coding tasks. -Neither section currently supports a broad benchmark-win claim. +## Evidence families -## ContextBench Implementation Pilot +| Evidence family | Attempts | Completed | Failed or otherwise excluded | What it answers | +| --- | ---: | ---: | ---: | --- | +| Corrected retrieval | 100 | 99 of 100 | 1 failed | Expected-file coverage, file precision, and reported `peakPrivateGb` | +| Repeated retrieval | 300 | 288 of 300 | 12 failed | Repeatability, setup, and runtime history | +| Full-agent pilot | 30 | 30 of 30 | 0 | Observed token and task-time results across two frozen tasks | +| Metered replay (excluded) | 100 | 100 counter records in historical summary | Tool-use validity unresolved | Why those replay records are not scored | -This is the current implementation-quality pilot for ContextBench. It is real scoreable evidence, but it is still a pilot because it covers one frozen task rather than the full frozen 20-task slice. +The first two families contain 400 retrieval attempts. The full-agent pilot and the metered replay remain separate from those retrieval measurements. -### Scope +## Retrieval results -- Protocol: `tests/fixtures/contextbench-benchmark-protocol.json` -- Task manifest: `tests/fixtures/contextbench-task-manifest.json` -- Selection file: `scripts/contextbench-five-lane-selections.json` -- Workflow: `.github/workflows/contextbench-five-lane-score.yml` -- Required lanes: `raw-native`, `codebase-context`, `codebase-memory-mcp`, `grepai`, `ripgrep-lexical` -- Model used for selection: `gpt-5.4-mini-high` -- Target task: `SWE-Bench-Pro__go__maintenance__bugfix__4df06349` -- Repository under test: `navidrome/navidrome` -- Base commit: `537e2fc033b71a4a69190b74f755ebc352bb4196` +Expected-file coverage means expected gold files recovered divided by expected gold files, averaged within the corrected retrieval-only family. File precision means expected gold files recovered divided by all files returned. The table reports both alongside completion status and the reported `peakPrivateGb` telemetry field. The underlying unit of that field was not independently verified, so these values are not labelled as GiB or treated as install size. -`CodeGraphContext` is not counted in this five-lane pilot because its supported CLI path indexed successfully but returned zero task-relevant candidates during readiness. That remains a readiness blocker, not a quality result. +| Tool | Completed | Failed | Expected-file coverage | File precision | Reported `peakPrivateGb` | +| --- | ---: | ---: | ---: | ---: | ---: | +| Raw search | 20/20 | 0 | 4.3% | 4.0% | 0.06 | +| Codebase Context | 20/20 | 0 | 25.7% | 11.5% | 2.90 | +| jCodeMunch | 20/20 | 0 | 27.1% | 11.0% | 0.33 | +| Repowise | 19/20 | 1 | 20.1% (failed run = 0) | 12.0% | 4.82 | +| context-mode | 20/20 | 0 | 16.8% | 11.7% | 0.15 | -### Current Audited Run +The corrected retrieval family therefore includes 99 completed attempts and 1 failed attempt. Repowise is the only tool with a failed attempt in this table. Its 20.1% is failure-inclusive: the failed attempt is counted as zero. Raw search is a deterministic lexical adapter in this protocol, not a full normal coding-agent baseline. These values describe file retrieval and returned-file precision, not a coding score. The [sanitized evidence extract](../results/benchmark-presentation-evidence.json) records the exact values and source hashes. -- Run: `25663469903` -- Job: `75329796667` -- Commit: `bbd3a8348aaec15809fd09dd8fc729e64df6d878` -- Artifact: `6915576867` -- Artifact digest: `sha256:718fd32049a2d98ed62fb0c15189d7dc9f1b027c202f286923de91d9f8985def` -- Artifact size: `88.9 KB` -- Uploaded files: `42` -- Status: `success` +## Paired token and time observation -The artifact contains `summary.json`, `publishable-summary.json`, `publishable-validation.json`, `humanized-summary.md`, logs, lane selections, lane predictions, and official evaluator score files. It intentionally excludes full cloned repos and evaluator caches so the evidence package is small enough to inspect. +The full-agent pilot used `gpt-5.4-mini-high`, with three runs per tool and task. It covers one frozen investigation task for each of two open-source codebases. The two tasks are PonyC and fmt. The paired tables compare the raw-native agent lane with Codebase Context; all five lanes remain available in the evidence extract. -### Quality Results +### PonyC -Only rows scored by the official ContextBench evaluator are included here. Setup failures, tool errors, empty predictions, and judge failures are reliability outcomes, not quality rows. +| Tool | Input tokens | Cached input tokens | Output tokens | Reasoning output tokens | Task time | +| --- | ---: | ---: | ---: | ---: | ---: | +| Raw search | 653,348 | 610,304 | 9,011 | 4,494 | 185,061 ms | +| Codebase Context | 223,558 | 190,976 | 5,564 | 3,024 | 114,664 ms | -| Lane | File cov | File prec | Span cov | Span prec | Line cov | Line prec | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| `raw-native` | 0.222 | 0.667 | 0.370 | 0.391 | 0.365 | 0.365 | -| `codebase-context` | 0.889 | 1.000 | 0.899 | 0.356 | 0.887 | 0.323 | -| `codebase-memory-mcp` | 0.222 | 0.667 | 0.346 | 0.380 | 0.315 | 0.337 | -| `grepai` | 0.333 | 0.500 | 0.048 | 0.042 | 0.059 | 0.061 | -| `ripgrep-lexical` | 0.222 | 0.667 | 0.401 | 0.341 | 0.419 | 0.302 | +### fmt -### Cost And Telemetry +| Tool | Input tokens | Cached input tokens | Output tokens | Reasoning output tokens | Task time | +| --- | ---: | ---: | ---: | ---: | ---: | +| Raw search | 290,186 | 272,384 | 7,567 | 4,024 | 142,002 ms | +| Codebase Context | 247,868 | 206,848 | 7,930 | 4,784 | 143,643 ms | -The report separates setup, indexing, query, selector, evaluator, and row-wall timing from quality. It also reports candidate counts, candidate token estimates when available, prediction token estimates, and selector token telemetry fields. +All values above are medians across three `gpt-5.4-mini-high` runs for the named task and tool. Cached input is included in input and must not be added to input again. Each column is aggregated independently. Codebase Context used fewer input tokens in both examples, but its median task time was slightly longer on fmt. This does not establish better answers or patches. -`n/a` means the measurement was explicitly unavailable, not zero and not a hidden failure. Current gaps are: +The prompt requested at most 12 shell or tool calls. The CLI did not hard-enforce that request. The result describes observed agent behavior, not universal or hard-call-controlled savings. -- Selector wall-clock and provider token telemetry were not captured in this proof artifact. -- `raw-native`, `codebase-context`, and `codebase-memory-mcp` emitted candidate counts but not candidate-pack bytes. -- `codebase-context` readiness did not emit index/query duration in the source artifact. +## What can be reproduced from the public repository -### Bias Controls - -The generated `publishable-validation.json` must pass these checks before the report is treated as evidence: - -- Quality rows come only from the official ContextBench evaluator. -- Failed or unscoreable rows stay out of the quality table. -- All required lanes are scoreable. -- Setup, index, and query costs are separate from quality. -- Timing and token fields exist or carry explicit unavailable reasons. -- The protocol is frozen and `claimAllowed` is `false`. -- The task manifest attests that lane outputs were not observed during task selection. - -### What This Supports - -- It supports saying that the benchmark harness can produce real official ContextBench scores across five lanes. -- It supports saying that setup/index/query cost and context/token cost are now tracked separately from quality. -- It supports saying that the one-task pilot found a strong `codebase-context` result on this specific task. - -### What This Does Not Support - -- It does not support claiming that `codebase-context` beats competitors overall. -- It does not support claiming patch correctness or productivity improvements. -- It does not replace the full frozen 20-task, repeated-run benchmark required for claim-bearing results. - -## Discovery Benchmark - -This section documents the current public discovery proof from the checked-in result artifacts on `master`. -It is a discovery benchmark, not an implementation-quality benchmark. - -## Discovery Scope - -- Frozen fixtures: - - `tests/fixtures/discovery-angular-spotify.json` - - `tests/fixtures/discovery-excalidraw.json` - - `tests/fixtures/discovery-benchmark-protocol.json` -- Frozen repos used in the current proof run: - - `repos/angular-spotify` - - `repos/excalidraw` -- Current gate artifact: - - `results/gate-evaluation.json` -- Comparator evidence: - - `results/comparator-evidence.json` - -## Discovery Reproduction - -Run the repo-local proof artifacts from the current `master` checkout: +The public repository snapshot at commit `cc54fb5` contains the [benchmark script](https://github.com/PatrickSys/codebase-context/blob/cc54fb5aee50ec177402e459b23be0fe8da7836e/scripts/contextbench-runner.mjs), [benchmark rules](https://github.com/PatrickSys/codebase-context/blob/cc54fb5aee50ec177402e459b23be0fe8da7836e/tests/fixtures/contextbench-benchmark-protocol.json), and [20-task manifest](https://github.com/PatrickSys/codebase-context/blob/cc54fb5aee50ec177402e459b23be0fe8da7836e/tests/fixtures/contextbench-task-manifest.json). Those links were reachable on October 5, 2026; earlier local-commit links were not. You can validate the checked-in fixtures with: ```bash -node scripts/run-eval.mjs repos/angular-spotify --mode=discovery --fixture-a=tests/fixtures/discovery-angular-spotify.json --skip-reindex --output=results/codebase-context-angular-spotify.json -node scripts/run-eval.mjs repos/excalidraw --mode=discovery --fixture-a=tests/fixtures/discovery-excalidraw.json --skip-reindex --output=results/codebase-context-excalidraw.json -node scripts/benchmark-comparators.mjs --repos repos/angular-spotify,repos/excalidraw --output results/comparator-evidence.json -node scripts/run-eval.mjs repos/angular-spotify repos/excalidraw --mode=discovery --fixture-a=tests/fixtures/discovery-angular-spotify.json --fixture-b=tests/fixtures/discovery-excalidraw.json --competitor-results=results/comparator-evidence.json --skip-reindex --output=results/gate-evaluation.json +node scripts/contextbench-runner.mjs --validate-fixtures ``` -## Discovery Current Result - -From `results/gate-evaluation.json`: - -- `status`: `pending_evidence` -- `suiteStatus`: `complete` -- `claimAllowed`: `false` -- `totalTasks`: `24` -- `averageUsefulness`: `0.75` -- `averageEstimatedTokens`: `1827.0833` -- `bestExampleUsefulnessRate`: `0.125` - -Repo-level outputs from the same rerun: - -| Repo | Tasks | Avg usefulness | Avg estimated tokens | Best-example usefulness | -| --- | ---: | ---: | ---: | ---: | -| `angular-spotify` | 12 | 0.8333 | 2138.4167 | 0.25 | -| `excalidraw` | 12 | 0.6667 | 1506.0833 | 0 | - -## Discovery Gate Truth - -The gate is intentionally still blocked. +They show the test structure and task format. They are not a byte-for-byte copy of every file used by the measured sessions; all three file hashes differ from the later 30-session pilot. The manifest names tasks and base commits but does not contain their full problem statements or gold-context values. The exact machine-specific raw run directories behind the 100-attempt retrieval table, the 300-attempt repeatability history, and the 30-session agent pilot are not checked into the public repository, so this checkout cannot regenerate the reported tables with one command. The [original schema-2 retrieval summary](../results/contextbench-engineering-note-summary.json) is preserved from local commit `18e0a43a90fe67e9df0d728b2134e9d430469883`; the [sanitized evidence extract](../results/benchmark-presentation-evidence.json) preserves the exact values and hashes used for this presentation. Fixture validation checks fixture consistency, not the reported measurements. -- The combined suite covers both public repos. -- `claimAllowed` remains `false` because comparator evidence still does not support a benchmark-win claim. -- Two comparator artifacts now return `status: "ok"`, but that does not yet close the gate: - - `raw Claude Code` still leaves the baseline `pending_evidence` because `averageFirstRelevantHit` is `null` - - `codebase-memory-mcp` now has real current metrics, but the gate still marks it `failed` on the frozen tolerance rule -- Three comparator lanes still fail setup entirely: `GrepAI`, `jCodeMunch`, and `CodeGraphContext`. +Local session artifacts retained outside this public checkout record Windows x64 with 16 logical CPUs, about 24 GiB of RAM, Node.js 24.14.1, Claude Code 2.1.173, and `gpt-5.4-mini-high` for the later agent pilot. Readers cannot independently inspect those environment files from this checkout. Exact version pins for all five tools in the retrieval families were not captured consistently enough to publish a trustworthy version matrix. -## Discovery Comparator Reality +This makes the public evidence partially reproducible: the method and current fixtures are inspectable, while the exact reported runs are not independently replayable from this checkout alone. The tables should be read as a transparent engineering report, not a paper-grade reproduction package. -The current comparator artifact records incomplete comparator evidence, not benchmark wins. +## Method and limits -| Comparator | Status | Current reason | -| --- | --- | --- | -| `codebase-memory-mcp` | comparator artifact: `ok`; gate: `failed` | Runs through the repaired graph-backed path and now records real metrics (`averageUsefulness: 0.1875`, `averageFirstRelevantHit: 1.2857`, `bestExampleUsefulnessRate: 0.5`), but the frozen gate still fails it on the required usefulness comparisons | -| `jCodeMunch` | `setup_failed` | `MCP error -32000: Connection closed` | -| `GrepAI` | `setup_failed` | Local Go binary and Ollama model path not present | -| `CodeGraphContext` | `setup_failed` | `MCP error -32000: Connection closed` | -| `raw Claude Code` | comparator artifact: `ok`; gate: `pending_evidence` | The explicit Haiku CLI runner now returns current metrics (`averageUsefulness: 0.0278`, `averageEstimatedTokens: 32.1667`), but the baseline still lacks `averageFirstRelevantHit`, so the gate keeps this lane as missing evidence | +The retrieval percentages, precision values, and reported `peakPrivateGb` fields come from the corrected retrieval-only family. The repeated retrieval family is reported as completion and repeatability history rather than merged into that table. The full-agent pilot reports input, cached input, output, reasoning output, and task time for two frozen investigation tasks. The metered replay is retained as history only and excluded from effectiveness or token-saving claims while tool-use validity is unresolved. -`CodeGraphContext` remains part of the frozen comparison frame. It is not omitted from the public story just because the lane still fails to start. +A historical fresh isolated Codebase Context indexing observation with embeddings enabled used the Transformers provider and `Xenova/bge-small-en-v1.5`. Its median was about 12 minutes 31 seconds. This is a historical observation in methodology and limits, not a normal-use estimate or a cross-tool speed ranking. -## Discovery Important Limitations +That index observation does not describe normal warm use, incremental indexing, query latency, install time, or cross-tool index-speed ranking. Shared model-cache state, actual GPU use, the exact scan, analysis, embedding, and storage stage durations, and equivalent cold work across competitors were not established by that observation. -- This benchmark measures discovery usefulness and payload cost only. -- It does not measure implementation correctness, patch quality, or end-to-end task completion. -- Comparator setup remains environment-sensitive, and the checked-in comparator outputs still do not satisfy the frozen claim gate. -- The reranker cache is currently corrupted on this machine. During the proof rerun, search fell back to original ordering after `Protobuf parsing failed` while still completing the harness. -- `averageFirstRelevantHit` remains `null` in the current gate output, which is enough to keep the raw-Claude baseline in `pending_evidence`. +The benchmark does not measure patch correctness, end-to-end task completion, or a universal token or time result. It also does not combine the four evidence families into a pooled score. Setup and run failures remain part of the record so that incomplete attempts are visible. -## Discovery Claims Supported +Historical note: the [earlier public report archive](./benchmark-prior-public-report.md) contains a one-task official pilot and a separate 24-task discovery report. Those records and their comparator statuses remain separate, are not folded into this current four-family summary, and retain their claim gates. -- It can support claims about the shipped discovery surfaces and their current measured outputs on the frozen public tasks. -- It can support claims that the proof gate is still blocked by comparator evidence. -- It cannot support claims that `codebase-context` beats the named comparators today. -- It cannot support claims about edit success, code quality, or implementation speed. +See the [comparison table](./comparison-table.md) for the retrieval family in a compact form. The repository walkthrough is in the [demo](./demo.md). diff --git a/docs/capabilities.md b/docs/capabilities.md index 7545148..748d5a1 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -6,10 +6,10 @@ Technical reference for what `codebase-context` ships today. The public product The server supports two transport modes: -| Mode | Command | MCP endpoint | -| ------------------- | ------------------------------------------- | ---------------------------- | -| **stdio** (default) | `npx -y codebase-context` | Spawned process stdin/stdout | -| **HTTP** | `npx -y codebase-context --http [--port N]` | `http://127.0.0.1:3100/mcp` | +| Mode | Command | MCP endpoint | +| ------------------- | ------------------------------------------------- | ---------------------------- | +| **stdio** (default) | `npx -y codebase-context@2.2.0` | Spawned process stdin/stdout | +| **HTTP** | `npx -y codebase-context@2.2.0 --http [--port N]` | `http://127.0.0.1:3100/mcp` | HTTP defaults to `127.0.0.1:3100`. Override with `--port`, `CODEBASE_CONTEXT_PORT`, or `server.port` in `~/.codebase-context/config.json`. @@ -26,7 +26,7 @@ Copy-pasteable client config templates are shipped in the package: - `templates/mcp/stdio/.mcp.json` — stdio setup for `.mcp.json`-style clients - `templates/mcp/http/.mcp.json` — HTTP setup for `.mcp.json`-style clients -Client transport support varies — see [README.md](../README.md) for a per-client matrix covering Claude Code, Cursor, Codex, Windsurf, VS Code, Claude Desktop, and OpenCode. +Use Node.js 22 or newer with the published 2.2.0 commands shown here. For client registration recipes and their verification limits, see the [client setup guide](./client-setup.md). ## CLI Reference @@ -49,15 +49,15 @@ For a command gallery with examples, see `docs/cli.md`. | `memory add` | `--type`, `--category`, `--memory`, `--reason` | `remember` | | `memory remove ` | — | — | -All commands accept `--json` for raw JSON output. Errors go to stderr with exit code 1. +Commands that list `--json` above support raw JSON output. For MCP client registration, follow the [published client setup recipes](./client-setup.md); do not use the broken `init` setup command in published 2.2.0. Errors go to stderr with exit code 1. ```bash # Quick examples -npx codebase-context status -npx codebase-context search --query "auth middleware" --intent edit -npx codebase-context refs --symbol "UserService" --limit 10 -npx codebase-context cycles --scope src/features -npx codebase-context reindex --incremental +npx -y codebase-context@2.2.0 status +npx -y codebase-context@2.2.0 search --query "auth middleware" --intent edit +npx -y codebase-context@2.2.0 refs --symbol "UserService" --limit 10 +npx -y codebase-context@2.2.0 cycles --scope src/features +npx -y codebase-context@2.2.0 reindex --incremental ``` ## Tool Surface @@ -104,12 +104,12 @@ Behavior matrix: Rules: -- If the client provides workspace context, that becomes the trusted workspace boundary for the session. In practice this usually comes from MCP roots. -- Treat seamless multi-project routing as evidence-backed only for roots-capable hosts. Without roots, explicit fallback is still required. -- If the server still cannot tell which project to use, a bootstrap path or explicit absolute `project` path remains the fallback. -- `project` is the canonical explicit selector when routing is ambiguous. +- The portable default is explicit selection: pass the repository's absolute path as `project`. A successful selection becomes the active project for that server process. +- Client-announced workspace roots remain a compatibility aid and constrain explicit selectors to that workspace. One valid root can auto-select; ambiguous roots require selection. Roots are not required for the documented setup. +- Without a configured folder, known root or active selection, the server returns `selection_required` instead of guessing a project. - `project` may point at a project path, file path, `file://` URI, or relative subproject path. - Later tool calls may omit `project`; the server falls back to the active project when one has already been established. +- Select again after a server restart or when changing repositories. Published-package A/B/A stdio routing was verified; it does not establish native agent continuation or concurrent HTTP isolation. See the [client setup guide](./client-setup.md#project-routing-contract) for the checks and their limits. - The server does not rely on `cwd` walk-up in MCP mode. - `codebase://context` serves the active project. Before selection in an unresolved multi-project session, it returns a workspace overview with candidate projects, readiness state, and project-scoped resource URIs. - `codebase://context/project/` serves a specific project directly and also makes that project active for later tool calls. @@ -282,29 +282,25 @@ Notes: ## Evaluation Harness -Current public proof bundle: [`docs/benchmark.md`](../docs/benchmark.md) and [`docs/comparison-table.md`](../docs/comparison-table.md). +Current public evidence: [`docs/benchmark.md`](../docs/benchmark.md) and [`docs/comparison-table.md`](../docs/comparison-table.md). -Reproducible evaluation is shipped as a CLI entrypoint backed by shared scoring/reporting code. +Reproducible evaluation is shipped as a CLI entrypoint backed by shared scoring and reporting code. -- **Command:** `npm run eval -- [codebaseB] --mode retrieval|discovery [--competitor-results ]` (builds first, then runs `scripts/run-eval.mjs`) +- **Command:** `pnpm run eval -- [codebaseB] --mode retrieval|discovery [--competitor-results ]` (builds first, then runs `scripts/run-eval.mjs`) +- **Contributor requirement:** this repository command uses the pinned `pnpm@10.27.0`; end users do not need pnpm for the `npx` CLI. - **Shared implementation:** `src/eval/harness.ts`, `src/eval/discovery-harness.ts`, and `src/eval/types.ts` -- **Frozen retrieval fixtures:** - - `tests/fixtures/eval-angular-spotify.json` - - `tests/fixtures/eval-controlled.json` + `tests/fixtures/codebases/eval-controlled/` -- **Frozen discovery fixtures:** - - `tests/fixtures/discovery-angular-spotify.json` - - `tests/fixtures/discovery-excalidraw.json` - - `tests/fixtures/discovery-benchmark-protocol.json` -- **Retrieval metrics:** Top-1 accuracy, Top-3 recall, spec contamination rate, and a gate pass/fail +- **Retrieval fixtures:** `tests/fixtures/eval-angular-spotify.json` and `tests/fixtures/eval-controlled.json` with `tests/fixtures/codebases/eval-controlled/` +- **Discovery fixtures:** `tests/fixtures/discovery-angular-spotify.json`, `tests/fixtures/discovery-excalidraw.json`, and `tests/fixtures/discovery-benchmark-protocol.json` +- **Retrieval metrics:** Top-1 accuracy, Top-3 recall, spec contamination rate, and a pass/fail result for the configured retrieval check - **Discovery metrics:** usefulness score, payload bytes, estimated tokens, first relevant hit, and best-example usefulness -- **Discovery gate:** discovery mode evaluates the frozen ship gate only when the full public suite and comparator metrics are available; missing comparator evidence is reported as pending, not silently treated as pass/fail -- **Current checked-in gate truth:** `results/gate-evaluation.json` remains `pending_evidence` with `claimAllowed: false`; the raw-Claude baseline still lacks `averageFirstRelevantHit`, `codebase-memory-mcp` still fails the frozen usefulness comparisons, and the remaining named lanes are still `setup_failed` -- **Limits:** discovery mode is discovery-only, uses current shipped surfaces only, and does not claim implementation quality; named competitor runs remain a documented hybrid/manual lane rather than a built-in automated benchmark +- **Public benchmark families:** the current documentation keeps 100 corrected retrieval attempts, 300 repeated retrieval attempts, 30 full-agent pilot attempts, and 100 invalid-token replays separate across five local code-context tools +- **Discovery limits:** discovery mode describes the shipped discovery surface and does not measure patch correctness or end-to-end task completion; named competitor runs remain a documented hybrid/manual lane rather than a built-in automated benchmark +- **Evidence limits:** missing or failed setup and comparator runs remain visible in their source artifacts and are not treated as a pass ## Limitations - **Symbol refs are not a call-graph.** `get_symbol_references` counts identifier-node occurrences in the AST (comments/strings excluded via Tree-sitter). It does not distinguish call sites from type annotations, variable assignments, or imports. Full call-site-specific analysis (`call_expression` nodes only) is a roadmap item. - **Impact is 2-hop max.** `computeImpactCandidates` walks direct importers then their importers. Full BFS reachability is on the roadmap. -- **Angular, React, Next.js, and NestJS have dedicated analyzers.** All other languages go through the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). +- **Published 2.2.0 has dedicated Angular, React, and Next.js analyzers.** NestJS support is in a newer source candidate and is not part of published 2.2.0. Other languages use the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). - **Default embedding model is `bge-small-en-v1.5` (512-token context).** Granite (8192 context) is opt-in via `EMBEDDING_MODEL`. OpenAI is opt-in via `EMBEDDING_PROVIDER=openai` — sends code externally. - **Patterns are file-level frequency counts.** Not semantic clustering. Rising/Declining trend is derived from git commit recency for files using each pattern, not from usage semantics. diff --git a/docs/cli.md b/docs/cli.md index 5025f65..dfda760 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -10,16 +10,18 @@ > Output depends on the repo you run it against. The examples below are illustrative (paths, counts, and detected frameworks will vary). > -> The CLI is intentionally single-project per invocation. MCP multi-project routing and trusted-root auto-discovery are only for the MCP server; the CLI still targets one root via `CODEBASE_ROOT` or the current working directory. +> These examples show an earlier CLI run; they do not verify the current package or an MCP connection. +> +> The commands below use published `codebase-context@2.2.0` and require Node.js 22 or newer. The CLI is intentionally single-project per invocation and targets one root via `CODEBASE_ROOT` or the current working directory. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). ## How to run ```bash -# Run from a repo root, or set CODEBASE_ROOT explicitly: -CODEBASE_ROOT=/path/to/repo npx -y codebase-context status +# CLI only: run from a repo root, or set CODEBASE_ROOT explicitly: +CODEBASE_ROOT=/path/to/repo npx -y codebase-context@2.2.0 status -# Every command supports --json (machine output). Human mode is default. -npx -y codebase-context patterns --json +# Commands that show --json in their help support machine output. Human mode is default. +npx -y codebase-context@2.2.0 patterns --json ``` ### ASCII fallback @@ -27,7 +29,7 @@ npx -y codebase-context patterns --json If your terminal doesn’t render Unicode box-drawing cleanly: ```bash -CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context patterns +CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context@2.2.0 patterns ``` ## Commands @@ -42,13 +44,27 @@ CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context patterns - `reindex` — rebuild index (full or incremental) - `style-guide` — find style guide sections in docs - `memory list|add|remove` — manage team memory (stored in `.codebase-context/memory.json`) +- MCP client registration — use the published 2.2.0 commands in the [client setup guide](./client-setup.md); published `init` is not a reliable setup route --- +## First use + +Run these CLI commands from the project root. Build or refresh the local index first; the first index may take a while while the project is scanned and local embeddings are created. Then inspect the conventions map and search for the code you need: + +```bash +npx -y codebase-context@2.2.0 reindex +npx -y codebase-context@2.2.0 map +npx -y codebase-context@2.2.0 search --query "auth middleware" +``` + +This is the CLI preparation flow. It is separate from MCP client registration and does not by itself show that an MCP client connected successfully. + ## `map` ```bash -npx -y codebase-context map +npx -y codebase-context@2.2.0 reindex +npx -y codebase-context@2.2.0 map ``` The conventions map - run this first on an unfamiliar repo. It shows architecture layers, active patterns with adoption rates and trend direction, and the golden files the team treats as the strongest examples. This is also what the MCP server delivers to AI agents via the `codebase://context` resource on first call, before search narrows to a specific local example. @@ -83,7 +99,7 @@ Example output (truncated): ## `metadata` ```bash -npx -y codebase-context metadata +npx -y codebase-context@2.2.0 metadata ``` Example output: @@ -104,7 +120,7 @@ Example output: ## `patterns` ```bash -npx -y codebase-context patterns +npx -y codebase-context@2.2.0 patterns ``` Example output (truncated): @@ -126,7 +142,8 @@ Example output (truncated): ## `search` ```bash -npx -y codebase-context search --query "file watcher" --intent edit --limit 3 +npx -y codebase-context@2.2.0 reindex +npx -y codebase-context@2.2.0 search --query "file watcher" --intent edit --limit 3 ``` Example output (truncated): @@ -147,7 +164,7 @@ Example output (truncated): ## `refs` ```bash -npx -y codebase-context refs --symbol "startFileWatcher" --limit 10 +npx -y codebase-context@2.2.0 refs --symbol "startFileWatcher" --limit 10 ``` Example output (truncated): @@ -166,7 +183,7 @@ Example output (truncated): ## `cycles` ```bash -npx -y codebase-context cycles --scope src +npx -y codebase-context@2.2.0 cycles --scope src ``` Example output: @@ -182,7 +199,7 @@ Example output: ## `status` ```bash -npx -y codebase-context status +npx -y codebase-context@2.2.0 status ``` Example output: @@ -201,8 +218,8 @@ Example output: ## `reindex` ```bash -npx -y codebase-context reindex -npx -y codebase-context reindex --incremental --reason "changed watcher logic" +npx -y codebase-context@2.2.0 reindex +npx -y codebase-context@2.2.0 reindex --incremental --reason "changed watcher logic" ``` > **MCP server mode**: if you're running codebase-context as an MCP server (long-running process), the index auto-refreshes via a file watcher — you don't need to call `reindex` between edits. Use `reindex` for one-shot CLI runs or to force a full rebuild. @@ -210,7 +227,7 @@ npx -y codebase-context reindex --incremental --reason "changed watcher logic" ## `style-guide` ```bash -npx -y codebase-context style-guide --query "naming" +npx -y codebase-context@2.2.0 style-guide --query "naming" ``` Example output: @@ -223,14 +240,14 @@ No style guides found. ## `memory` ```bash -npx -y codebase-context memory list -npx -y codebase-context memory list --query "watcher" +npx -y codebase-context@2.2.0 memory list +npx -y codebase-context@2.2.0 memory list --query "watcher" -npx -y codebase-context memory add \ +npx -y codebase-context@2.2.0 memory add \ --type gotcha \ --category tooling \ --memory "Use pnpm, not npm" \ --reason "Workspace support and speed" -npx -y codebase-context memory remove +npx -y codebase-context@2.2.0 memory remove ``` diff --git a/docs/client-setup.md b/docs/client-setup.md index 032b0cf..7770e7f 100644 --- a/docs/client-setup.md +++ b/docs/client-setup.md @@ -1,146 +1,265 @@ # Client Setup -Full setup instructions for each AI client. This guide is about transport and wiring, not a different product mode: each client gets the same bounded conventions map first and local-pattern discovery second. For the quick-start summary, see [README.md](../README.md). +Codebase Context runs as a local MCP server: a tool your AI editor or command-line client can call while it works. Register it without a fixed folder, then let the agent supply its current repository through the `project` tool parameter. These examples use published npm `2.2.0`. -## Transport modes +Use Node.js 22 or newer for these recipes. The package declares Node 18+, but native dependency requirements are narrower than that broad declaration; these checks do not establish Node 18.0 compatibility. -| Mode | How it runs | When to use | -| ---- | ----------- | ------------ | -| **stdio** (default) | Process spawned by the client | One AI client, simple setup | -| **HTTP** | Long-lived server at `http://127.0.0.1:3100/mcp` | Multiple clients sharing one server | +The copyable registration commands target PowerShell or a POSIX terminal. Windows Command Prompt was not tested; OpenCode's quoted separator is shell-dependent. -Start the HTTP server: +## Current published-package recipes (updated 2026-10-06) -```bash -npx -y codebase-context --http # default port 3100 -npx -y codebase-context --http --port 4000 -``` +Claude Code, Codex CLI and OpenCode registration commands were checked again on October 6 without a project argument. The other clients retain their older config-shape evidence; no-folder native agent use is unverified for them. See the dated notes below. -Copy-pasteable templates: [`templates/mcp/stdio/.mcp.json`](../templates/mcp/stdio/.mcp.json) and [`templates/mcp/http/.mcp.json`](../templates/mcp/http/.mcp.json). +Each client section says how far its isolated Windows check went: -## Project routing contract +| Status | What it means | +| ------------------------------ | ------------------------------------------------------------------------------------------ | +| **Connected locally** | The isolated client started and connected to the server. | +| **Config recognized locally** | The isolated client found the config but still required approval before loading it. | +| **Config written locally** | The isolated command wrote the expected config. A live agent session was not claimed. | +| **Official instructions only** | The config follows current official docs, but this client was not fully exercised locally. | -Automatic multi-project routing is evidence-backed only when the MCP host announces workspace roots. Treat that as the primary path. +These checks cover setup and startup state. They do not measure how an agent handles a coding task. -If the host does not send roots, or still cannot tell which project is active, use one of the explicit fallbacks instead: +After registration, start a new agent session in your project and ask: -- start the server with a single bootstrap path -- set `CODEBASE_ROOT` -- retry tool calls with `project` +> Use Codebase Context to find [feature] in this repository. Pass this repository's absolute path as project when checking get_indexing_status and searching. Wait for indexing if needed, read codebase://context, then search_codebase and open a returned source file. Show me the relevant files. -If multiple projects are available and no active project can be inferred safely, the server returns `selection_required` instead of guessing. +The first index may take a while and download a local model. You can prepare it separately with `npx -y codebase-context@2.2.0 reindex` from the project folder. Index readiness, client connection and useful agent behavior are separate checks. ## Claude Code +**Status:** Connected locally + +**Checked:** 2026-10-06 with Claude Code 2.1.291 + +**Scope:** User scope makes the registration available across projects. The agent selects the repository in tool calls. + +User setup: + ```bash -claude mcp add codebase-context -- npx -y codebase-context +claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.2.0 ``` -Claude Code only supports stdio. HTTP is not available for this client. +Optional shared project setup: -## Claude Desktop +```bash +claude mcp add --scope project --transport stdio codebase-context -- npx -y codebase-context@2.2.0 +``` -Add to `claude_desktop_config.json`: +`--scope project` writes shared `.mcp.json` and requires project-server approval. `--scope local` instead stores a private association for the current project in `~/.claude.json`; it does not write `.mcp.json`. -```json -{ - "mcpServers": { - "codebase-context": { - "command": "npx", - "args": ["-y", "codebase-context"] - } - } -} +**What was checked:** With isolated user configuration, the no-folder user command wrote the expected stdio arguments and `claude mcp get` reported Connected. + +**Limit:** Connection proof does not establish an agent investigation. The optional shared-project variant follows the official scope contract; it was not exercised in this October check. + +**Official docs:** https://code.claude.com/docs/en/mcp + +## Codex CLI + +**Status:** Config written locally + +**Checked:** 2026-10-06 with Codex CLI 0.160.0 + +**Scope:** The command writes user config. A trusted project can use a project `.codex/config.toml` instead. + +```bash +codex mcp add codebase-context -- npx -y codebase-context@2.2.0 ``` -Claude Desktop only supports stdio. +Equivalent project config: -## Cursor +```toml +[mcp_servers.codebase-context] +command = "npx" +args = ["-y", "codebase-context@2.2.0"] +startup_timeout_sec = 120 +``` -**Stdio** — add to `.cursor/mcp.json` in your project (copy from [`templates/mcp/stdio/.mcp.json`](../templates/mcp/stdio/.mcp.json)): +**What was checked:** The command ran with an isolated `CODEX_HOME`, wrote `config.toml`, and `codex mcp get` returned the same command and arguments. -```json -{ - "mcpServers": { - "codebase-context": { - "command": "npx", - "args": ["-y", "codebase-context"] - } - } -} +**Limit:** The CLI command is user-scoped. A trusted project can instead use .codex/config.toml. The double dash is required. + +**Official docs:** [Codex MCP configuration](https://learn.chatgpt.com/docs/extend/mcp?surface=cli). The supported `startup_timeout_sec` option defaults to ten seconds; the project example allows 120 seconds for initial `npx` package preparation. This does not prove Desktop startup or guarantee completion of a cold download. + +## Codex Desktop: published 2.2.0 + +Create or merge `.codex/config.toml` in the project you want to search. Preserve any existing settings: + +```toml +[mcp_servers.codebase-context] +command = "npx" +args = ["-y", "codebase-context@2.2.0"] +startup_timeout_sec = 120 ``` -**HTTP** — start the server first, then add to `.cursor/mcp.json` (copy from [`templates/mcp/http/.mcp.json`](../templates/mcp/http/.mcp.json)): +Trust the project if asked, restart Codex and start a new task there. Use the first-use prompt above so the agent selects the current repository. The CLI command in the preceding section writes user-level config; this recipe uses a trusted project's config instead. -```json -{ - "mcpServers": { - "codebase-context": { - "type": "http", - "url": "http://127.0.0.1:3100/mcp" - } - } -} +**Verification boundary:** isolated Windows CLI and SDK/MCP checks on October 5 indexed a six-file fixture with published `2.2.0`, selected its project, retrieved context and found a source that the host could open. The exact `npx -y codebase-context@2.2.0 ` entrypoint also passed using a warmed isolated npm cache and prepared index. This does not establish a fully cold download or a fresh Codex Desktop agent session. The September Desktop session below used a packed candidate; keep it separate from published-package acceptance. + +On October 6 the exact published `npx` entrypoint was also tested without a folder, environment root or MCP roots, launched from an unrelated directory. Its first tool call returned `selection_required`; an explicit `project` call selected the prepared fixture, search found the expected symbol, context returned its map, and the runner host opened the returned source. A later call without `project` retained that selection. This is SDK/protocol and runner-host evidence with warmed caches and a pre-indexed fixture, not a native Desktop task. + +## Unreleased installer candidate + +The next release is expected to support one project-only command from the target project: + +The `init --client codex --yes` flow is an unreleased source candidate. Published npm `2.2.0` does not include these flags. Use the published configuration above rather than an `@latest` installer command; candidate verification uses an exact built or installed candidate. + +A successful `init --client codex --yes` writes the project-scoped `.codex/config.toml` and prepares the index for that root. If config writing or index preparation fails, setup is incomplete and Codex is not ready to use. + +For candidate verification before publication, build the source checkout and run its absolute `dist/index.js` path from the project you want to configure: + +```powershell +# In the codebase-context source checkout +npm run build + +# In the target project, using the built CLI path +node C:\path\to\codebase-context\dist\index.js init --client codex --yes ``` -## Windsurf +Use `--root C:\path\to\project` when the command is run from elsewhere. Here, `root` means the project folder to configure and index. The candidate writes the project-scoped `/.codex/config.toml`, preserves unrelated Codex settings, prepares the index for that folder, and does not write `AGENTS.md` or global client configuration. After a successful result, trust the project in Codex Desktop if prompted, restart or reopen Codex, start a new task in that project, and ask a small project question. A written config and ready index do not prove that Codex loaded or queried the server. -Open Settings > MCP and add (stdio only — HTTP is not documented for Windsurf yet): +Before release, the generated `npx` server command still resolves the published package unless the candidate is installed in the target project's `node_modules`. Our packaged test installs the candidate in a disposable project and checks its file hash. Running the built setup command alone does not prove that Codex will launch that same candidate. -```json -{ - "mcpServers": { - "codebase-context": { - "command": "npx", - "args": ["-y", "codebase-context"] - } - } -} +**Checked on 2026-09-15:** the packed candidate installed and indexed an isolated Windows project. A fresh Codex Desktop task then called the actual `search_codebase` MCP tool and located the requested chart component with its source file and lines. The normal Desktop task flow established project trust; no trust-file edits or replacement CLI/SDK search were used for that answer. This covers the small prepared project with shared caches, not a cold-machine install or the published npm package. + +## Optional fixed-folder setup and CLI preparation + +To set a default project before the first tool call, append its full path to the server arguments, for example `args = ["-y", "codebase-context@2.2.0", "C:/projects/my-app"]`. Use a quoted literal shell argument in registration commands. This makes the default fixed across repositories when the registration is user-scoped; explicit tool selection can still choose another allowed project. + +To prepare the index separately, run from the project folder: + +```powershell +npx -y codebase-context@2.2.0 reindex ``` -## Codex +Do not use published `2.2.0`'s interactive `init` as the setup route. October 6 isolated checks found that it ignores `--help`, `--client` and `--yes`; its Claude/Codex registration executes `mcp` instead of the client executable, and its HTTP recipes require a separately started server. It can exit zero after registration fails. The unreleased installer above is a separate implementation. + +## Gemini CLI + +**Status:** Config written locally + +**Checked:** 2026-08-11 with Gemini CLI 0.41.1 + +**Scope:** User or project. -**Stdio:** +User setup: ```bash -codex mcp add codebase-context npx -y codebase-context +gemini mcp add --scope user codebase-context npx -y codebase-context@2.2.0 ``` -**HTTP** — start the server first (`npx -y codebase-context --http`), then save a config file and pass it: +Project setup: + +```bash +gemini mcp add --scope project codebase-context npx -y codebase-context@2.2.0 +``` + +Gemini writes an `mcpServers` entry in its settings. Claude, Codex, and Copilot CLI use `--`; Gemini does not. + +**What was checked:** Both commands wrote the expected config in isolated homes and projects. The user-scoped list found the server as disconnected; no live agent connection was claimed. + +**Limit:** Gemini does not use the double-dash separator. The isolated project-scoped list did not enumerate the project entry, so only config generation is proved. + +**Official docs:** https://geminicli.com/docs/tools/mcp-server/ + +## Cursor + +**Status:** Config recognized locally + +**Checked:** 2026-08-11 with Cursor 3.15.6 and Cursor Agent 2026.04.17-787b533 + +**Scope:** Project. + +Create `.cursor/mcp.json`: ```json { "mcpServers": { "codebase-context": { - "type": "http", - "url": "http://127.0.0.1:3100/mcp" + "command": "npx", + "args": ["-y", "codebase-context@2.2.0"] } } } ``` -```bash -codex --mcp-config /path/to/mcp-http.json -``` +Checked config shape: `.cursor/mcp.json with mcpServers.codebase-context command npx and args [-y, codebase-context]`. + +**What was checked:** Cursor Agent loaded the isolated workspace config and showed the server pending approval. + +**Limit:** User approval is required. The official one-click route exists, but a generated deep link was not accepted as proof. + +**Official docs:** https://docs.cursor.com/context/model-context-protocol + +## VS Code with GitHub Copilot -## VS Code (Copilot) +**Status:** Official instructions only -Add `.vscode/mcp.json` to your project root. VS Code uses `servers` instead of `mcpServers`: +**Checked:** 2026-08-11 against VS Code 1.125.1 help and current official docs + +**Scope:** User command or workspace config; not locally runtime-tested. + +Official user command: + +```powershell +code --add-mcp '{"name":"codebase-context","command":"npx","args":["-y","codebase-context@2.2.0"]}' +``` + +Deterministic workspace config at `.vscode/mcp.json`: ```json { "servers": { "codebase-context": { "command": "npx", - "args": ["-y", "codebase-context"] + "args": ["-y", "codebase-context@2.2.0"] } } } ``` +**What was checked:** Local CLI help exposes `--add-mcp`. An isolated GUI launch could not provide deterministic completion, so it was not counted as a local runtime check. + +**Limit:** The command is official but not accepted as a completed local runtime test. Workspace config remains the deterministic manual fallback. + +**Official docs:** https://code.visualstudio.com/docs/agent-customization/mcp-servers + +## GitHub Copilot CLI + +**Status:** Config written locally + +**Checked:** 2026-08-11 with GitHub Copilot CLI 1.0.24 + +**Scope:** User config. + +```bash +copilot mcp add codebase-context -- npx -y codebase-context@2.2.0 +``` + +**What was checked:** The command ran with an isolated `--config-dir` and wrote the expected `mcp-config.json` command, arguments, and tool filter. + +**Limit:** The CLI writes user configuration by default. VS Code workspace mcp.json is a separate surface. + +**Official docs:** https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/add-mcp-servers + ## OpenCode -Add `opencode.json` to your project root: +**Status:** Connected locally + +**Checked:** 2026-10-06 with stable OpenCode 1.18.34 + +**Scope:** User registration by default. + +Run once: + +```bash +opencode mcp add codebase-context '--' npx -y codebase-context@2.2.0 +``` + +Keep the quoted `'--'`: the installed Windows PowerShell npm shim consumes an unquoted separator. The quoted form also works in POSIX shells. + +For a project-local alternative, merge this into `opencode.json`: ```json { @@ -148,73 +267,103 @@ Add `opencode.json` to your project root: "mcp": { "codebase-context": { "type": "local", - "command": ["npx", "-y", "codebase-context"], - "enabled": true + "command": ["npx", "-y", "codebase-context@2.2.0"], + "enabled": true, + "timeout": 120000 } } } ``` -OpenCode also supports interactive setup via `opencode mcp add`. +Checked config shape: `opencode.json with a local command array and timeout 120000`. -## Single-project fallback +The longer timeout is intentional because the isolated cold start took about 38 seconds while `npx` prepared the package and server. -If you only use one repo, append a project path: +**What was checked:** With home, config, data, cache, temporary and workspace paths redirected, the no-folder command wrote user configuration and `opencode mcp list` reported Connected. An August 11 check with OpenCode 1.18.5 also connected using the project config shape and an explicit folder; the no-folder project variant is not a new native acceptance check. -```bash -codex mcp add codebase-context npx -y codebase-context "/path/to/your/project" -``` +**Limit:** Stable OpenCode 1.x and beta OpenCode 2 use different native config shapes. The stable V1 shape is the public default; do not mix schemas. -Or set an environment variable: +**Official docs:** https://opencode.ai/docs/mcp-servers/ -```bash -CODEBASE_ROOT=/path/to/your/project -``` +## Windsurf -## Test a local build +**Status:** Official instructions only -Build the local branch first: +**Checked:** 2026-08-11 against current official docs; Windsurf was not installed locally -```bash -pnpm build -``` +**Scope:** User config. -Then point your MCP client at the local build: +Create `~/.codeium/windsurf/mcp_config.json`: ```json { "mcpServers": { "codebase-context": { - "command": "node", - "args": ["/dist/index.js"] + "command": "npx", + "args": ["-y", "codebase-context@2.2.0"] } } } ``` -If the default setup is not enough for your client, pass a project path explicitly: +Checked config shape: `~/.codeium/windsurf/mcp_config.json with mcpServers.codebase-context command npx and args [-y, codebase-context]`. + +**What was checked:** The config shape was checked against the current official page. The local client was unavailable. + +**Limit:** Current ownership and local behavior are not independently verified on this machine; publish as official config, not locally tested. + +**Official docs:** https://docs.windsurf.com/windsurf/cascade/mcp + +## Optional advanced HTTP server + +Use `stdio` for the simplest setup. HTTP can let several clients share one long-running local process, but support depends on the client and this path is not part of the default setup proof: + +```bash +npx -y codebase-context@2.2.0 --http +npx -y codebase-context@2.2.0 --http --port 4000 +``` + +The documented default endpoint is `http://127.0.0.1:3100/mcp`. Config-shape templates are available in [`templates/mcp/stdio/.mcp.json`](../templates/mcp/stdio/.mcp.json) and [`templates/mcp/http/.mcp.json`](../templates/mcp/http/.mcp.json); pin the published package as in the recipes above. HTTP does not discover a client's project folder; the agent still has to select its repository. + +The [MCP July 28, 2026 HTTP revision](https://modelcontextprotocol.io/specification/2026-07-28/basic/transports/streamable-http) removed protocol sessions and the GET stream endpoint. This source checkout uses SDK 1.27.1 and the earlier protocol/session model. The stdio checks do not validate its concurrent HTTP project isolation or compatibility with the new revision. A protocol migration and host testing are separate from these setup recipes. + +Client support for HTTP differs. Follow that client's official docs and keep stdio as the fallback. + +## Project routing contract + +The documented default is agent-directed selection. Start the stdio server without a folder; call `get_indexing_status` with the current repository's absolute path as `project`, wait for readiness and pass the same `project` when searching. A successful selection becomes the default for later calls in that server process, including `codebase://context`. Select another project explicitly when changing repositories. This lets one registration serve multiple projects without changing client configuration. + +CBC deliberately does not index the server process's working directory when no project is configured. Without a configured folder, known root or explicit selection, tools return `selection_required`. They do not guess or index the user's home directory. + +Some clients also announce workspace roots. One valid root can auto-select; several roots with no active selection return `selection_required` and available projects. Explicit selectors must stay within announced roots. Relative monorepo paths require a known workspace context; use absolute repository paths for the portable first-use recipe. + +[MCP deprecated Roots on July 28, 2026](https://modelcontextprotocol.io/specification/2026-07-28/client/roots), recommending explicit directories/files in tool parameters, resource URIs or server configuration. CBC's older Roots support remains a compatibility aid rather than a required setup step. + +October 6 verification: 18 focused current-source routing tests passed, including ambiguous roots, explicit selection, subsequent calls and root changes. The exact published server also switched between two isolated prepared repositories A to B to A in one no-roots stdio process: search paths, context map and subsequent status calls followed each selection. The runner host opened the expected distinct sources. A native agent session across real repositories remains a validation step; stdio process-local selection does not establish concurrent HTTP isolation. + +A follow-up recorded raw `search_codebase` calls containing only `query` and `mode`, with no `project`: they returned A, B and A after those explicit selections. The caller does not automatically add the full path; CBC routes omitted-project calls using its selected project. This state lasts for that stdio process. Select again after a server restart or when changing repositories. + +A native Codex CLI controlled-fixture trial with an ordinary question supplied the correct absolute `project` without path instructions. Its MCP call was denied by the noninteractive run's `never` approval policy, so native retrieval and continuation did not pass. If a host refuses a tool call for approval, repeating the folder path will not resolve that permission issue; use the client's trust/approval flow. The test retained the denial and did not count fallback source/index reads as CBC acceptance. + +## Test a local build + +Build the source checkout: + +```bash +pnpm build +``` + +Then replace the `npx` command in your client config with the local entry point: ```json { "mcpServers": { "codebase-context": { "command": "node", - "args": ["/dist/index.js", "/path/to/your/project"] + "args": ["/dist/index.js"] } } } ``` -Check these three flows: - -1. **Single project** — call `search_codebase` or `metadata`. Routing is automatic. - -2. **Multiple projects on a roots-capable host** — open two repos or a monorepo. Call `codebase://context`. Expected: workspace overview, then automatic routing once a project is active. - -3. **Ambiguous or no-roots selection** — start without a bootstrap path, call `search_codebase`. Expected: `selection_required`. Retry with `project` set to `apps/dashboard` or `/repos/customer-portal`. - -For monorepos, test all three selector forms: - -- relative subproject path: `apps/dashboard` -- repo path: `/repos/customer-portal` -- file path: `/repos/monorepo/apps/dashboard/src/auth/guard.ts` +For one explicit project, append its path to the arguments or set `CODEBASE_ROOT`. diff --git a/docs/comparison-table.md b/docs/comparison-table.md index aee2eaa..1b55f11 100644 --- a/docs/comparison-table.md +++ b/docs/comparison-table.md @@ -1,35 +1,26 @@ -# Comparator Summary +# Corrected retrieval comparison -This table summarizes the current comparator evidence from `results/comparator-evidence.json`. -It is a setup-status table first, not a marketing scoreboard. +This table summarizes the corrected retrieval-only family across five local code-context tools. In this fixed setup, Codebase Context recovered 25.7% of expected gold files at 11.5% file precision; this does not show better completed coding tasks. Expected-file coverage means expected gold files recovered divided by expected gold files. File precision means expected gold files recovered divided by all files returned. The reported `peakPrivateGb` field is copied from retained telemetry; its underlying unit was not independently verified. -| Comparator | Intended role in gate | Current status | Evidence summary | -| --- | --- | --- | --- | -| `raw Claude Code` | Baseline for payload cost and at least one usefulness comparison | comparator artifact: `ok`; gate: `pending_evidence` | The Haiku-backed Claude CLI runner now returns current payloads, but the checked-in baseline still has `averageFirstRelevantHit: null`, so the gate still records missing baseline metrics. | -| `GrepAI` | Named MCP comparator | `setup_failed` | Requires the GrepAI binary plus a local Ollama embedding setup that is not present in this proof environment. | -| `jCodeMunch` | Named MCP comparator | `setup_failed` | The MCP server still closes on startup during the current rerun, so no comparable discovery metrics were produced. | -| `codebase-memory-mcp` | Named MCP comparator | comparator artifact: `ok`; gate: `failed` | The repaired graph-backed runner now produces real current metrics, but the frozen gate still fails this lane because `codebase-context` does not stay within tolerance on every required usefulness metric. | -| `CodeGraphContext` | Graph-native comparator in the relaunch frame | `setup_failed` | The MCP server still closes on startup during the current rerun, so this lane remains missing evidence. | +| Tool | Completed | Failed | Expected-file coverage | File precision | Reported `peakPrivateGb` | +| --- | ---: | ---: | ---: | ---: | ---: | +| Raw search | 20/20 | 0 | 4.3% | 4.0% | 0.06 | +| Codebase Context | 20/20 | 0 | 25.7% | 11.5% | 2.90 | +| jCodeMunch | 20/20 | 0 | 27.1% | 11.0% | 0.33 | +| Repowise | 19/20 | 1 | 20.1% (failed run = 0) | 12.0% | 4.82 | +| context-mode | 20/20 | 0 | 16.8% | 11.7% | 0.15 | -## Reading This Table +## Reading notes -- `setup_failed` means the lane was attempted and did not reach a credible metric-producing state. -- `pending_evidence` in the gate means the lane is still missing one or more required metrics. -- `failed` in the gate means the lane has real metrics, but the frozen comparison rule still does not pass. -- A missing metric is not treated as a win for `codebase-context`. -- The combined gate in `results/gate-evaluation.json` remains `pending_evidence`, and `claimAllowed` stays `false`, until these lanes produce real metrics. +- Expected-file coverage is the percentage of expected gold files recovered, averaged within the corrected retrieval-only family. +- File precision is the percentage of returned files that matched the expected gold files. +- The table covers 100 attempts: 99 completed and 1 failed. Repowise completed 19 of 20 attempts; the other tools completed 20 of 20. +- Repowise's 20.1% is failure-inclusive: its failed attempt is counted as zero. +- Raw search is a deterministic lexical adapter in this protocol, not a full normal coding-agent baseline. +- `peakPrivateGb` is a reported telemetry field, not a verified unit claim, install size, or steady-state idle memory. +- The full-agent pilot, paired token observations, repeated retrieval history, and metered replay are separate evidence families. Their values should not be added to this table; the replay remains excluded while tool-use validity is unresolved. +- The measurements do not establish patch correctness or end-to-end task completion. They also do not establish a universal token, time, or coding result. -## Current codebase-context result +The [sanitized evidence extract](../results/benchmark-presentation-evidence.json) records the exact values and source hashes. -For reference, the current combined discovery output across `angular-spotify` and `excalidraw` is: - -| Metric | codebase-context | -| --- | ---: | -| `totalTasks` | 24 | -| `averageUsefulness` | 0.75 | -| `averagePayloadBytes` | 7306.4583 | -| `averageEstimatedTokens` | 1827.0833 | -| `bestExampleUsefulnessRate` | 0.125 | -| `gate.status` | `pending_evidence` | - -Those numbers are not compared here as head-to-head wins because the comparator lanes above did not produce matching metrics. +Historical note: the [earlier public report archive](./benchmark-prior-public-report.md) contains a one-task official pilot and a separate 24-task discovery report; it remains separate and keeps its existing claim gates. diff --git a/docs/demo.md b/docs/demo.md index 825b2ce..8b06f23 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -1,13 +1,23 @@ # Demo Script -This walkthrough uses real CLI output captured against `repos/angular-spotify` during the Phase 10 proof rerun. -Run it from the repo root with `CODEBASE_ROOT` pointed at the frozen sample repo. The public flow is simple: start with the conventions map, then search for the local example you need. +This walkthrough shows real CLI output captured from the open-source `angular-spotify` repository during a local proof run. That sample checkout is not bundled here. + +To try the same CLI flow, use Node.js 22 or newer and open a terminal in the repository you want to inspect. These commands use published `codebase-context@2.2.0`; the CLI uses the current directory as the project root. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). + +The saved output below is from an earlier CLI run, not the published-package commands shown here. The machine-specific repository root is omitted from the returned file path. This capture does not verify the current package or an MCP connection. + +## 0. Build The Local Index + +```bash +npx -y codebase-context@2.2.0 reindex +``` + +Run this before the first map or search in a repository. Later runs can use `reindex --incremental` after the code changes. ## 1. Start With The Conventions Map ```bash -$env:CODEBASE_ROOT='C:\Users\bitaz\Repos\codebase-context\repos\angular-spotify' -node dist/index.js map --json +npx -y codebase-context@2.2.0 map --json ``` Captured output excerpt: @@ -40,8 +50,7 @@ What this shows: ## 2. Search With Edit Intent ```bash -$env:CODEBASE_ROOT='C:\Users\bitaz\Repos\codebase-context\repos\angular-spotify' -node dist/index.js search --query "auth headers" --intent edit --limit 3 --json +npx -y codebase-context@2.2.0 search --query "auth headers" --intent edit --limit 3 --json ``` Captured output excerpt: @@ -66,7 +75,7 @@ Captured output excerpt: }, "results": [ { - "file": "C:\\Users\\bitaz\\Repos\\codebase-context\\repos\\angular-spotify\\libs\\web\\auth\\util\\src\\lib\\interceptors\\auth.interceptor.ts:10-42", + "file": "repos/angular-spotify/libs/web/auth/util/src/lib/interceptors/auth.interceptor.ts:10-42", "type": "interceptor:core" } ] @@ -82,8 +91,7 @@ What this shows: ## 3. Check A Team Pattern Directly ```bash -$env:CODEBASE_ROOT='C:\Users\bitaz\Repos\codebase-context\repos\angular-spotify' -node dist/index.js patterns --category state --json +npx -y codebase-context@2.2.0 patterns --category state --json ``` Captured output excerpt: @@ -117,4 +125,5 @@ What this shows: ## Caveats - These excerpts were captured from the current local proof run and will change if the frozen sample repo or index state changes. -- The discovery benchmark gate is still `pending_evidence`, and `claimAllowed` remains `false`, so this walkthrough demonstrates shipped behavior, not a released performance claim. +- File paths in the excerpts belong to that capture; your output will use the repository you run the command from. +- The benchmark documents observed local evidence and its limitations; this walkthrough demonstrates shipped behavior, not a universal performance or coding result. diff --git a/results/benchmark-presentation-evidence.json b/results/benchmark-presentation-evidence.json new file mode 100644 index 0000000..6f3e44f --- /dev/null +++ b/results/benchmark-presentation-evidence.json @@ -0,0 +1,1251 @@ +{ + "schemaVersion": 1, + "artifactType": "benchmark-presentation-source-extract", + "description": "Allowlisted fields extracted from retained reports. This is not a new benchmark run or independent validation of those reports. Full source byte hashes identify the inputs; private filesystem paths and orchestration costs are omitted.", + "provenance": { + "corrected": { + "file": "contextbench-engineering-note-summary.json", + "sha256": "ad8cac1ad86747463212b1665c3458e8f9560fbcde728d3dc2d74d1d6fe3e17f", + "generatedAt": "2026-06-04T08:28:24Z", + "schemaVersion": 2, + "gitRef": "18e0a43a90fe67e9df0d728b2134e9d430469883", + "gitBlobSha256": "a26599e1302ebdbb34e259530c8d775a9325bd42714e5b03398c545ad1abaf6c", + "checkoutByteDifference": "Source copy uses mixed CRLF/LF; normalizing line endings yields the Git blob byte-for-byte (LF)." + }, + "history": { + "file": "contextbench-engineering-note-summary.json", + "sha256": "29c1a03bff051d9bc90284c5585e4f2cc1e94f9cc9da5d01d1b36e73dac4e87e", + "generatedAt": "2026-06-07T19:09:33Z", + "schemaVersion": 4, + "gitRef": "67b7ae2d1541eef3f591a9a58daf5d09ead8dcf1", + "gitBlobSha256": "29c1a03bff051d9bc90284c5585e4f2cc1e94f9cc9da5d01d1b36e73dac4e87e", + "checkoutByteDifference": "CRLF versus LF only" + }, + "pilot": { + "file": "REPORT.claim-session.json", + "sha256": "1c3368185c01a5e52bb7ba55466f9e5a053a26afe896a60b16b80c0743a5bd69", + "generatedAt": "2026-06-12T02:23:14.817Z", + "schemaVersion": 1, + "sessionId": "claim-bearing-2task-5lane-repeat3-claimclean2e21-20260612T002612Z" + } + }, + "correctedRetrieval": { + "sourceFields": [ + "rows", + "qualitySnapshot", + "costSnapshot" + ], + "publicationReadiness": { + "claimBearingPublicationAllowed": false, + "claimScope": "20x1_retrieval_overlap_only", + "unresolvedBlockers": [ + "single_repeat_no_stability_claim", + "retrieval_overlap_only_no_patch_quality", + "provider_dollar_cost_not_measured_because_no_paid_provider_use", + "repowise_one_no_context_evaluator_failure", + "cold_vs_warm_install_dependency_cache_not_fully_normalized" + ], + "publicUseLevel": "narrow_retrieval_overlap_case_study", + "status": "engineering_note_ready_with_claim_blocks", + "blockedClaimTypes": [ + "benchmark winner", + "best tool", + "paper-grade benchmark", + "resource-efficiency claim", + "generalized agent coding quality claim", + "agent patch correctness claim", + "developer productivity claim" + ] + }, + "rows": { + "evaluatorFailedRows": 1, + "repeatsPerTaskLane": 1, + "lanes": 5, + "evaluatorCompletedRows": 99, + "retrievalOverlapRows": 100, + "appendixOnlyLanes": [ + { + "reason": "bounded local indexing readiness failed earlier; not part of this corrected 20x1 headline table", + "laneId": "GrepAI" + } + ], + "sameDenominatorRowsPresent": true, + "tasks": 20 + }, + "qualitySnapshot": [ + { + "laneId": "codebase-context", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.25722222222222224, + "filePrecisionMean": 0.11499999999999999, + "symbolCoverageMean": 0.1396239637555427, + "spanCoverageMean": 0.08573378715275093, + "lineCoverageMean": 0.0878787249873908 + }, + { + "laneId": "context-mode", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.16833333333333333, + "filePrecisionMean": 0.11666666666666665, + "symbolCoverageMean": 0.02731443994601889, + "spanCoverageMean": 0.011708371581594209, + "lineCoverageMean": 0.01103832178724339 + }, + { + "laneId": "jcodemunch-full-mcp", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.2713888888888889, + "filePrecisionMean": 0.11000000000000001, + "symbolCoverageMean": 0.19046586746586747, + "spanCoverageMean": 0.08523281976808295, + "lineCoverageMean": 0.08706990517203586 + }, + { + "laneId": "raw-native", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.043333333333333335, + "filePrecisionMean": 0.04, + "symbolCoverageMean": 0.007186234817813765, + "spanCoverageMean": 0.00461426238619191, + "lineCoverageMean": 0.003664930124865624 + }, + { + "laneId": "repowise-selfhosted-mcp", + "rows": 20, + "evaluatorCompletedRows": 19, + "status": "completed:19,judge_failed:1", + "fileCoverageMean": 0.2013888888888889, + "filePrecisionMean": 0.12, + "symbolCoverageMean": 0.046063588114671705, + "spanCoverageMean": 0.03033601885780799, + "lineCoverageMean": 0.024685983349898594 + } + ], + "costSnapshot": [ + { + "laneId": "codebase-context", + "setupStatuses": "not_required:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 750.8, + "indexSumMinutes": 226.6, + "rowWallSumMinutes": 231.5, + "peakPrivateGb": 2.9, + "modelEvidence": "Xenova/bge-small-en-v1.5:20" + }, + { + "laneId": "context-mode", + "setupStatuses": "completed:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 18.6, + "indexSumMinutes": 12.5, + "rowWallSumMinutes": 16, + "peakPrivateGb": 0.15, + "modelEvidence": "unknown:20" + }, + { + "laneId": "jcodemunch-full-mcp", + "setupStatuses": "completed:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 11.3, + "indexSumMinutes": 4.9, + "rowWallSumMinutes": 10.9, + "peakPrivateGb": 0.33, + "modelEvidence": "unknown:20" + }, + { + "laneId": "raw-native", + "setupStatuses": "not_required:20", + "indexStatuses": "not_required:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 0, + "indexSumMinutes": 0, + "rowWallSumMinutes": 6.8, + "peakPrivateGb": 0.06, + "modelEvidence": "unknown:20" + }, + { + "laneId": "repowise-selfhosted-mcp", + "setupStatuses": "completed:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "1:20", + "indexMedianSeconds": 294.3, + "indexSumMinutes": 154.2, + "rowWallSumMinutes": 200.7, + "peakPrivateGb": 4.82, + "modelEvidence": "unknown:20" + } + ], + "failedRows": [ + { + "taskId": "Multi-SWE-Bench__typescript__maintenance__bugfix__8d2ac221", + "laneId": "repowise-selfhosted-mcp", + "status": "judge_failed", + "reason": "official_result_error:no_context_extracted" + } + ] + }, + "repeatedRetrievalAndReplayHistory": { + "sourceFields": [ + "retrievalChecks.sourceRuns", + "publicationReadiness" + ], + "sourceRuns": [ + { + "name": "corrected-20x1", + "checks": 100, + "checksScored": 99, + "checksWithNoUsableContext": 1 + }, + { + "name": "repeat-20x3", + "checks": 300, + "checksScored": 288, + "checksWithNoUsableContext": 12 + }, + { + "name": "metered-context-consumption", + "checks": 100, + "checksScored": 100, + "checksWithNoUsableContext": 0, + "tokenCountersCaptured": 100 + } + ], + "publicationReadiness": { + "claimBearingPublicationAllowed": false, + "claimScope": "500_check_practical_code_context_case_study", + "unresolvedBlockers": [ + "combined_local_evidence_sets_not_preregistered_repeated_protocol", + "retrieval_overlap_and_context_consumption_only_no_patch_quality", + "token_metering_is_context_consumption_not_patch_workflow", + "provider_dollar_cost_not_calculated_from_model_pricing", + "repowise_thirteen_checks_with_no_usable_context", + "cold_vs_warm_install_dependency_cache_not_fully_normalized" + ], + "publicUseLevel": "narrow_code_context_case_study", + "status": "engineering_note_ready_with_claim_blocks", + "blockedClaimTypes": [ + "benchmark winner", + "best tool", + "research-paper benchmark", + "resource-efficiency claim", + "generalized agent coding quality claim", + "agent patch correctness claim", + "developer productivity claim", + "token-savings claim" + ] + }, + "replayCaution": "The historical schema4 summary records completed metering; the later presentation describes no-tool-use replay exclusion. This extract does not reconcile the validity of that replay. No replay token results are presented." + }, + "twoTaskAgentPilot": { + "publicationReadiness": { + "construction": "Publication readiness is a claim gate derived from existing report evidence. It is not owner approval and not a new benchmark score.", + "status": "not_publication_ready", + "publicUseLevel": "engineering_note_or_internal_audit_only", + "claimBearingPublicationAllowed": false, + "claimScope": "pilot_only", + "qualityRowCount": 30, + "qualityTaskCount": 2, + "scoreableQualityLaneCount": 5, + "missingQualityLaneCount": 0, + "missingQualityLaneIds": [], + "reliabilityRowCount": 0, + "repeatConfidenceStatus": "repeat_ready", + "resourceTelemetryGapRows": [], + "unresolvedBlockers": [ + "pilot_task_count" + ], + "blockerActions": [ + { + "blocker": "pilot_task_count", + "action": "Add scoreable quality rows across the required task count before broad benchmark claims." + } + ], + "allowedNonClaimUses": [ + "Describe scoreable quality rows only within their narrow denominator." + ], + "blockedClaimTypes": [ + "benchmark winner", + "paper-grade benchmark", + "four-lane quality comparison", + "resource-efficiency claim", + "generalized agent coding quality claim" + ] + }, + "summary": { + "totalRows": 30, + "taskCount": 2, + "qualityTaskCount": 2, + "laneCount": 5, + "qualityRowCount": 30, + "reliabilityRowCount": 0, + "statusCounts": { + "completed": 30 + }, + "taskIds": [ + "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27" + ], + "qualityTaskIds": [ + "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27" + ], + "laneIds": [ + "codebase-context", + "context-mode", + "jcodemunch-full-mcp", + "raw-native", + "repowise-selfhosted-mcp" + ] + }, + "sourceFields": [ + "qualityRows", + "costRows" + ], + "qualityRows": [ + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.475, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.5, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.3619047619047619, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.4, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.475, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.5, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0.02857142857142857, + "symbolPrecision": 0.16666666666666666, + "spanCoverage": 0.00467177045029839, + "spanPrecision": 0.5272108843537415, + "lineCoverage": 0.002026342451874367, + "linePrecision": 0.4, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.3333333333333333, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.2923076923076923, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.3333333333333333, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.475, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.5, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.4, + "symbolCoverage": 0.10526315789473684, + "symbolPrecision": 0.4, + "spanCoverage": 0.006640106241699867, + "spanPrecision": 0.5272727272727272, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.4, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.3287037037037037, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.25, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.3287037037037037, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.25, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.3287037037037037, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.25, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 1, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 1, + "lineCoverage": 0.002398081534772182, + "linePrecision": 1, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.6666666666666666, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.6909090909090909, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.6666666666666666, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.4, + "symbolCoverage": 0.05263157894736842, + "symbolPrecision": 0.25, + "spanCoverage": 0.006731693913999175, + "spanPrecision": 0.5856573705179283, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.4, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.475, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.5, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.6666666666666666, + "symbolCoverage": 0.10526315789473684, + "symbolPrecision": 0.6666666666666666, + "spanCoverage": 0.004258826761917846, + "spanPrecision": 0.7560975609756098, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.6666666666666666, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.6666666666666666, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.7169811320754716, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.6666666666666666, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.3333333333333333, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.21385542168674698, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.16666666666666666, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.3287037037037037, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.25, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.6666666666666666, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.6909090909090909, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.6666666666666666, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.475, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.5, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.25, + "filePrecision": 0.3333333333333333, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0009158767229930851, + "spanPrecision": 0.31746031746031744, + "lineCoverage": 0.001199040767386091, + "linePrecision": 0.3333333333333333, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.6666666666666666, + "symbolCoverage": 0.02857142857142857, + "symbolPrecision": 0.2, + "spanCoverage": 0.0028633431792151427, + "spanPrecision": 0.6643356643356644, + "lineCoverage": 0.002026342451874367, + "linePrecision": 0.6666666666666666, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0.02857142857142857, + "symbolPrecision": 0.14285714285714285, + "spanCoverage": 0.0028633431792151427, + "spanPrecision": 0.521978021978022, + "lineCoverage": 0.002026342451874367, + "linePrecision": 0.4, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + } + ], + "costRows": [ + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 651991, + "cachedInputTokens": 562688, + "outputTokens": 8569, + "reasoningOutputTokens": 5477, + "taskWallTimeMs": 158527, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 682131, + "cachedInputTokens": 642048, + "outputTokens": 10057, + "reasoningOutputTokens": 5312, + "taskWallTimeMs": 224679, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 400544, + "cachedInputTokens": 317952, + "outputTokens": 6954, + "reasoningOutputTokens": 4131, + "taskWallTimeMs": 131028, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 347800, + "cachedInputTokens": 314368, + "outputTokens": 13393, + "reasoningOutputTokens": 9227, + "taskWallTimeMs": 213283, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 273980, + "cachedInputTokens": 244224, + "outputTokens": 7323, + "reasoningOutputTokens": 4019, + "taskWallTimeMs": 140876, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 241423, + "cachedInputTokens": 206848, + "outputTokens": 5932, + "reasoningOutputTokens": 3550, + "taskWallTimeMs": 119511, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 428418, + "cachedInputTokens": 398336, + "outputTokens": 7056, + "reasoningOutputTokens": 2584, + "taskWallTimeMs": 150948, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 230103, + "cachedInputTokens": 200192, + "outputTokens": 5962, + "reasoningOutputTokens": 3079, + "taskWallTimeMs": 153087, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 725790, + "cachedInputTokens": 665088, + "outputTokens": 9735, + "reasoningOutputTokens": 6607, + "taskWallTimeMs": 197729, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 324871, + "cachedInputTokens": 296448, + "outputTokens": 9513, + "reasoningOutputTokens": 5744, + "taskWallTimeMs": 170065, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 224215, + "cachedInputTokens": 177664, + "outputTokens": 4517, + "reasoningOutputTokens": 1669, + "taskWallTimeMs": 115595, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 263549, + "cachedInputTokens": 243200, + "outputTokens": 10605, + "reasoningOutputTokens": 7467, + "taskWallTimeMs": 170770, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 223558, + "cachedInputTokens": 190976, + "outputTokens": 4990, + "reasoningOutputTokens": 2899, + "taskWallTimeMs": 110708, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 269851, + "cachedInputTokens": 234496, + "outputTokens": 6028, + "reasoningOutputTokens": 3024, + "taskWallTimeMs": 121854, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 205432, + "cachedInputTokens": 130048, + "outputTokens": 5564, + "reasoningOutputTokens": 3530, + "taskWallTimeMs": 114664, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 215407, + "cachedInputTokens": 167424, + "outputTokens": 7930, + "reasoningOutputTokens": 4784, + "taskWallTimeMs": 143643, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 247868, + "cachedInputTokens": 206848, + "outputTokens": 5463, + "reasoningOutputTokens": 3246, + "taskWallTimeMs": 106404, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 534029, + "cachedInputTokens": 498688, + "outputTokens": 13790, + "reasoningOutputTokens": 10169, + "taskWallTimeMs": 238528, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 541233, + "cachedInputTokens": 497152, + "outputTokens": 9011, + "reasoningOutputTokens": 4494, + "taskWallTimeMs": 185061, + "contextToolCallCount": null, + "contextToolCompletedCallCount": null + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 856202, + "cachedInputTokens": 798720, + "outputTokens": 12581, + "reasoningOutputTokens": 7297, + "taskWallTimeMs": 257215, + "contextToolCallCount": null, + "contextToolCompletedCallCount": null + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 653348, + "cachedInputTokens": 610304, + "outputTokens": 6730, + "reasoningOutputTokens": 3640, + "taskWallTimeMs": 179577, + "contextToolCallCount": null, + "contextToolCompletedCallCount": null + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 826946, + "cachedInputTokens": 740864, + "outputTokens": 17144, + "reasoningOutputTokens": 12215, + "taskWallTimeMs": 294443, + "contextToolCallCount": null, + "contextToolCompletedCallCount": null + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 251601, + "cachedInputTokens": 220672, + "outputTokens": 7386, + "reasoningOutputTokens": 3991, + "taskWallTimeMs": 141981, + "contextToolCallCount": null, + "contextToolCompletedCallCount": null + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 290186, + "cachedInputTokens": 272384, + "outputTokens": 7567, + "reasoningOutputTokens": 4024, + "taskWallTimeMs": 142002, + "contextToolCallCount": null, + "contextToolCompletedCallCount": null + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 304532, + "cachedInputTokens": 271872, + "outputTokens": 7060, + "reasoningOutputTokens": 4975, + "taskWallTimeMs": 146469, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 610092, + "cachedInputTokens": 546304, + "outputTokens": 8915, + "reasoningOutputTokens": 5830, + "taskWallTimeMs": 236889, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 541331, + "cachedInputTokens": 480256, + "outputTokens": 10847, + "reasoningOutputTokens": 5649, + "taskWallTimeMs": 204901, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 182974, + "cachedInputTokens": 167424, + "outputTokens": 9561, + "reasoningOutputTokens": 7486, + "taskWallTimeMs": 146629, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 333030, + "cachedInputTokens": 292352, + "outputTokens": 12256, + "reasoningOutputTokens": 8613, + "taskWallTimeMs": 198105, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 518028, + "cachedInputTokens": 475136, + "outputTokens": 10446, + "reasoningOutputTokens": 6931, + "taskWallTimeMs": 200453, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + } + ], + "aggregation": "Per-task, per-lane median of three costRows values for each counter independently. Cached input is a subset of input; component medians need not sum." + } +} diff --git a/results/contextbench-engineering-note-summary.json b/results/contextbench-engineering-note-summary.json new file mode 100644 index 0000000..03b0ebb --- /dev/null +++ b/results/contextbench-engineering-note-summary.json @@ -0,0 +1,213 @@ +{ + "schemaVersion": 2, + "artifactType": "contextbench-engineering-note-summary", + "generatedAt": "2026-06-04T08:28:24Z", + "source": { + "scorerScriptSha256": "sha256:13e859f6485735d76b1591a45a7211338f9fed9ff5566776ac8455db6e820e79", + "localSummary": "gate5-official-retrieval-scoring-current-script-20260604T1027/GATE5-SCORING-SUMMARY.json", + "localSummarySha256": "sha256:e5ecd3ee45b9577ae32d409de960edc1a7c2761159280cb7f8e7b7a85555387b", + "boundary": "The full run artifacts are local evidence. This checked-in summary is the public engineering-note artifact and does not make winner, leaderboard, paper-grade, resource-efficiency, or patch-quality claims.", + "localRunRoot": "benchmark-runs/contextbench/phase41/corrected-20x1-20260603T103632", + "localRunRootTracked": false + }, + "publicationReadiness": { + "claimBearingPublicationAllowed": false, + "claimScope": "20x1_retrieval_overlap_only", + "unresolvedBlockers": [ + "single_repeat_no_stability_claim", + "retrieval_overlap_only_no_patch_quality", + "provider_dollar_cost_not_measured_because_no_paid_provider_use", + "repowise_one_no_context_evaluator_failure", + "cold_vs_warm_install_dependency_cache_not_fully_normalized" + ], + "publicUseLevel": "narrow_retrieval_overlap_case_study", + "status": "engineering_note_ready_with_claim_blocks", + "blockedClaimTypes": [ + "benchmark winner", + "best tool", + "paper-grade benchmark", + "resource-efficiency claim", + "generalized agent coding quality claim", + "agent patch correctness claim", + "developer productivity claim" + ] + }, + "rows": { + "evaluatorFailedRows": 1, + "repeatsPerTaskLane": 1, + "lanes": 5, + "evaluatorCompletedRows": 99, + "retrievalOverlapRows": 100, + "appendixOnlyLanes": [ + { + "reason": "bounded local indexing readiness failed earlier; not part of this corrected 20x1 headline table", + "laneId": "GrepAI" + } + ], + "sameDenominatorRowsPresent": true, + "tasks": 20 + }, + "methodology": { + "notMeasured": "patch correctness, developer productivity, provider-dollar spend, multi-repeat confidence intervals", + "scoring": "official ContextBench evaluator on saved retrieval trajectories using per-task gold; scorer gold was not read during retrieval", + "lanes": [ + "raw-native", + "codebase-context", + "jcodemunch-full-mcp", + "repowise-selfhosted-mcp", + "context-mode" + ], + "measured": "retrieval overlap, setup/index time, row wall time, process memory telemetry, GPU availability probe, failure rows", + "metricView": "failure-inclusive means; judge_failed/no_context rows count as zero in the public summary", + "publicFrame": "we benchmarked these tools on open-source repos", + "taskSet": "20 frozen ContextBench-style open-source tasks" + }, + "qualitySnapshot": [ + { + "laneId": "codebase-context", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.25722222222222224, + "filePrecisionMean": 0.11499999999999999, + "symbolCoverageMean": 0.1396239637555427, + "spanCoverageMean": 0.08573378715275093, + "lineCoverageMean": 0.0878787249873908 + }, + { + "laneId": "context-mode", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.16833333333333333, + "filePrecisionMean": 0.11666666666666665, + "symbolCoverageMean": 0.02731443994601889, + "spanCoverageMean": 0.011708371581594209, + "lineCoverageMean": 0.01103832178724339 + }, + { + "laneId": "jcodemunch-full-mcp", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.2713888888888889, + "filePrecisionMean": 0.11000000000000001, + "symbolCoverageMean": 0.19046586746586747, + "spanCoverageMean": 0.08523281976808295, + "lineCoverageMean": 0.08706990517203586 + }, + { + "laneId": "raw-native", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.043333333333333335, + "filePrecisionMean": 0.04, + "symbolCoverageMean": 0.007186234817813765, + "spanCoverageMean": 0.00461426238619191, + "lineCoverageMean": 0.003664930124865624 + }, + { + "laneId": "repowise-selfhosted-mcp", + "rows": 20, + "evaluatorCompletedRows": 19, + "status": "completed:19,judge_failed:1", + "fileCoverageMean": 0.2013888888888889, + "filePrecisionMean": 0.12, + "symbolCoverageMean": 0.046063588114671705, + "spanCoverageMean": 0.03033601885780799, + "lineCoverageMean": 0.024685983349898594 + } + ], + "costSnapshot": [ + { + "laneId": "codebase-context", + "setupStatuses": "not_required:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 750.8, + "indexSumMinutes": 226.6, + "rowWallSumMinutes": 231.5, + "peakPrivateGb": 2.9, + "modelEvidence": "Xenova/bge-small-en-v1.5:20" + }, + { + "laneId": "context-mode", + "setupStatuses": "completed:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 18.6, + "indexSumMinutes": 12.5, + "rowWallSumMinutes": 16.0, + "peakPrivateGb": 0.15, + "modelEvidence": "unknown:20" + }, + { + "laneId": "jcodemunch-full-mcp", + "setupStatuses": "completed:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 11.3, + "indexSumMinutes": 4.9, + "rowWallSumMinutes": 10.9, + "peakPrivateGb": 0.33, + "modelEvidence": "unknown:20" + }, + { + "laneId": "raw-native", + "setupStatuses": "not_required:20", + "indexStatuses": "not_required:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 0.0, + "indexSumMinutes": 0.0, + "rowWallSumMinutes": 6.8, + "peakPrivateGb": 0.06, + "modelEvidence": "unknown:20" + }, + { + "laneId": "repowise-selfhosted-mcp", + "setupStatuses": "completed:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "1:20", + "indexMedianSeconds": 294.3, + "indexSumMinutes": 154.2, + "rowWallSumMinutes": 200.7, + "peakPrivateGb": 4.82, + "modelEvidence": "unknown:20" + } + ], + "failedRows": [ + { + "taskId": "Multi-SWE-Bench__typescript__maintenance__bugfix__8d2ac221", + "laneId": "repowise-selfhosted-mcp", + "status": "judge_failed", + "reason": "official_result_error:no_context_extracted", + "officialResults": "benchmark-runs/contextbench/phase41/corrected-20x1-20260603T103632/gate5-official-retrieval-scoring/rows/064-Multi-SWE-Bench__typescript__maintenance__bugfix__8d2ac221/repowise-selfhosted-mcp/official-results.jsonl", + "trajectory": "benchmark-runs/contextbench/phase41/corrected-20x1-20260603T103632/gate4-corrected-20x1/13-Multi-SWE-Bench-typescript-maintenance-bugfix-8d2ac221/repowise-selfhosted-mcp/runs/repowise-selfhosted-mcp-Multi-SWE-Bench__typescript__maintenance__bugfix__8d2ac221-1-retrieval/trajectory.json" + } + ], + "interpretation": [ + "jCodeMunch and codebase-context are close on file/span retrieval in this fixed adapter run; jCodeMunch has stronger symbol coverage but lower file precision.", + "codebase-context completed all 20 retrieval rows in this run but was expensive to index with local Transformers embeddings.", + "jCodeMunch is much cheaper to index in this adapter run, but its exact cold/warm install/cache boundary still needs care before cost claims.", + "context-mode is lightweight and low-cost, with lower recall ceiling in this run.", + "Repowise indexed and returned rows, but used the highest peak private memory, every index exited nonzero with a documented benign failure mode, and one row produced no extracted context.", + "raw-native is a deterministic lexical baseline here, not a free-form raw agent ceiling; it had low retrieval-overlap on these 20 tasks." + ], + "caveats": { + "publicDocs": "passed: README/docs/package-facing benchmark wording is aligned to the narrow 100-row retrieval-overlap engineering-note framing", + "overclaiming": "passed: public wording keeps winner, leaderboard, paper-grade, resource-efficiency, patch-quality, and productivity claims blocked", + "patchQuality": "blocked: retrieval-overlap only, no downstream patch/test success measured", + "costAccounting": "partial_pass: setup/index/resource telemetry captured; provider-dollar costs remain none/unknown because no paid provider use was approved", + "repeatStability": "blocked: 1 repeat per lane/task, no paper-grade stability or confidence claim", + "artifactCompleteness": "passed: 100 rows have summary, trajectory, raw trace, setup, telemetry, and gold", + "failureInclusion": "passed: Repowise judge_failed/no_context row is retained and zero-filled metric view exists", + "sameDenominator": "passed: 20 tasks x 5 lanes present" + }, + "metaCost": { + "note": "Benchmark orchestration cost is authoring/evaluation overhead, not a lane score.", + "latestKnownCodexGoalTokens": "owner-reported 2.29B input and 7.48M output tokens before this corrected scoring pass; exact all-run total not independently audited here", + "gate5ScoringPeakPrivateMb": 65.1, + "gate5ScoringRuntimeSeconds": 72.4 + } +} diff --git a/tests/mcp-client-templates.test.ts b/tests/mcp-client-templates.test.ts index 03cf820..05baaa8 100644 --- a/tests/mcp-client-templates.test.ts +++ b/tests/mcp-client-templates.test.ts @@ -73,18 +73,21 @@ describe('templates/mcp/http/.mcp.json', () => { }); // --------------------------------------------------------------------------- -// README references templates and all four target clients +// README links the shipped guide for advanced templates and covers target clients // --------------------------------------------------------------------------- describe('README.md client setup documentation', () => { const readme = readText('README.md'); + const clientSetup = readText('docs/client-setup.md'); it('references the stdio template path', () => { - expect(readme).toContain('templates/mcp/stdio/.mcp.json'); + expect(readme).toContain('./docs/client-setup.md'); + expect(clientSetup).toContain('templates/mcp/stdio/.mcp.json'); }); it('references the HTTP template path', () => { - expect(readme).toContain('templates/mcp/http/.mcp.json'); + expect(readme).toContain('./docs/client-setup.md'); + expect(clientSetup).toContain('templates/mcp/http/.mcp.json'); }); it('mentions Claude Code', () => { @@ -103,8 +106,11 @@ describe('README.md client setup documentation', () => { expect(readme).toContain('Windsurf'); }); - it('includes the HTTP endpoint URL', () => { - expect(readme).toContain('127.0.0.1:3100/mcp'); + it('links the advanced HTTP endpoint without making it the default setup', () => { + expect(readme).toContain('./docs/client-setup.md'); + expect(readme).toContain('stdio'); + expect(clientSetup).toContain('127.0.0.1:3100/mcp'); + expect(clientSetup).toContain('not part of the default setup proof'); }); }); @@ -128,15 +134,21 @@ describe('docs/capabilities.md transport documentation', () => { }); it('covers all four target clients', () => { - expect(caps).toContain('Claude Code'); - expect(caps).toContain('Cursor'); - expect(caps).toContain('Codex'); - expect(caps).toContain('Windsurf'); - }); - - it('states the roots-first routing fallback explicitly', () => { - expect(caps).toContain('roots-capable hosts'); - expect(caps).toContain('explicit fallback is still required'); + expect(caps).toContain('./client-setup.md'); + const clientSetup = readText('docs/client-setup.md'); + for (const client of ['Claude Code', 'Cursor', 'Codex', 'Windsurf']) { + expect(clientSetup).toContain(client); + } + }); + + it('states explicit selection as the portable default and scopes compatibility behavior', () => { + expect(caps).toContain('portable default is explicit selection'); + expect(caps).toContain('absolute path as `project`'); + expect(caps).toContain('server returns `selection_required` instead of guessing'); + expect(caps).toContain('Roots are not required for the documented setup'); + expect(caps).toContain( + 'does not establish native agent continuation or concurrent HTTP isolation' + ); }); }); @@ -144,16 +156,27 @@ describe('docs/client-setup.md multi-project guidance', () => { const clientSetup = readText('docs/client-setup.md'); it('documents the project routing contract', () => { + expect(clientSetup).toContain("current repository's absolute path as `project`"); expect(clientSetup).toContain( - 'Automatic multi-project routing is evidence-backed only when the MCP host announces workspace roots.' + 'Without a configured folder, known root or explicit selection, tools return `selection_required`' ); + expect(clientSetup).toContain("They do not guess or index the user's home directory"); + expect(clientSetup).toContain('One valid root can auto-select'); expect(clientSetup).toContain( - 'the server returns `selection_required` instead of guessing' + 'several roots with no active selection return `selection_required`' ); + expect(clientSetup).toContain('Explicit selectors must stay within announced roots'); }); - it('keeps the three verification flows aligned with the roots-first contract', () => { - expect(clientSetup).toContain('Multiple projects on a roots-capable host'); - expect(clientSetup).toContain('Ambiguous or no-roots selection'); + it('separates process-local routing proof from native and HTTP acceptance', () => { + expect(clientSetup).toContain('A to B to A in one no-roots stdio process'); + expect(clientSetup).toContain('CBC routes omitted-project calls using its selected project'); + expect(clientSetup).toContain( + 'Select again after a server restart or when changing repositories' + ); + expect(clientSetup).toContain('native retrieval and continuation did not pass'); + expect(clientSetup).toContain( + 'stdio process-local selection does not establish concurrent HTTP isolation' + ); }); }); diff --git a/tests/proof-truth-surfaces.test.ts b/tests/proof-truth-surfaces.test.ts index cabb7a5..778f838 100644 --- a/tests/proof-truth-surfaces.test.ts +++ b/tests/proof-truth-surfaces.test.ts @@ -49,6 +49,7 @@ const gateArtifact = readJson('results/gate-evaluation.json'); const comparatorEvidence = readJson('results/comparator-evidence.json'); const benchmarkDoc = readText('docs/benchmark.md'); +const discoveryDoc = readText('docs/benchmark-prior-public-report.md'); const comparisonDoc = readText('docs/comparison-table.md'); const registryChecklist = readText('docs/registry-sync-checklist.md'); const readme = readText('README.md'); @@ -72,17 +73,19 @@ describe('proof truth surfaces', () => { expect(comparatorEvidence['codebase-memory-mcp']).toBeDefined(); }); - it('keeps the proof docs aligned to the current gate artifact', () => { - expectContains(benchmarkDoc, [ + it('keeps archived discovery proof aligned to its retained gate artifact', () => { + expectContains(discoveryDoc, [ 'discovery benchmark', `\`${gateArtifact.gate.status}\``, '`claimAllowed`' ]); - expectContains(comparisonDoc, [ - 'Comparator Summary', - `\`${gateArtifact.gate.status}\``, - `claimAllowed\` stays \`${String(gateArtifact.gate.claimAllowed)}\`` - ]); + expect(discoveryDoc).toContain( + `\`claimAllowed\` remains \`${String(gateArtifact.gate.claimAllowed)}\`` + ); + for (const currentDoc of [benchmarkDoc, comparisonDoc]) { + expect(currentDoc).toContain('./benchmark-prior-public-report.md'); + expect(currentDoc).toContain('separate 24-task discovery report'); + } expectContains(registryChecklist, [ `claimAllowed: ${String(gateArtifact.gate.claimAllowed)}`, gateArtifact.gate.status @@ -96,8 +99,8 @@ describe('proof truth surfaces', () => { if (rawClaude.averageFirstRelevantHit === null) { expect(rawClaudeGate.status).toBe('pending_evidence'); expect(rawClaudeGate.missingMetrics ?? []).toContain('averageFirstRelevantHit'); - expect(benchmarkDoc).toMatch(/raw Claude Code[\s\S]*averageFirstRelevantHit[\s\S]*null/i); - expect(comparisonDoc).toMatch(/raw Claude Code[\s\S]*pending_evidence/i); + expect(discoveryDoc).toMatch(/raw Claude Code[\s\S]*averageFirstRelevantHit[\s\S]*null/i); + expect(discoveryDoc).toMatch(/raw Claude Code[\s\S]*pending_evidence/i); expect(registryChecklist).toContain('averageFirstRelevantHit: null'); } }); @@ -108,8 +111,7 @@ describe('proof truth surfaces', () => { ); if (codebaseMemoryGate?.status === 'failed') { - expect(benchmarkDoc).toMatch(/codebase-memory-mcp[\s\S]*gate: `failed`/i); - expect(comparisonDoc).toMatch(/codebase-memory-mcp[\s\S]*gate: `failed`/i); + expect(discoveryDoc).toMatch(/codebase-memory-mcp[\s\S]*gate: `failed`/i); expect(registryChecklist).toContain('comparator artifact `ok` but gate `failed`'); } @@ -118,19 +120,35 @@ describe('proof truth surfaces', () => { .map(([name]) => name); for (const comparatorName of setupFailedComparators) { - expect(benchmarkDoc).toContain(`\`${comparatorName}\``); - expect(comparisonDoc).toContain(`\`${comparatorName}\``); + expect(discoveryDoc).toContain(`\`${comparatorName}\``); } }); - it('keeps package-facing proof mentions secondary and discovery-only', () => { - expectContains(readme, ['discovery-only proof', gateArtifact.gate.status, 'claimAllowed']); + it('keeps package-facing evidence scoped to retrieval and historical CLI observations', () => { + expectContains(readme, [ + '/docs/benchmark.md', + '/docs/benchmark-prior-public-report.md', + 'retrieval observations, not a winner or a coding-quality claim', + 'does not establish patch correctness or end-to-end coding quality' + ]); expectContains(capabilities, [ - 'discovery-only', - gateArtifact.gate.status, - `claimAllowed: ${String(gateArtifact.gate.claimAllowed)}` + 'docs/benchmark.md', + 'does not measure patch correctness or end-to-end task completion', + 'missing or failed setup and comparator runs remain visible' + ]); + expectContains(demo, [ + 'does not verify the current package or an MCP connection', + 'machine-specific repository root is omitted', + 'not a universal performance or coding result' + ]); + expectContains(benchmarkDoc, [ + 'does not measure patch correctness', + 'does not combine the four evidence families into a pooled score' + ]); + expectContains(comparisonDoc, [ + 'do not establish patch correctness', + 'failed attempt is counted as zero' ]); - expectContains(demo, [gateArtifact.gate.status, 'claimAllowed']); }); it('keeps shared planning summaries aligned to the same proof posture', () => { From a9cd0bd6fcb327ea393b194ee9c8507e44f9c72a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:18:12 +0200 Subject: [PATCH 094/100] fix: support Node 22 and 24 and explain first-run indexing * fix(search): clarify first-use indexing guidance * fix(search): clarify index retry guidance * fix: require Node 22 and test supported LTS versions --- .github/workflows/test.yml | 7 ++- package.json | 2 +- src/index.ts | 4 +- src/tools/search-codebase.ts | 5 +- tests/multi-project-routing.test.ts | 30 +++++++++- tests/search-codebase-auto-heal.test.ts | 78 ++++++++++++++++++------- 6 files changed, 97 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8496e23..d518c79 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: version: 10 - uses: actions/setup-node@v4 with: - node-version: '20' + node-version: '24' cache: 'pnpm' - run: pnpm install --frozen-lockfile @@ -35,6 +35,9 @@ jobs: test: name: Functional Tests runs-on: ubuntu-latest + strategy: + matrix: + node: [22, 24] steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 @@ -42,7 +45,7 @@ jobs: version: 10 - uses: actions/setup-node@v4 with: - node-version: '20' + node-version: ${{ matrix.node }} cache: 'pnpm' - run: pnpm install --frozen-lockfile diff --git a/package.json b/package.json index 2b75047..f7ca7f5 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ ], "packageManager": "pnpm@10.27.0", "engines": { - "node": ">=18.0.0" + "node": ">=22.0.0" }, "keywords": [ "mcp", diff --git a/src/index.ts b/src/index.ts index ab539b8..e7c829e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1041,7 +1041,7 @@ export function registerHandlers(target: Server): void { type: 'text', text: JSON.stringify({ status: 'indexing', - message: 'Index build in progress - please retry shortly' + message: 'Index build in progress. Check indexing status, then retry.' }) } ] @@ -1068,7 +1068,7 @@ export function registerHandlers(target: Server): void { type: 'text', text: JSON.stringify({ status: 'indexing', - message: 'Index rebuild in progress - please retry shortly', + message: 'Index rebuild in progress. Check indexing status, then retry.', index: indexSignal }) } diff --git a/src/tools/search-codebase.ts b/src/tools/search-codebase.ts index 7a4f822..d68f888 100644 --- a/src/tools/search-codebase.ts +++ b/src/tools/search-codebase.ts @@ -183,7 +183,7 @@ export async function handle( text: JSON.stringify( { status: 'indexing', - message: 'Index is still being built. Retry in a moment.', + message: 'Index build in progress. Check indexing status, then retry.', progress: ctx.indexState.indexer?.getProgress() }, null, @@ -233,7 +233,8 @@ export async function handle( text: JSON.stringify( { status: 'indexing', - message: 'Index was corrupt. Rebuild started — retry shortly.' + message: + 'Index is unavailable or not ready. Indexing has started. Check indexing status, then retry.' }, null, 2 diff --git a/tests/multi-project-routing.test.ts b/tests/multi-project-routing.test.ts index fa8f181..41a81f1 100644 --- a/tests/multi-project-routing.test.ts +++ b/tests/multi-project-routing.test.ts @@ -367,6 +367,30 @@ describe('multi-project routing', () => { } }); + it('reports status-check guidance when an index build is already running', async () => { + const { server } = await import('../src/index.js'); + const { getOrCreateProject } = await import('../src/project-state.js'); + const handler = (server as unknown as TestServer)._requestHandlers.get('tools/call'); + if (!handler) throw new Error('tools/call handler not registered'); + + const project = getOrCreateProject(primaryRoot); + const originalStatus = project.indexState.status; + project.indexState.status = 'indexing'; + try { + const response = await callTool(handler, 20, 'search_codebase', { + query: 'feature', + project: primaryRoot + }); + const payload = parsePayload(response) as { status: string; message: string }; + + expect(payload.status).toBe('indexing'); + expect(payload.message).toBe('Index build in progress. Check indexing status, then retry.'); + expect(searchMocks.search).not.toHaveBeenCalled(); + } finally { + project.indexState.status = originalStatus; + } + }); + it('triggers a background rebuild for a corrupted explicit project without falling back to cwd', async () => { delete process.env.CODEBASE_ROOT; delete process.argv[2]; @@ -398,7 +422,7 @@ describe('multi-project routing', () => { }; expect(payload.status).toBe('indexing'); - expect(payload.message).toContain('retry shortly'); + expect(payload.message).toBe('Index rebuild in progress. Check indexing status, then retry.'); expect(payload.index?.action).toBe('rebuild-started'); expect(String(payload.index?.reason || '')).toContain('Index meta'); } finally { @@ -591,7 +615,9 @@ describe('multi-project routing', () => { it('lists bounded and full context resources for active and project-scoped flows', async () => { const { server } = await import('../src/index.js'); const toolHandler = (server as unknown as TestServer)._requestHandlers.get('tools/call'); - const resourcesHandler = (server as unknown as TestServer)._requestHandlers.get('resources/list'); + const resourcesHandler = (server as unknown as TestServer)._requestHandlers.get( + 'resources/list' + ); if (!toolHandler || !resourcesHandler) { throw new Error('required handlers not registered'); diff --git a/tests/search-codebase-auto-heal.test.ts b/tests/search-codebase-auto-heal.test.ts index 13cd1a0..2be3bb2 100644 --- a/tests/search-codebase-auto-heal.test.ts +++ b/tests/search-codebase-auto-heal.test.ts @@ -143,33 +143,71 @@ describe('search_codebase auto-heal', () => { } }); - it('fires background re-index and returns retry message when IndexCorruptedError is thrown', async () => { - const { IndexCorruptedError } = await import('../src/errors/index.js'); + it.each([ + [ + 'missing initial index metadata', + 'Index meta missing or unreadable (rebuild required): ENOENT' + ], + ['genuine index corruption', 'LanceDB index corrupted: missing vector column'] + ])( + 'starts background re-index and returns retry guidance for %s', + async (_scenario, errorMessage) => { + const { IndexCorruptedError } = await import('../src/errors/index.js'); - searchMocks.search.mockRejectedValueOnce( - new IndexCorruptedError('LanceDB index corrupted: missing vector column') - ); + searchMocks.search.mockRejectedValueOnce(new IndexCorruptedError(errorMessage)); - const { server } = await import('../src/index.js'); - const handler = (server as any)._requestHandlers.get('tools/call'); + const { server } = await import('../src/index.js'); + const handler = (server as any)._requestHandlers.get('tools/call'); - const response = await handler({ - jsonrpc: '2.0', - id: 1, - method: 'tools/call', - params: { - name: 'search_codebase', - arguments: { - query: 'test' + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { + query: 'test' + } } + }); + + const payload = JSON.parse(response.content[0].text); + expect(payload.status).toBe('indexing'); + expect(payload.message).toBe( + 'Index is unavailable or not ready. Indexing has started. Check indexing status, then retry.' + ); + expect(searchMocks.search).toHaveBeenCalledTimes(1); + }, + 15000 + ); + + it('returns status-check guidance without searching while an index build is active', async () => { + if (!tempRoot) throw new Error('tempRoot not initialized'); + + const { handle } = await import('../src/tools/search-codebase.js'); + const baseDir = path.join(tempRoot, CODEBASE_CONTEXT_DIRNAME); + const response = await handle( + { query: 'search something' }, + { + indexState: { status: 'indexing' }, + paths: { + baseDir, + memory: path.join(baseDir, 'memory.json'), + intelligence: path.join(baseDir, 'intelligence.json'), + health: path.join(baseDir, 'health.json'), + keywordIndex: path.join(baseDir, KEYWORD_INDEX_FILENAME), + vectorDb: path.join(baseDir, VECTOR_DB_DIRNAME) + }, + rootPath: tempRoot, + performIndexing: vi.fn() } - }); + ); - const payload = JSON.parse(response.content[0].text); + const payload = JSON.parse(response.content?.[0]?.text ?? '{}'); expect(payload.status).toBe('indexing'); - expect(payload.message).toContain('retry'); - expect(searchMocks.search).toHaveBeenCalledTimes(1); - }, 15000); + expect(payload.message).toBe('Index build in progress. Check indexing status, then retry.'); + expect(searchMocks.search).not.toHaveBeenCalled(); + }); it('returns invalid_params when search_codebase query is missing', async () => { const { server } = await import('../src/index.js'); From 0d733f1429ebb29e6566b0428b5de69807ffe299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:05:35 +0200 Subject: [PATCH 095/100] fix: show usage when you run the help command * fix: route help to CLI usage * test: keep the original help response timeout --- src/cli.ts | 2 +- src/index.ts | 7 ++++- tests/cli-entrypoint-runtime.test.ts | 47 +++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index c7a0ed4..2e5f17e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -256,7 +256,7 @@ function booleanFlag(flags: Flags, key: string, usage: string): boolean { export async function handleCliCommand(argv: string[]): Promise { const rawCommand = argv[0]; - if (!rawCommand || rawCommand === '--help') { + if (!rawCommand || rawCommand === '--help' || rawCommand === 'help') { printUsage(); return; } diff --git a/src/index.ts b/src/index.ts index e7c829e..3e7a989 100644 --- a/src/index.ts +++ b/src/index.ts @@ -93,7 +93,8 @@ const CLI_SUBCOMMANDS = [ 'refs', 'cycles', 'init', - 'map' + 'map', + 'help' ]; // Check if this module is the entry point. @@ -108,6 +109,10 @@ const isDirectCliSubcommand = // Resolve optional bootstrap root with validation handled later in main(). function resolveRootPath(): string | undefined { + if (isDirectRun && directSubcommand === 'help') { + return undefined; + } + const envPath = process.env.CODEBASE_ROOT; // Walk argv starting at position 2, skip known flags and their values. diff --git a/tests/cli-entrypoint-runtime.test.ts b/tests/cli-entrypoint-runtime.test.ts index 7bdb291..1bef25a 100644 --- a/tests/cli-entrypoint-runtime.test.ts +++ b/tests/cli-entrypoint-runtime.test.ts @@ -1,5 +1,7 @@ import { spawnSync } from 'node:child_process'; -import { resolve } from 'node:path'; +import { mkdir, mkdtemp, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; const root = resolve(import.meta.dirname, '..'); @@ -54,4 +56,47 @@ describe('CLI entrypoint runtime', () => { expect(parsed.rootPath).toBe(root); expect(parsed.hint).toContain('refresh_index'); }, 120_000); + + it('prints usage for help from a project containing a directory named help', async () => { + const projectRoot = await mkdtemp(join(tmpdir(), 'cbc-cli-help-')); + const helpDirectory = join(projectRoot, 'help'); + + const env = { ...process.env }; + delete env.CODEBASE_ROOT; + + try { + await mkdir(helpDirectory); + + const result = spawnSync( + process.execPath, + ['--import', import.meta.resolve('tsx'), entrypoint, 'help'], + { + cwd: projectRoot, + env, + encoding: 'utf8', + timeout: 10_000 + } + ); + + expect({ + errorCode: result.error?.code, + exitCode: result.status, + helpText: result.stdout.includes('codebase-context [options]'), + commandsSection: result.stdout.includes('Commands:'), + stderr: result.stderr, + projectEntries: await readdir(projectRoot), + helpEntries: await readdir(helpDirectory) + }).toEqual({ + errorCode: undefined, + exitCode: 0, + helpText: true, + commandsSection: true, + stderr: '', + projectEntries: ['help'], + helpEntries: [] + }); + } finally { + await rm(projectRoot, { recursive: true, force: true }); + } + }, 70_000); }); From 5021b4fe8e27141926b745675ecd322ec9178aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:08:14 +0200 Subject: [PATCH 096/100] docs: make setup instructions clear before and after the release * docs: align 2.4.0 release guidance * fix: align release workflow with 2.4.0 * docs: explain real setup and 2.4.0 changes * fix(release): require an explicit manual publish tag * docs: qualify unreleased version commands across guides * Fix 2.4.0 setup notices --- .github/workflows/publish-npm-on-release.yml | 4 +- README.md | 58 ++++++++------ docs/capabilities.md | 26 ++++--- docs/cli.md | 54 ++++++------- docs/client-setup.md | 82 ++++++++------------ docs/demo.md | 14 ++-- docs/review-context.md | 6 +- tests/release-truth-surfaces.test.ts | 63 ++++++++++++++- 8 files changed, 185 insertions(+), 122 deletions(-) diff --git a/.github/workflows/publish-npm-on-release.yml b/.github/workflows/publish-npm-on-release.yml index 4ca7cf4..68755f1 100644 --- a/.github/workflows/publish-npm-on-release.yml +++ b/.github/workflows/publish-npm-on-release.yml @@ -8,9 +8,9 @@ on: workflow_dispatch: inputs: tag: - description: 'Tag to publish (e.g. v2.3.0)' + description: 'Release tag to publish' required: true - default: 'v2.3.0' + type: string permissions: contents: read diff --git a/README.md b/README.md index 6249c02..a6ee760 100644 --- a/README.md +++ b/README.md @@ -2,39 +2,47 @@ [![npm version](https://img.shields.io/npm/v/codebase-context)](https://www.npmjs.com/package/codebase-context) [![license](https://img.shields.io/npm/l/codebase-context)](./LICENSE) [![node](https://img.shields.io/node/v/codebase-context)](https://github.com/PatrickSys/codebase-context/blob/master/package.json) -## Your coding agent doesn't understand your codebase. - -Coding agents can read files, but they still have to discover how your repository is organized, which patterns your team follows, and which examples are worth copying. - -Codebase Context gives an agent a local view of that information through code search, team patterns, strong examples, and project memory. It runs as an MCP server - a local tool that your editor or command-line agent can call while it works - and keeps the index on your machine by default. +Codebase Context gives coding agents a map of your repository through MCP, for developers who want their agent to find relevant code and follow the team's patterns. Search, examples and project memory run locally by default. ## Set up your AI client -Choose your coding tool and run its command once. Use Node.js 22 or newer. These commands use published npm `2.2.0` and do not require a project folder in your configuration: +**These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). + +Use Node.js 22 or newer. Choose your coding tool and run its command once; the examples target the 2.4.0 release: ```bash # Claude Code -claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.2.0 +claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.4.0 # Codex CLI -codex mcp add codebase-context -- npx -y codebase-context@2.2.0 +codex mcp add codebase-context -- npx -y codebase-context@2.4.0 # OpenCode 1.x (keep the quoted separator on Windows) -opencode mcp add codebase-context '--' npx -y codebase-context@2.2.0 +opencode mcp add codebase-context '--' npx -y codebase-context@2.4.0 ``` Start a new agent session in your project, then ask: -> Use Codebase Context to find [feature] in this repository. Pass this repository's absolute path as project when checking get_indexing_status and searching. Wait for indexing if needed, read codebase://context, then search_codebase and open a returned source file. Show me the relevant files. +> Use Codebase Context to find [feature] in this repository. Pass this repository's absolute path as project, wait for indexing if needed, and open the relevant source files. + +Replace `[feature]` with something you want to find. The first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. + +### New in 2.4.0 -Replace `[feature]` with something you want to find. The agent supplies the repository path in its tool calls, so the registration can serve different projects. Initial indexing may need a local model download. The October 6 isolated checks proved these client registrations and published-package project selection/search; they did not establish a full native agent investigation. +- Opt-in NestJS support from community contributor Aaron Olin. See the [analyzer options](./docs/capabilities.md#analyzers). +- Better React/Next.js detection and a fix for reranker passage scoring. +- Transformers.js v4 for local models. + +Known limits: [Intel Mac indexing crash (#68)](https://github.com/PatrickSys/codebase-context/issues/68), [custom OpenAI endpoints (#70)](https://github.com/PatrickSys/codebase-context/issues/70) and [Ollama (#71)](https://github.com/PatrickSys/codebase-context/issues/71) remain open. + +The existing [benchmark](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark.md) reports 25.7% expected-file recovery versus jCodeMunch's 27.1% in a fixed retrieval comparison; it does not establish a winner or coding quality. For Codex Desktop, create or merge `.codex/config.toml` in the project you want to search: ```toml [mcp_servers.codebase-context] command = "npx" -args = ["-y", "codebase-context@2.2.0"] +args = ["-y", "codebase-context@2.4.0"] startup_timeout_sec = 120 ``` @@ -44,13 +52,13 @@ Other clients use their own setup commands: | Client | Shortest current setup | | --------------------------- | ---------------------------------------------------------------------------- | -| Gemini CLI | `gemini mcp add --scope user codebase-context npx -y codebase-context@2.2.0` | +| Gemini CLI | `gemini mcp add --scope user codebase-context npx -y codebase-context@2.4.0` | | Cursor | Add `.cursor/mcp.json` | | VS Code with GitHub Copilot | Add `.vscode/mcp.json` | -| GitHub Copilot CLI | `copilot mcp add codebase-context -- npx -y codebase-context@2.2.0` | +| GitHub Copilot CLI | `copilot mcp add codebase-context -- npx -y codebase-context@2.4.0` | | Windsurf | Add `~/.codeium/windsurf/mcp_config.json` | -Check an existing same-name entry before replacing it. To give the server a default folder, append that folder's absolute path to the `npx` arguments. The [client setup guide](./docs/client-setup.md) covers scopes, optional fixed-folder configuration, verification limits and the unreleased installer. Published `2.2.0`'s interactive `init` has registration bugs; use the commands above. +Check an existing same-name entry before replacing it. To give the server a default folder, append that folder's absolute path to the `npx` arguments. The [client setup guide](./docs/client-setup.md) covers scopes, optional fixed-folder configuration and the interactive setup wizard's limitations. The [client setup guide](./docs/client-setup.md) has the exact commands and config for every client, plus what was checked locally and what still relies on official instructions. @@ -64,32 +72,36 @@ The default connection is `stdio` (standard input/output): your client starts th ### Team patterns and examples -`get_team_patterns` shows the approaches used in the repository and points to representative files. Published `2.2.0` includes dedicated analyzers for Angular, React and Next.js, with a generic analyzer for other stacks. NestJS support belongs to the newer source candidate. +`get_team_patterns` shows the approaches used in the repository and points to representative files. Version `2.4.0` includes dedicated analyzers for Angular, React, Next.js and opt-in NestJS, with a generic analyzer for other stacks. ### Project memory `remember` stores a convention, decision, gotcha, or past failure for the project. `get_memory` retrieves relevant entries in later sessions, including when the agent or editor changes. +### Review context + +The `codebase-context-review` CLI creates a bounded context packet from a committed Git diff. It gives a reviewer context; it does not review code or prove review quality. See the [review-context guide](./docs/review-context.md). + ## How it works 1. **Index locally.** Codebase Context scans the project, builds a keyword index, and creates local semantic embeddings - numeric representations used to match code by meaning as well as exact words. 2. **Understand the repository.** The agent can request a compact codebase map with structure, patterns, and representative files. 3. **Find the code for the task.** Search returns ranked files and symbols; the agent reads the selected files before editing. -The same information is available from the terminal. Run these commands from your project root. The first index can take a while because it scans the project and creates local embeddings; once it is ready, inspect the map and search for the code you need: +The same information is available from the terminal. Run these commands from your project root. The first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. Once it is ready, inspect the map and search for the code you need: ```bash # Build or refresh the local index -npx -y codebase-context@2.2.0 reindex +npx -y codebase-context@2.4.0 reindex # Repository structure, patterns, and representative files -npx -y codebase-context@2.2.0 map +npx -y codebase-context@2.4.0 map # Ranked code search -npx -y codebase-context@2.2.0 search --query "auth middleware" +npx -y codebase-context@2.4.0 search --query "auth middleware" # Current team patterns -npx -y codebase-context@2.2.0 patterns +npx -y codebase-context@2.4.0 patterns ``` One stdio server can route across several repositories. Supply `project` in tool calls to select the intended repository; a successful selection becomes the default for later calls in that process. Some clients also announce workspace roots: one root can auto-select, while an ambiguous selection asks for a project instead of guessing. MCP deprecated Roots in its July 2026 revision, so explicit project selection is the documented default rather than a dependency on client discovery. @@ -145,8 +157,8 @@ The method and failures are documented so the measurements can be inspected with - Retrieval measurements describe expected-file coverage, file precision, and reported `peakPrivateGb`, not patch correctness or end-to-end coding quality. - The paired token observation covers two frozen investigation tasks and records observed agent behavior; it is not a universal token or time guarantee. - Setup checks differ by client. The detailed guide distinguishes a written config, a config recognized by the client, a local connection, and instructions checked only against official docs. -- Published `2.2.0` has dedicated Angular, React and Next.js analyzers; NestJS support is in the newer source candidate. Other projects use the generic analyzer and the language parsers available for that stack. -- The default searchable-chunk limit is 5,000 per project. Larger repositories can raise it in `.codebase-context/config.json`. +- Version `2.4.0` has dedicated Angular, React, Next.js and opt-in NestJS analyzers. Other projects use the generic analyzer and the language parsers available for that stack. +- The default searchable-chunk limit is 5,000 per project. In `~/.codebase-context/config.json`, set `projects[].parsing.maxChunks` for a project that needs a higher limit. - The agent must identify its repository in tool calls when no default or unambiguous client root is available. Concurrent HTTP client isolation is not established by the stdio routing checks. ## Reference diff --git a/docs/capabilities.md b/docs/capabilities.md index 748d5a1..59806d6 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -1,6 +1,8 @@ # Capabilities Reference -Technical reference for what `codebase-context` ships today. The public product posture is map first, find second: the bounded conventions map is the first-call surface, and search narrows to the right local example after that. For the user-facing overview, see [README.md](../README.md). +Technical reference for the 2.4.0 setup preview. The public product posture is map first, find second: the bounded conventions map is the first-call surface, and search narrows to the right local example after that. For the user-facing overview, see [README.md](../README.md). + +**These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). ## Transport Modes @@ -8,8 +10,8 @@ The server supports two transport modes: | Mode | Command | MCP endpoint | | ------------------- | ------------------------------------------------- | ---------------------------- | -| **stdio** (default) | `npx -y codebase-context@2.2.0` | Spawned process stdin/stdout | -| **HTTP** | `npx -y codebase-context@2.2.0 --http [--port N]` | `http://127.0.0.1:3100/mcp` | +| **stdio** (default) | `npx -y codebase-context@2.4.0` | Spawned process stdin/stdout | +| **HTTP** | `npx -y codebase-context@2.4.0 --http [--port N]` | `http://127.0.0.1:3100/mcp` | HTTP defaults to `127.0.0.1:3100`. Override with `--port`, `CODEBASE_CONTEXT_PORT`, or `server.port` in `~/.codebase-context/config.json`. @@ -20,13 +22,14 @@ Per-project config overrides supported today: - `projects[].excludePatterns`: merged with the built-in exclusion set for that project at index time - `projects[].analyzerHints.analyzer`: prefers a registered analyzer by name for that project and falls back safely when the name is missing or invalid - `projects[].analyzerHints.extensions`: adds project-local source extensions for indexing and auto-refresh watching without changing defaults for other projects +- `projects[].parsing.maxChunks`: sets the maximum number of searchable chunks indexed for that project Copy-pasteable client config templates are shipped in the package: - `templates/mcp/stdio/.mcp.json` — stdio setup for `.mcp.json`-style clients - `templates/mcp/http/.mcp.json` — HTTP setup for `.mcp.json`-style clients -Use Node.js 22 or newer with the published 2.2.0 commands shown here. For client registration recipes and their verification limits, see the [client setup guide](./client-setup.md). +Use Node.js 22 or newer with the 2.4.0 commands shown here. For 2.4.0 setup preview recipes and their verification limits, see the [client setup guide](./client-setup.md). ## CLI Reference @@ -49,15 +52,15 @@ For a command gallery with examples, see `docs/cli.md`. | `memory add` | `--type`, `--category`, `--memory`, `--reason` | `remember` | | `memory remove ` | — | — | -Commands that list `--json` above support raw JSON output. For MCP client registration, follow the [published client setup recipes](./client-setup.md); do not use the broken `init` setup command in published 2.2.0. Errors go to stderr with exit code 1. +Commands that list `--json` above support raw JSON output. For MCP client registration, follow the [2.4.0 setup preview recipes](./client-setup.md). The existing `init` wizard is interactive and generates HTTP setup; see that guide for its current limitations. Errors go to stderr with exit code 1. ```bash # Quick examples -npx -y codebase-context@2.2.0 status -npx -y codebase-context@2.2.0 search --query "auth middleware" --intent edit -npx -y codebase-context@2.2.0 refs --symbol "UserService" --limit 10 -npx -y codebase-context@2.2.0 cycles --scope src/features -npx -y codebase-context@2.2.0 reindex --incremental +npx -y codebase-context@2.4.0 status +npx -y codebase-context@2.4.0 search --query "auth middleware" --intent edit +npx -y codebase-context@2.4.0 refs --symbol "UserService" --limit 10 +npx -y codebase-context@2.4.0 cycles --scope src/features +npx -y codebase-context@2.4.0 reindex --incremental ``` ## Tool Surface @@ -273,6 +276,7 @@ Impact is 2-hop transitive: direct importers (hop 1) and their importers (hop 2) - **Angular**: signals, standalone components, control flow syntax, lifecycle hooks, DI patterns, component metadata - **React**: function/class components, custom hooks, context usage, memoization, Suspense, ecosystem signal extraction - **Next.js**: App Router and Pages Router detection, route/API classification, route paths, `"use client"`, metadata exports +- **NestJS (opt-in)**: modules, controllers, providers and route handlers, contributed by Aaron Olin. The library indexer accepts `config: { analyzers: { nestjs: { enabled: true } } }`; the CLI/server project configuration does not currently expose analyzer enablement. - **Generic**: 30+ have indexing/retrieval coverage including PHP, Ruby, Swift, Scala, Shell, config/markup., 10 languages have full symbol extraction (Tree-sitter: TypeScript, JavaScript, Python, Java, Kotlin, C, C++, C#, Go, Rust). Notes: @@ -301,6 +305,6 @@ Reproducible evaluation is shipped as a CLI entrypoint backed by shared scoring - **Symbol refs are not a call-graph.** `get_symbol_references` counts identifier-node occurrences in the AST (comments/strings excluded via Tree-sitter). It does not distinguish call sites from type annotations, variable assignments, or imports. Full call-site-specific analysis (`call_expression` nodes only) is a roadmap item. - **Impact is 2-hop max.** `computeImpactCandidates` walks direct importers then their importers. Full BFS reachability is on the roadmap. -- **Published 2.2.0 has dedicated Angular, React, and Next.js analyzers.** NestJS support is in a newer source candidate and is not part of published 2.2.0. Other languages use the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). +- **The 2.4.0 preview source has dedicated Angular, React, Next.js and NestJS analyzers.** Other languages use the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). - **Default embedding model is `bge-small-en-v1.5` (512-token context).** Granite (8192 context) is opt-in via `EMBEDDING_MODEL`. OpenAI is opt-in via `EMBEDDING_PROVIDER=openai` — sends code externally. - **Patterns are file-level frequency counts.** Not semantic clustering. Rising/Declining trend is derived from git commit recency for files using each pattern, not from usage semantics. diff --git a/docs/cli.md b/docs/cli.md index dfda760..88d5b96 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -10,18 +10,20 @@ > Output depends on the repo you run it against. The examples below are illustrative (paths, counts, and detected frameworks will vary). > -> These examples show an earlier CLI run; they do not verify the current package or an MCP connection. +> **These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). > -> The commands below use published `codebase-context@2.2.0` and require Node.js 22 or newer. The CLI is intentionally single-project per invocation and targets one root via `CODEBASE_ROOT` or the current working directory. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). +> These output examples come from an earlier CLI run; they were not captured from `2.4.0` and do not verify an MCP connection. +> +> The commands below use `codebase-context@2.4.0` and require Node.js 22 or newer. The CLI is intentionally single-project per invocation and targets one root via `CODEBASE_ROOT` or the current working directory. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). ## How to run ```bash # CLI only: run from a repo root, or set CODEBASE_ROOT explicitly: -CODEBASE_ROOT=/path/to/repo npx -y codebase-context@2.2.0 status +CODEBASE_ROOT=/path/to/repo npx -y codebase-context@2.4.0 status # Commands that show --json in their help support machine output. Human mode is default. -npx -y codebase-context@2.2.0 patterns --json +npx -y codebase-context@2.4.0 patterns --json ``` ### ASCII fallback @@ -29,7 +31,7 @@ npx -y codebase-context@2.2.0 patterns --json If your terminal doesn’t render Unicode box-drawing cleanly: ```bash -CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context@2.2.0 patterns +CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context@2.4.0 patterns ``` ## Commands @@ -44,18 +46,18 @@ CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context@2.2.0 patterns - `reindex` — rebuild index (full or incremental) - `style-guide` — find style guide sections in docs - `memory list|add|remove` — manage team memory (stored in `.codebase-context/memory.json`) -- MCP client registration — use the published 2.2.0 commands in the [client setup guide](./client-setup.md); published `init` is not a reliable setup route +- MCP client registration — use the 2.4.0 recipes in the [client setup guide](./client-setup.md); `init` is an interactive HTTP setup wizard with the limitations described there --- ## First use -Run these CLI commands from the project root. Build or refresh the local index first; the first index may take a while while the project is scanned and local embeddings are created. Then inspect the conventions map and search for the code you need: +Run these CLI commands from the project root. Build or refresh the local index first; the first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. Then inspect the conventions map and search for the code you need: ```bash -npx -y codebase-context@2.2.0 reindex -npx -y codebase-context@2.2.0 map -npx -y codebase-context@2.2.0 search --query "auth middleware" +npx -y codebase-context@2.4.0 reindex +npx -y codebase-context@2.4.0 map +npx -y codebase-context@2.4.0 search --query "auth middleware" ``` This is the CLI preparation flow. It is separate from MCP client registration and does not by itself show that an MCP client connected successfully. @@ -63,8 +65,8 @@ This is the CLI preparation flow. It is separate from MCP client registration an ## `map` ```bash -npx -y codebase-context@2.2.0 reindex -npx -y codebase-context@2.2.0 map +npx -y codebase-context@2.4.0 reindex +npx -y codebase-context@2.4.0 map ``` The conventions map - run this first on an unfamiliar repo. It shows architecture layers, active patterns with adoption rates and trend direction, and the golden files the team treats as the strongest examples. This is also what the MCP server delivers to AI agents via the `codebase://context` resource on first call, before search narrows to a specific local example. @@ -99,7 +101,7 @@ Example output (truncated): ## `metadata` ```bash -npx -y codebase-context@2.2.0 metadata +npx -y codebase-context@2.4.0 metadata ``` Example output: @@ -120,7 +122,7 @@ Example output: ## `patterns` ```bash -npx -y codebase-context@2.2.0 patterns +npx -y codebase-context@2.4.0 patterns ``` Example output (truncated): @@ -142,8 +144,8 @@ Example output (truncated): ## `search` ```bash -npx -y codebase-context@2.2.0 reindex -npx -y codebase-context@2.2.0 search --query "file watcher" --intent edit --limit 3 +npx -y codebase-context@2.4.0 reindex +npx -y codebase-context@2.4.0 search --query "file watcher" --intent edit --limit 3 ``` Example output (truncated): @@ -164,7 +166,7 @@ Example output (truncated): ## `refs` ```bash -npx -y codebase-context@2.2.0 refs --symbol "startFileWatcher" --limit 10 +npx -y codebase-context@2.4.0 refs --symbol "startFileWatcher" --limit 10 ``` Example output (truncated): @@ -183,7 +185,7 @@ Example output (truncated): ## `cycles` ```bash -npx -y codebase-context@2.2.0 cycles --scope src +npx -y codebase-context@2.4.0 cycles --scope src ``` Example output: @@ -199,7 +201,7 @@ Example output: ## `status` ```bash -npx -y codebase-context@2.2.0 status +npx -y codebase-context@2.4.0 status ``` Example output: @@ -218,8 +220,8 @@ Example output: ## `reindex` ```bash -npx -y codebase-context@2.2.0 reindex -npx -y codebase-context@2.2.0 reindex --incremental --reason "changed watcher logic" +npx -y codebase-context@2.4.0 reindex +npx -y codebase-context@2.4.0 reindex --incremental --reason "changed watcher logic" ``` > **MCP server mode**: if you're running codebase-context as an MCP server (long-running process), the index auto-refreshes via a file watcher — you don't need to call `reindex` between edits. Use `reindex` for one-shot CLI runs or to force a full rebuild. @@ -227,7 +229,7 @@ npx -y codebase-context@2.2.0 reindex --incremental --reason "changed watcher lo ## `style-guide` ```bash -npx -y codebase-context@2.2.0 style-guide --query "naming" +npx -y codebase-context@2.4.0 style-guide --query "naming" ``` Example output: @@ -240,14 +242,14 @@ No style guides found. ## `memory` ```bash -npx -y codebase-context@2.2.0 memory list -npx -y codebase-context@2.2.0 memory list --query "watcher" +npx -y codebase-context@2.4.0 memory list +npx -y codebase-context@2.4.0 memory list --query "watcher" -npx -y codebase-context@2.2.0 memory add \ +npx -y codebase-context@2.4.0 memory add \ --type gotcha \ --category tooling \ --memory "Use pnpm, not npm" \ --reason "Workspace support and speed" -npx -y codebase-context@2.2.0 memory remove +npx -y codebase-context@2.4.0 memory remove ``` diff --git a/docs/client-setup.md b/docs/client-setup.md index 7770e7f..3d47d49 100644 --- a/docs/client-setup.md +++ b/docs/client-setup.md @@ -1,12 +1,14 @@ # Client Setup -Codebase Context runs as a local MCP server: a tool your AI editor or command-line client can call while it works. Register it without a fixed folder, then let the agent supply its current repository through the `project` tool parameter. These examples use published npm `2.2.0`. +**These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). -Use Node.js 22 or newer for these recipes. The package declares Node 18+, but native dependency requirements are narrower than that broad declaration; these checks do not establish Node 18.0 compatibility. +Codebase Context runs as a local MCP server: a tool your AI editor or command-line client can call while it works. Register it without a fixed folder, then let the agent supply its current repository through the `project` tool parameter. These recipes target the 2.4.0 release. + +Use Node.js 22 or newer for these recipes. The package still declares Node 18+, but the built pack failed to index source on Node 18.20.4 with `ReferenceError: crypto is not defined` in UUID chunk creation. Node 22 is the lowest version used successfully in the package check; the engine declaration remains an open compatibility issue. The copyable registration commands target PowerShell or a POSIX terminal. Windows Command Prompt was not tested; OpenCode's quoted separator is shell-dependent. -## Current published-package recipes (updated 2026-10-06) +## Client recipes for 2.4.0 Claude Code, Codex CLI and OpenCode registration commands were checked again on October 6 without a project argument. The other clients retain their older config-shape evidence; no-folder native agent use is unverified for them. See the dated notes below. @@ -25,7 +27,7 @@ After registration, start a new agent session in your project and ask: > Use Codebase Context to find [feature] in this repository. Pass this repository's absolute path as project when checking get_indexing_status and searching. Wait for indexing if needed, read codebase://context, then search_codebase and open a returned source file. Show me the relevant files. -The first index may take a while and download a local model. You can prepare it separately with `npx -y codebase-context@2.2.0 reindex` from the project folder. Index readiness, client connection and useful agent behavior are separate checks. +The first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. You can prepare it separately with `npx -y codebase-context@2.4.0 reindex` from the project folder. Index readiness, client connection and useful agent behavior are separate checks. ## Claude Code @@ -38,13 +40,13 @@ The first index may take a while and download a local model. You can prepare it User setup: ```bash -claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.2.0 +claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.4.0 ``` Optional shared project setup: ```bash -claude mcp add --scope project --transport stdio codebase-context -- npx -y codebase-context@2.2.0 +claude mcp add --scope project --transport stdio codebase-context -- npx -y codebase-context@2.4.0 ``` `--scope project` writes shared `.mcp.json` and requires project-server approval. `--scope local` instead stores a private association for the current project in `~/.claude.json`; it does not write `.mcp.json`. @@ -64,7 +66,7 @@ claude mcp add --scope project --transport stdio codebase-context -- npx -y code **Scope:** The command writes user config. A trusted project can use a project `.codex/config.toml` instead. ```bash -codex mcp add codebase-context -- npx -y codebase-context@2.2.0 +codex mcp add codebase-context -- npx -y codebase-context@2.4.0 ``` Equivalent project config: @@ -72,7 +74,7 @@ Equivalent project config: ```toml [mcp_servers.codebase-context] command = "npx" -args = ["-y", "codebase-context@2.2.0"] +args = ["-y", "codebase-context@2.4.0"] startup_timeout_sec = 120 ``` @@ -82,58 +84,40 @@ startup_timeout_sec = 120 **Official docs:** [Codex MCP configuration](https://learn.chatgpt.com/docs/extend/mcp?surface=cli). The supported `startup_timeout_sec` option defaults to ten seconds; the project example allows 120 seconds for initial `npx` package preparation. This does not prove Desktop startup or guarantee completion of a cold download. -## Codex Desktop: published 2.2.0 +## Codex Desktop: 2.4.0 recipe Create or merge `.codex/config.toml` in the project you want to search. Preserve any existing settings: ```toml [mcp_servers.codebase-context] command = "npx" -args = ["-y", "codebase-context@2.2.0"] +args = ["-y", "codebase-context@2.4.0"] startup_timeout_sec = 120 ``` Trust the project if asked, restart Codex and start a new task there. Use the first-use prompt above so the agent selects the current repository. The CLI command in the preceding section writes user-level config; this recipe uses a trusted project's config instead. -**Verification boundary:** isolated Windows CLI and SDK/MCP checks on October 5 indexed a six-file fixture with published `2.2.0`, selected its project, retrieved context and found a source that the host could open. The exact `npx -y codebase-context@2.2.0 ` entrypoint also passed using a warmed isolated npm cache and prepared index. This does not establish a fully cold download or a fresh Codex Desktop agent session. The September Desktop session below used a packed candidate; keep it separate from published-package acceptance. - -On October 6 the exact published `npx` entrypoint was also tested without a folder, environment root or MCP roots, launched from an unrelated directory. Its first tool call returned `selection_required`; an explicit `project` call selected the prepared fixture, search found the expected symbol, context returned its map, and the runner host opened the returned source. A later call without `project` retained that selection. This is SDK/protocol and runner-host evidence with warmed caches and a pre-indexed fixture, not a native Desktop task. - -## Unreleased installer candidate - -The next release is expected to support one project-only command from the target project: - -The `init --client codex --yes` flow is an unreleased source candidate. Published npm `2.2.0` does not include these flags. Use the published configuration above rather than an `@latest` installer command; candidate verification uses an exact built or installed candidate. +**Verification boundary:** isolated Windows CLI and SDK/MCP checks on October 5 exercised published `2.2.0` on a six-file fixture; the exact `npx -y codebase-context@2.2.0 ` entrypoint passed with a warmed isolated npm cache and prepared index. This is historical evidence for `2.2.0`, not verification of `2.4.0`, a fully cold download or a fresh Codex Desktop agent session. -A successful `init --client codex --yes` writes the project-scoped `.codex/config.toml` and prepares the index for that root. If config writing or index preparation fails, setup is incomplete and Codex is not ready to use. - -For candidate verification before publication, build the source checkout and run its absolute `dist/index.js` path from the project you want to configure: - -```powershell -# In the codebase-context source checkout -npm run build - -# In the target project, using the built CLI path -node C:\path\to\codebase-context\dist\index.js init --client codex --yes -``` +On October 6 the exact published `2.2.0` `npx` entrypoint was tested without a folder, environment root or MCP roots, launched from an unrelated directory. That check does not verify the `2.4.0` package. Its first tool call returned `selection_required`; an explicit `project` call selected the prepared fixture, search found the expected symbol, context returned its map, and the runner host opened the returned source. A later call without `project` retained that selection. This is SDK/protocol and runner-host evidence with warmed caches and a pre-indexed fixture, not a native Desktop task. -Use `--root C:\path\to\project` when the command is run from elsewhere. Here, `root` means the project folder to configure and index. The candidate writes the project-scoped `/.codex/config.toml`, preserves unrelated Codex settings, prepares the index for that folder, and does not write `AGENTS.md` or global client configuration. After a successful result, trust the project in Codex Desktop if prompted, restart or reopen Codex, start a new task in that project, and ask a small project question. A written config and ready index do not prove that Codex loaded or queried the server. +## Interactive setup wizard -Before release, the generated `npx` server command still resolves the published package unless the candidate is installed in the target project's `node_modules`. Our packaged test installs the candidate in a disposable project and checks its file hash. Running the built setup command alone does not prove that Codex will launch that same candidate. +Run `npx -y codebase-context@2.4.0 init` from the target project to open the existing interactive wizard. It asks you to choose Claude Code, Cursor, Codex or OpenCode, previews HTTP configuration and asks before applying it. It separately offers to write an instruction block to `CLAUDE.md`, `.cursorrules` or `AGENTS.md` where applicable. -**Checked on 2026-09-15:** the packed candidate installed and indexed an isolated Windows project. A fresh Codex Desktop task then called the actual `search_codebase` MCP tool and located the requested chart component with its source file and lines. The normal Desktop task flow established project trust; no trust-file edits or replacement CLI/SDK search were used for that answer. This covers the small prepared project with shared caches, not a cold-machine install or the published npm package. +The wizard does not accept non-interactive client/root flags or prepare an index. Its HTTP configurations need a separately started server. In the current source, Claude/Codex registration attempts to execute `mcp` directly and can report failure while the wizard still exits successfully. Use the direct stdio commands or configuration above for those clients; the wizard is not a verified one-step setup. ## Optional fixed-folder setup and CLI preparation -To set a default project before the first tool call, append its full path to the server arguments, for example `args = ["-y", "codebase-context@2.2.0", "C:/projects/my-app"]`. Use a quoted literal shell argument in registration commands. This makes the default fixed across repositories when the registration is user-scoped; explicit tool selection can still choose another allowed project. +To set a default project before the first tool call, append its full path to the server arguments, for example `args = ["-y", "codebase-context@2.4.0", "C:/projects/my-app"]`. Use a quoted literal shell argument in registration commands. This makes the default fixed across repositories when the registration is user-scoped; explicit tool selection can still choose another allowed project. To prepare the index separately, run from the project folder: ```powershell -npx -y codebase-context@2.2.0 reindex +npx -y codebase-context@2.4.0 reindex ``` -Do not use published `2.2.0`'s interactive `init` as the setup route. October 6 isolated checks found that it ignores `--help`, `--client` and `--yes`; its Claude/Codex registration executes `mcp` instead of the client executable, and its HTTP recipes require a separately started server. It can exit zero after registration fails. The unreleased installer above is a separate implementation. +The interactive wizard limitations described above are still present in the 2.4.0 source. A successful wizard exit alone does not establish working client registration. ## Gemini CLI @@ -146,13 +130,13 @@ Do not use published `2.2.0`'s interactive `init` as the setup route. October 6 User setup: ```bash -gemini mcp add --scope user codebase-context npx -y codebase-context@2.2.0 +gemini mcp add --scope user codebase-context npx -y codebase-context@2.4.0 ``` Project setup: ```bash -gemini mcp add --scope project codebase-context npx -y codebase-context@2.2.0 +gemini mcp add --scope project codebase-context npx -y codebase-context@2.4.0 ``` Gemini writes an `mcpServers` entry in its settings. Claude, Codex, and Copilot CLI use `--`; Gemini does not. @@ -178,7 +162,7 @@ Create `.cursor/mcp.json`: "mcpServers": { "codebase-context": { "command": "npx", - "args": ["-y", "codebase-context@2.2.0"] + "args": ["-y", "codebase-context@2.4.0"] } } } @@ -203,7 +187,7 @@ Checked config shape: `.cursor/mcp.json with mcpServers.codebase-context command Official user command: ```powershell -code --add-mcp '{"name":"codebase-context","command":"npx","args":["-y","codebase-context@2.2.0"]}' +code --add-mcp '{"name":"codebase-context","command":"npx","args":["-y","codebase-context@2.4.0"]}' ``` Deterministic workspace config at `.vscode/mcp.json`: @@ -213,7 +197,7 @@ Deterministic workspace config at `.vscode/mcp.json`: "servers": { "codebase-context": { "command": "npx", - "args": ["-y", "codebase-context@2.2.0"] + "args": ["-y", "codebase-context@2.4.0"] } } } @@ -234,7 +218,7 @@ Deterministic workspace config at `.vscode/mcp.json`: **Scope:** User config. ```bash -copilot mcp add codebase-context -- npx -y codebase-context@2.2.0 +copilot mcp add codebase-context -- npx -y codebase-context@2.4.0 ``` **What was checked:** The command ran with an isolated `--config-dir` and wrote the expected `mcp-config.json` command, arguments, and tool filter. @@ -254,7 +238,7 @@ copilot mcp add codebase-context -- npx -y codebase-context@2.2.0 Run once: ```bash -opencode mcp add codebase-context '--' npx -y codebase-context@2.2.0 +opencode mcp add codebase-context '--' npx -y codebase-context@2.4.0 ``` Keep the quoted `'--'`: the installed Windows PowerShell npm shim consumes an unquoted separator. The quoted form also works in POSIX shells. @@ -267,7 +251,7 @@ For a project-local alternative, merge this into `opencode.json`: "mcp": { "codebase-context": { "type": "local", - "command": ["npx", "-y", "codebase-context@2.2.0"], + "command": ["npx", "-y", "codebase-context@2.4.0"], "enabled": true, "timeout": 120000 } @@ -277,7 +261,7 @@ For a project-local alternative, merge this into `opencode.json`: Checked config shape: `opencode.json with a local command array and timeout 120000`. -The longer timeout is intentional because the isolated cold start took about 38 seconds while `npx` prepared the package and server. +The longer timeout allows for initial `npx` package preparation and server startup; that time varies with the machine, network and cache. **What was checked:** With home, config, data, cache, temporary and workspace paths redirected, the no-folder command wrote user configuration and `opencode mcp list` reported Connected. An August 11 check with OpenCode 1.18.5 also connected using the project config shape and an explicit folder; the no-folder project variant is not a new native acceptance check. @@ -300,7 +284,7 @@ Create `~/.codeium/windsurf/mcp_config.json`: "mcpServers": { "codebase-context": { "command": "npx", - "args": ["-y", "codebase-context@2.2.0"] + "args": ["-y", "codebase-context@2.4.0"] } } } @@ -319,8 +303,8 @@ Checked config shape: `~/.codeium/windsurf/mcp_config.json with mcpServers.codeb Use `stdio` for the simplest setup. HTTP can let several clients share one long-running local process, but support depends on the client and this path is not part of the default setup proof: ```bash -npx -y codebase-context@2.2.0 --http -npx -y codebase-context@2.2.0 --http --port 4000 +npx -y codebase-context@2.4.0 --http +npx -y codebase-context@2.4.0 --http --port 4000 ``` The documented default endpoint is `http://127.0.0.1:3100/mcp`. Config-shape templates are available in [`templates/mcp/stdio/.mcp.json`](../templates/mcp/stdio/.mcp.json) and [`templates/mcp/http/.mcp.json`](../templates/mcp/http/.mcp.json); pin the published package as in the recipes above. HTTP does not discover a client's project folder; the agent still has to select its repository. @@ -339,7 +323,7 @@ Some clients also announce workspace roots. One valid root can auto-select; seve [MCP deprecated Roots on July 28, 2026](https://modelcontextprotocol.io/specification/2026-07-28/client/roots), recommending explicit directories/files in tool parameters, resource URIs or server configuration. CBC's older Roots support remains a compatibility aid rather than a required setup step. -October 6 verification: 18 focused current-source routing tests passed, including ambiguous roots, explicit selection, subsequent calls and root changes. The exact published server also switched between two isolated prepared repositories A to B to A in one no-roots stdio process: search paths, context map and subsequent status calls followed each selection. The runner host opened the expected distinct sources. A native agent session across real repositories remains a validation step; stdio process-local selection does not establish concurrent HTTP isolation. +October 6 verification: 18 focused current-source routing tests passed, including ambiguous roots, explicit selection, subsequent calls and root changes. The exact published `2.2.0` server also switched between two isolated prepared repositories A to B to A in one no-roots stdio process; search paths, context map and subsequent status calls followed each selection, and the runner host opened the expected distinct sources. This historical package check does not verify `2.4.0` or establish concurrent HTTP isolation. A native agent session across real repositories remains a validation step; stdio process-local selection does not establish concurrent HTTP isolation. A follow-up recorded raw `search_codebase` calls containing only `query` and `mode`, with no `project`: they returned A, B and A after those explicit selections. The caller does not automatically add the full path; CBC routes omitted-project calls using its selected project. This state lasts for that stdio process. Select again after a server restart or when changing repositories. diff --git a/docs/demo.md b/docs/demo.md index 8b06f23..81d0021 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -2,14 +2,16 @@ This walkthrough shows real CLI output captured from the open-source `angular-spotify` repository during a local proof run. That sample checkout is not bundled here. -To try the same CLI flow, use Node.js 22 or newer and open a terminal in the repository you want to inspect. These commands use published `codebase-context@2.2.0`; the CLI uses the current directory as the project root. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). +**These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). -The saved output below is from an earlier CLI run, not the published-package commands shown here. The machine-specific repository root is omitted from the returned file path. This capture does not verify the current package or an MCP connection. +To try the same CLI flow, use Node.js 22 or newer and open a terminal in the repository you want to inspect. These commands target `codebase-context@2.4.0`; the CLI uses the current directory as the project root. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). + +The saved output below is from an earlier CLI run, not from `2.4.0` or the commands shown here. The machine-specific repository root is omitted from the returned file path. This capture does not verify the current package or an MCP connection. ## 0. Build The Local Index ```bash -npx -y codebase-context@2.2.0 reindex +npx -y codebase-context@2.4.0 reindex ``` Run this before the first map or search in a repository. Later runs can use `reindex --incremental` after the code changes. @@ -17,7 +19,7 @@ Run this before the first map or search in a repository. Later runs can use `rei ## 1. Start With The Conventions Map ```bash -npx -y codebase-context@2.2.0 map --json +npx -y codebase-context@2.4.0 map --json ``` Captured output excerpt: @@ -50,7 +52,7 @@ What this shows: ## 2. Search With Edit Intent ```bash -npx -y codebase-context@2.2.0 search --query "auth headers" --intent edit --limit 3 --json +npx -y codebase-context@2.4.0 search --query "auth headers" --intent edit --limit 3 --json ``` Captured output excerpt: @@ -91,7 +93,7 @@ What this shows: ## 3. Check A Team Pattern Directly ```bash -npx -y codebase-context@2.2.0 patterns --category state --json +npx -y codebase-context@2.4.0 patterns --category state --json ``` Captured output excerpt: diff --git a/docs/review-context.md b/docs/review-context.md index e35df6a..f21b3d6 100644 --- a/docs/review-context.md +++ b/docs/review-context.md @@ -4,6 +4,8 @@ It is deliberately **not** an AI reviewer. It does not call an LLM, decide whether code is correct, or post comments. Its job is narrower and testable: compile the changed surface, stable diff signals, related codebase context, and current conventions into a versioned input for a reviewer or evaluation harness. +**These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). + ## Why this exists A PR reviewer needs more than the patch, but dumping an entire repository into a model is expensive and hard to reproduce. The review-context path creates a clean boundary between: @@ -26,13 +28,13 @@ node dist/review-bin.js --base origin/main --head HEAD After package publication, the package also exposes: ```bash -npx codebase-context-review --base origin/main --head HEAD +npm exec --yes --package=codebase-context@2.4.0 -- codebase-context-review --base origin/main --head HEAD ``` Use `--json` for the complete machine-readable packet: ```bash -npx codebase-context-review \ +npm exec --yes --package=codebase-context@2.4.0 -- codebase-context-review \ --base origin/main \ --head HEAD \ --max-queries 8 \ diff --git a/tests/release-truth-surfaces.test.ts b/tests/release-truth-surfaces.test.ts index f362ba4..49d56d4 100644 --- a/tests/release-truth-surfaces.test.ts +++ b/tests/release-truth-surfaces.test.ts @@ -114,8 +114,65 @@ describe('release truth surfaces', () => { expect(visualsDoc).toContain('Historical snapshot'); }); - it('keeps the manual publish fallback aligned to the package version', () => { - expect(workflow).toContain(`description: 'Tag to publish (e.g. v${expectedVersion})'`); - expect(workflow).toContain(`default: 'v${expectedVersion}'`); + it('requires an explicit release tag for the manual publish fallback', () => { + const manualTagInput = workflow.match(/^ {6}tag:\r?\n((?: {8}[^\r\n]*\r?\n)+)/m)?.[1] ?? ''; + + expect(manualTagInput).toMatch(/^\s*required:\s*true\s*$/m); + expect(manualTagInput).toMatch(/^\s*type:\s*string\s*$/m); + expect(manualTagInput).not.toMatch(/^\s*default:/m); + expect(workflow).toMatch(/^ release:\r?\n {4}types: \[published\]/m); + expect(workflow).toContain( + 'ref: ${{ github.event.inputs.tag || github.event.release.tag_name }}' + ); + }); + + it('keeps 2.4.0 setup guidance accurate before and after publication', () => { + const versionNotice = 'These commands use 2.4.0.'; + const fallbackNotice = 'If that version is not available on npm yet, use the published 2.2.0'; + const currentVersionGuide = + 'https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md'; + const setupPages = [ + [ + 'README.md', + 'claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.4.0' + ], + [ + 'docs/client-setup.md', + 'claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.4.0' + ], + ['docs/capabilities.md', '| **stdio** (default) |'], + ['docs/cli.md', 'CODEBASE_ROOT=/path/to/repo npx -y codebase-context@2.4.0 status'], + ['docs/demo.md', 'npx -y codebase-context@2.4.0 reindex'], + ['docs/review-context.md', 'pnpm build'] + ] as const; + + for (const [path, firstCommand] of setupPages) { + const page = readText(path); + const noticeIndex = page.indexOf(versionNotice); + const guideIndex = page.indexOf(currentVersionGuide); + const commandIndex = page.indexOf(firstCommand); + + expect( + noticeIndex, + `${path} should explain the current-version fallback` + ).toBeGreaterThanOrEqual(0); + expect(guideIndex, `${path} should link the published 2.2.0 setup`).toBeGreaterThan( + noticeIndex + ); + expect(page.slice(noticeIndex, guideIndex)).toContain(fallbackNotice); + expect(page.slice(noticeIndex, guideIndex)).toContain('[setup guide]('); + expect(page).not.toContain('**2.4.0 setup preview:**'); + expect(commandIndex, `${path} should retain its first runnable command`).toBeGreaterThan(0); + expect(noticeIndex, `${path} should qualify commands before they appear`).toBeLessThan( + commandIndex + ); + } + }); + + it('does not describe the unpublished 2.4.0 preview as current or published', () => { + const capabilities = readText('docs/capabilities.md'); + + expect(capabilities).not.toContain('what `codebase-context` ships today'); + expect(capabilities).not.toContain('Published 2.4.0 has'); }); }); From cfaa90b2c2180f6c87df5d3bd978868025af6655 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:37:09 +0200 Subject: [PATCH 097/100] fix: set up Claude Code and Codex from your project folder --- docs/client-setup.md | 16 +- src/cli-init.ts | 495 ++++++++++++++++++++++++------- tests/cli-init-bootstrap.test.ts | 33 +++ tests/cli-init.test.ts | 443 ++++++++++++++++++++++++--- 4 files changed, 842 insertions(+), 145 deletions(-) create mode 100644 tests/cli-init-bootstrap.test.ts diff --git a/docs/client-setup.md b/docs/client-setup.md index 3d47d49..c052e65 100644 --- a/docs/client-setup.md +++ b/docs/client-setup.md @@ -103,9 +103,21 @@ On October 6 the exact published `2.2.0` `npx` entrypoint was tested without a f ## Interactive setup wizard -Run `npx -y codebase-context@2.4.0 init` from the target project to open the existing interactive wizard. It asks you to choose Claude Code, Cursor, Codex or OpenCode, previews HTTP configuration and asks before applying it. It separately offers to write an instruction block to `CLAUDE.md`, `.cursorrules` or `AGENTS.md` where applicable. +Run setup from the project you want the client to search: -The wizard does not accept non-interactive client/root flags or prepare an index. Its HTTP configurations need a separately started server. In the current source, Claude/Codex registration attempts to execute `mcp` directly and can report failure while the wizard still exits successfully. Use the direct stdio commands or configuration above for those clients; the wizard is not a verified one-step setup. +```bash +# Claude Code +npx -y codebase-context@2.4.0 init --client claude-code --yes + +# Codex +npx -y codebase-context@2.4.0 init --client codex --yes +``` + +Claude Code writes `.mcp.json`; Codex writes `.codex/config.toml`. Both configs start the installed server with the selected project. Setup uses the current folder unless `--root` or `CODEBASE_ROOT` selects another one; `--root` takes priority. + +`--yes` writes the config without prompts. Run `init` without flags to choose a client interactively and optionally add project instructions. Cursor and OpenCode still use HTTP and need a separately running server. + +Existing settings are kept. If a different Codebase Context entry is already present, setup leaves the file unchanged and reports the conflict. Restart the client after setup and approve the server if asked. The first use builds the code index. Rerun setup if the Codebase Context installation moves, because the config uses its installed location. ## Optional fixed-folder setup and CLI preparation diff --git a/src/cli-init.ts b/src/cli-init.ts index e9efc76..bf126cc 100644 --- a/src/cli-init.ts +++ b/src/cli-init.ts @@ -10,28 +10,54 @@ import path from 'path'; import * as fs from 'node:fs/promises'; -import { execFileSync } from 'child_process'; import { select, confirm } from '@inquirer/prompts'; export type Client = 'claude-code' | 'cursor' | 'codex' | 'opencode'; -export type McpConfigResult = - | { kind: 'file'; path: string; content: string } - | { kind: 'command'; args: string[] }; +export type McpConfigResult = { kind: 'file'; path: string; content: string }; + +export type McpConfigOptions = { + rootPath?: string; + nodeExecutable?: string; + serverEntrypoint?: string; +}; type JsonObject = Record; -export function generateMcpConfig(client: Client): McpConfigResult { +function quoteTomlString(value: string): string { + return JSON.stringify(value); +} + +export function generateMcpConfig(client: Client, options: McpConfigOptions = {}): McpConfigResult { + const rootPath = path.resolve(options.rootPath ?? process.cwd()); + const nodeExecutable = options.nodeExecutable ?? process.execPath; + const serverEntrypoint = path.resolve( + options.serverEntrypoint ?? process.argv[1] ?? path.join(rootPath, 'dist', 'index.js') + ); + const stdioArgs = [serverEntrypoint, rootPath]; + switch (client) { case 'claude-code': return { - kind: 'command', - args: ['mcp', 'add', '--transport', 'http', 'codebase-context', 'http://127.0.0.1:3100/mcp'] + kind: 'file', + path: path.join(rootPath, '.mcp.json'), + content: JSON.stringify( + { + mcpServers: { + 'codebase-context': { + command: nodeExecutable, + args: stdioArgs + } + } + }, + null, + 2 + ) }; case 'cursor': return { kind: 'file', - path: '.cursor/mcp.json', + path: path.join(rootPath, '.cursor', 'mcp.json'), content: JSON.stringify( { mcpServers: { @@ -47,13 +73,18 @@ export function generateMcpConfig(client: Client): McpConfigResult { }; case 'codex': return { - kind: 'command', - args: ['mcp', 'add', 'codebase-context', 'http://127.0.0.1:3100/mcp'] + kind: 'file', + path: path.join(rootPath, '.codex', 'config.toml'), + content: + `[mcp_servers.codebase-context]\n` + + `command = ${quoteTomlString(nodeExecutable)}\n` + + `args = [${stdioArgs.map(quoteTomlString).join(', ')}]\n` + + 'startup_timeout_sec = 120\n' }; case 'opencode': return { kind: 'file', - path: 'opencode.json', + path: path.join(rootPath, 'opencode.json'), content: JSON.stringify( { $schema: 'https://opencode.ai/config.json', @@ -71,6 +102,34 @@ export function generateMcpConfig(client: Client): McpConfigResult { } } +export function resolveInitRoot( + args: string[], + envRoot: string | undefined, + cwd = process.cwd() +): string { + let explicitRoot: string | undefined; + + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === '--root') { + const value = args[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error('Expected a directory path after --root.'); + } + explicitRoot = value; + index++; + } else if (arg?.startsWith('--root=')) { + const value = arg.slice('--root='.length); + if (!value) { + throw new Error('Expected a directory path after --root.'); + } + explicitRoot = value; + } + } + + return path.resolve(explicitRoot ?? (envRoot ? envRoot : cwd)); +} + export function generateInstructionBlock(): string { return ` ## Codebase Context (MCP) @@ -127,57 +186,99 @@ function isJsonObject(value: unknown): value is JsonObject { return typeof value === 'object' && value !== null && !Array.isArray(value); } +function normalizeJson(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(normalizeJson); + } + if (isJsonObject(value)) { + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, normalizeJson(entry)]) + ); + } + return value; +} + +function jsonValuesEqual(left: unknown, right: unknown): boolean { + return JSON.stringify(normalizeJson(left)) === JSON.stringify(normalizeJson(right)); +} + +function isFileNotFound(error: unknown): boolean { + return typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT'; +} + +async function readExistingConfig(filePath: string): Promise { + try { + return await fs.readFile(filePath, 'utf8'); + } catch (error) { + if (isFileNotFound(error)) return null; + throw error; + } +} + /** * Merge generated MCP config into an existing JSON config file when possible. - * Preserves unrelated keys and only updates the codebase-context server entry. + * Preserves unrelated keys and refuses to replace a different codebase-context entry. */ export async function _buildMergedMcpContent( filePath: string, generatedContent: string, - client: Extract + client: Extract ): Promise<{ content: string; mergedFromExisting: boolean }> { + const existingText = await readExistingConfig(filePath); + if (existingText === null) { + return { content: generatedContent, mergedFromExisting: false }; + } + let existing: unknown; try { - existing = JSON.parse(await fs.readFile(filePath, 'utf8')); - } catch { - return { content: generatedContent, mergedFromExisting: false }; + existing = JSON.parse(existingText) as unknown; + } catch (error) { + throw new Error(`Cannot merge ${client} config at ${filePath}: existing JSON is invalid.`, { + cause: error + }); } const generated = JSON.parse(generatedContent) as unknown; if (!isJsonObject(existing) || !isJsonObject(generated)) { - return { content: generatedContent, mergedFromExisting: false }; + throw new Error(`Cannot merge ${client} config at ${filePath}: expected a JSON object.`); } - if (client === 'cursor') { - const existingServers = isJsonObject(existing.mcpServers) ? existing.mcpServers : {}; - const generatedServers = isJsonObject(generated.mcpServers) ? generated.mcpServers : {}; - return { - content: JSON.stringify( - { - ...existing, - ...generated, - mcpServers: { - ...existingServers, - ...generatedServers - } - }, - null, - 2 - ), - mergedFromExisting: true - }; + const serverKey = client === 'opencode' ? 'mcp' : 'mcpServers'; + const existingServersValue = existing[serverKey]; + if (existingServersValue !== undefined && !isJsonObject(existingServersValue)) { + throw new Error( + `Cannot merge ${client} config at ${filePath}: ${serverKey} must be a JSON object.` + ); + } + const generatedServersValue = generated[serverKey]; + if (!isJsonObject(generatedServersValue)) { + throw new Error(`Invalid generated ${client} MCP config.`); + } + const existingServers = isJsonObject(existingServersValue) ? existingServersValue : {}; + const hasExistingCodebaseContext = Object.prototype.hasOwnProperty.call( + existingServers, + 'codebase-context' + ); + const generatedCodebaseContext = generatedServersValue['codebase-context']; + if ( + hasExistingCodebaseContext && + !jsonValuesEqual(existingServers['codebase-context'], generatedCodebaseContext) + ) { + throw new Error( + `Cannot merge ${client} config at ${filePath}: it already contains a different codebase-context configuration.` + ); } - const existingMcp = isJsonObject(existing.mcp) ? existing.mcp : {}; - const generatedMcp = isJsonObject(generated.mcp) ? generated.mcp : {}; return { content: JSON.stringify( { ...existing, ...generated, - mcp: { - ...existingMcp, - ...generatedMcp + [serverKey]: { + ...existingServers, + ...generatedServersValue } }, null, @@ -187,56 +288,232 @@ export async function _buildMergedMcpContent( }; } -export async function handleInitCli(_argv: string[]): Promise { - console.log('\nSet up codebase-context for your AI client\n'); +function normalizeTomlNewlines(value: string): string { + return value.replace(/\r\n/g, '\n'); +} - const client = await select({ - message: 'Which client?', - choices: [ - { name: 'Claude Code', value: 'claude-code' }, - { name: 'Cursor', value: 'cursor' }, - { name: 'Codex', value: 'codex' }, - { name: 'OpenCode', value: 'opencode' } - ] - }); +function tomlKeyPattern(key: string): string { + return `(?:${key}|"${key}"|'${key}')`; +} - const mcpResult = generateMcpConfig(client); +/** + * Appends the project MCP table to Codex TOML, preserving other settings and + * refusing to replace an existing codebase-context table. + */ +export async function _buildMergedCodexContent( + filePath: string, + generatedContent: string +): Promise<{ content: string; mergedFromExisting: boolean }> { + const existing = await readExistingConfig(filePath); + if (existing === null) { + return { content: generatedContent, mergedFromExisting: false }; + } - console.log('\n--- MCP Config Preview ---'); - if (mcpResult.kind === 'file') { - try { - await fs.access(mcpResult.path); - console.log(`Warning: ${mcpResult.path} already exists; existing entries will be preserved.`); - } catch { - // file does not exist + const normalizedExisting = normalizeTomlNewlines(existing); + const normalizedGenerated = normalizeTomlNewlines(generatedContent).trim(); + const mcpServersKey = tomlKeyPattern('mcp_servers'); + const codebaseContextKey = tomlKeyPattern('codebase-context'); + const entryHeader = new RegExp( + `^\\s*\\[{1,2}\\s*${mcpServersKey}\\s*\\.\\s*${codebaseContextKey}(?:\\s*\\.\\s*[^\\]]+)?\\s*\\]{1,2}\\s*(?:#.*)?$`, + 'gm' + ); + const existingEntry = entryHeader.exec(normalizedExisting); + + if (existingEntry) { + const nextTable = /^\s*\[{1,2}[^\r\n]+\]{1,2}\s*(?:#.*)?$/gm; + nextTable.lastIndex = entryHeader.lastIndex; + const nextTableMatch = nextTable.exec(normalizedExisting); + const sectionEnd = nextTableMatch?.index ?? normalizedExisting.length; + const section = normalizedExisting.slice(existingEntry.index, sectionEnd).trim(); + const exactEntryHeader = new RegExp( + `^\\[\\s*${mcpServersKey}\\s*\\.\\s*${codebaseContextKey}\\s*\\]\\s*(?:#.*)?$`, + 'm' + ); + const comparableSection = section.replace(exactEntryHeader, '[mcp_servers.codebase-context]'); + if (comparableSection === normalizedGenerated) { + return { content: existing, mergedFromExisting: true }; } - console.log(`File: ${mcpResult.path}\n${mcpResult.content}`); - } else { - console.log(`Command to run: ${mcpResult.args[0]} ${mcpResult.args.slice(1).join(' ')}`); + throw new Error( + `Cannot merge Codex config at ${filePath}: it already contains a different codebase-context configuration.` + ); } - const applyMcp = await confirm({ - message: 'Apply MCP config? [y/N]', - default: false - }); - - const instructionPath = resolveInstructionFilePath(client, process.cwd()); + const dottedEntry = new RegExp( + `^\\s*${mcpServersKey}\\s*\\.\\s*${codebaseContextKey}\\s*\\.`, + 'm' + ); + if (dottedEntry.test(normalizedExisting)) { + throw new Error( + `Cannot merge Codex config at ${filePath}: it already contains a different codebase-context configuration.` + ); + } - let applyInstruction = false; - if (instructionPath !== null) { - let fileExists = false; - try { - await fs.access(instructionPath); - fileExists = true; - } catch { - // does not exist + const parentTableHeader = new RegExp(`^\\s*\\[\\s*${mcpServersKey}\\s*\\]\\s*(?:#.*)?$`, 'gm'); + let parentTable = parentTableHeader.exec(normalizedExisting); + while (parentTable) { + const nextTable = /^\s*\[{1,2}[^\r\n]+\]{1,2}\s*(?:#.*)?$/gm; + nextTable.lastIndex = parentTableHeader.lastIndex; + const nextTableMatch = nextTable.exec(normalizedExisting); + const sectionEnd = nextTableMatch?.index ?? normalizedExisting.length; + const parentSettings = normalizedExisting.slice(parentTableHeader.lastIndex, sectionEnd); + const childEntry = new RegExp(`^\\s*${codebaseContextKey}\\s*(?:=|\\.)`, 'm'); + if (childEntry.test(parentSettings)) { + throw new Error( + `Cannot merge Codex config at ${filePath}: it already contains a different codebase-context configuration.` + ); } + parentTable = parentTableHeader.exec(normalizedExisting); + } + + const firstTable = /^\s*\[/m.exec(normalizedExisting); + const rootSettings = firstTable + ? normalizedExisting.slice(0, firstTable.index) + : normalizedExisting; + const inlineServers = new RegExp(`^\\s*${mcpServersKey}\\s*=`, 'm'); + if (inlineServers.test(rootSettings)) { + throw new Error( + `Cannot merge Codex config at ${filePath}: mcp_servers is already configured inline.` + ); + } + + const prefix = normalizedExisting.trimEnd(); + return { + content: `${prefix ? `${prefix}\n\n` : ''}${normalizedGenerated}\n`, + mergedFromExisting: true + }; +} + +type InitOptions = { + client?: Client; + rootPath: string; + yes: boolean; + help: boolean; +}; + +function parseClient(value: string): Client { + if (value === 'claude-code' || value === 'cursor' || value === 'codex' || value === 'opencode') { + return value; + } + throw new Error(`Unknown client "${value}". Choose claude-code, cursor, codex, or opencode.`); +} + +function parseInitOptions(args: string[]): InitOptions { + if (args.includes('--help') || args.includes('-h')) { + return { rootPath: process.cwd(), yes: false, help: true }; + } - if (fileExists) { - console.log(`\n--- Instruction Block Preview (will append to ${instructionPath}) ---`); + let client: Client | undefined; + let yes = false; + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === '--yes' || arg === '-y') { + yes = true; + } else if (arg === '--client') { + const value = args[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error('Expected a client after --client.'); + } + client = parseClient(value); + index++; + } else if (arg?.startsWith('--client=')) { + client = parseClient(arg.slice('--client='.length)); + } else if (arg === '--root') { + index++; + } else if (arg?.startsWith('--root=')) { + // Root values are validated once by resolveInitRoot below. } else { - console.log(`\n--- Instruction Block Preview (will create ${instructionPath}) ---`); + throw new Error(`Unknown init option "${arg}".`); } + } + + return { + ...(client ? { client } : {}), + rootPath: resolveInitRoot(args, process.env.CODEBASE_ROOT, process.cwd()), + yes, + help: false + }; +} + +function printInitUsage(): void { + console.log('codebase-context init [options]'); + console.log(''); + console.log('Set up MCP config for an AI client.'); + console.log(''); + console.log('Options:'); + console.log(' --client claude-code, cursor, codex, or opencode'); + console.log(' --root Project root (default: CODEBASE_ROOT or cwd)'); + console.log(' --yes, -y Write the MCP config without prompts'); + console.log(' --help, -h Show this help'); +} + +function getCurrentCliEntrypoint(): string { + const entrypoint = process.argv[1]; + if (!entrypoint) { + throw new Error('Cannot locate the codebase-context CLI entrypoint.'); + } + return path.resolve(entrypoint); +} + +export async function handleInitCli(args: string[]): Promise { + let options: InitOptions; + try { + options = parseInitOptions(args); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + printInitUsage(); + process.exitCode = 1; + return; + } + + if (options.help) { + printInitUsage(); + return; + } + if (options.yes && !options.client) { + console.error('Error: --yes requires --client.'); + printInitUsage(); + process.exitCode = 1; + return; + } + + console.log('\nSet up codebase-context for your AI client\n'); + + const client = + options.client ?? + (await select({ + message: 'Which client?', + choices: [ + { name: 'Claude Code', value: 'claude-code' }, + { name: 'Cursor', value: 'cursor' }, + { name: 'Codex', value: 'codex' }, + { name: 'OpenCode', value: 'opencode' } + ] + })); + + const mcpResult = generateMcpConfig(client, { + rootPath: options.rootPath, + nodeExecutable: process.execPath, + serverEntrypoint: getCurrentCliEntrypoint() + }); + + console.log('\n--- MCP Config Preview ---'); + console.log( + 'Existing unrelated settings will be preserved; a different codebase-context entry will stop setup.' + ); + console.log(`File: ${mcpResult.path}\n${mcpResult.content}`); + + const applyMcp = options.yes + ? true + : await confirm({ + message: 'Apply MCP config? [y/N]', + default: false + }); + + const instructionPath = options.yes ? null : resolveInstructionFilePath(client, options.rootPath); + + let applyInstruction = false; + if (instructionPath !== null) { + console.log(`\n--- Instruction Block Preview (will append to ${instructionPath}) ---`); console.log(generateInstructionBlock()); applyInstruction = await confirm({ @@ -245,47 +522,45 @@ export async function handleInitCli(_argv: string[]): Promise { }); } - // Execute confirmed actions - - if (applyMcp) { - if (mcpResult.kind === 'file') { - const dir = path.dirname(mcpResult.path); - if (dir && dir !== '.') { - await fs.mkdir(dir, { recursive: true }); + try { + if (applyMcp) { + try { + await fs.access(options.rootPath); + } catch { + throw new Error(`Project root does not exist or is not accessible: ${options.rootPath}`); } + const dir = path.dirname(mcpResult.path); + await fs.mkdir(dir, { recursive: true }); const mergedConfig = - client === 'cursor' || client === 'opencode' - ? await _buildMergedMcpContent(mcpResult.path, mcpResult.content, client) - : { content: mcpResult.content, mergedFromExisting: false }; + client === 'codex' + ? await _buildMergedCodexContent(mcpResult.path, mcpResult.content) + : await _buildMergedMcpContent(mcpResult.path, mcpResult.content, client); await fs.writeFile(mcpResult.path, mergedConfig.content, 'utf8'); if (mergedConfig.mergedFromExisting) { console.log(`Merged: ${mcpResult.path} (existing entries preserved)`); } console.log(`Written: ${mcpResult.path}`); - } else { - const [cmd, ...rest] = mcpResult.args; - try { - execFileSync(cmd, rest, { stdio: 'inherit' }); - } catch { - console.log( - `Could not run '${cmd}' automatically. Run it yourself:\n ${cmd} ${rest.join(' ')}` - ); - } } - } - if (applyInstruction && instructionPath !== null) { - const status = await _appendInstructionBlock(instructionPath); - if (status === 'skipped') { - console.log(`Instruction block already present in ${instructionPath}, skipping.`); - } else { - console.log(`Written: ${instructionPath}`); + if (applyInstruction && instructionPath !== null) { + const status = await _appendInstructionBlock(instructionPath); + if (status === 'skipped') { + console.log(`Instruction block already present in ${instructionPath}, skipping.`); + } else { + console.log(`Written: ${instructionPath}`); + } } + } catch (error) { + console.error(`Init failed: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + return; } - console.log( - '\nNext steps:\n' + - ' Run `npx codebase-context map` to see your codebase conventions\n' + - ' Start the HTTP server: npx codebase-context --http\n' - ); + const nextSteps = + client === 'claude-code' || client === 'codex' + ? ' Restart or reopen the client in this project to load the MCP server.\n' + + ' Run `npx codebase-context map` to see your codebase conventions\n' + : ' Run `npx codebase-context map` to see your codebase conventions\n' + + ' Start the HTTP server: npx codebase-context --http\n'; + console.log(`\nNext steps:\n${nextSteps}`); } diff --git a/tests/cli-init-bootstrap.test.ts b/tests/cli-init-bootstrap.test.ts new file mode 100644 index 0000000..c16c3be --- /dev/null +++ b/tests/cli-init-bootstrap.test.ts @@ -0,0 +1,33 @@ +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { generateMcpConfig } from '../src/cli-init.js'; +import { resolveRootPath } from '../src/index.js'; + +describe('init bootstrap root', () => { + it('passes the configured project root through the MCP server argv path', () => { + const projectRoot = path.resolve('C:/projects/An App With Spaces'); + const previousArgv = process.argv; + const previousEnvRoot = process.env.CODEBASE_ROOT; + const generated = generateMcpConfig('claude-code', { + rootPath: projectRoot, + nodeExecutable: process.execPath, + serverEntrypoint: path.resolve('C:/codebase-context/dist/index.js') + }); + if (generated.kind !== 'file') throw new Error('expected project MCP config'); + + const parsed = JSON.parse(generated.content) as { + mcpServers: { 'codebase-context': { command: string; args: string[] } }; + }; + const server = parsed.mcpServers['codebase-context']; + process.argv = [server.command, ...server.args]; + process.env.CODEBASE_ROOT = path.resolve('C:/projects/Other App'); + + try { + expect(resolveRootPath()).toBe(projectRoot); + } finally { + process.argv = previousArgv; + if (previousEnvRoot === undefined) delete process.env.CODEBASE_ROOT; + else process.env.CODEBASE_ROOT = previousEnvRoot; + } + }); +}); diff --git a/tests/cli-init.test.ts b/tests/cli-init.test.ts index 1dc718e..8d139a2 100644 --- a/tests/cli-init.test.ts +++ b/tests/cli-init.test.ts @@ -13,55 +13,82 @@ vi.mock('node:fs/promises', async (importOriginal) => { }; }); +vi.mock('@inquirer/prompts', () => ({ + select: vi.fn(), + confirm: vi.fn() +})); + import * as fsMod from 'node:fs/promises'; +import { confirm, select } from '@inquirer/prompts'; import { generateMcpConfig, generateInstructionBlock, resolveInstructionFilePath, + resolveInitRoot, _appendInstructionBlock, - _buildMergedMcpContent + _buildMergedMcpContent, + _buildMergedCodexContent, + handleInitCli } from '../src/cli-init.js'; // --- generateMcpConfig --- describe('generateMcpConfig', () => { - it('claude-code returns a command with mcp add --transport http', () => { - const result = generateMcpConfig('claude-code'); - expect(result.kind).toBe('command'); - if (result.kind !== 'command') return; - expect(result.args[0]).toBe('mcp'); - expect(result.args).toContain('--transport'); - expect(result.args).toContain('http'); - expect(result.args).toContain('codebase-context'); - expect(result.args).toContain('http://127.0.0.1:3100/mcp'); + const projectRoot = path.resolve('C:/projects/Example Project'); + const nodeExecutable = path.resolve('C:/Program Files/nodejs/node.exe'); + const serverEntrypoint = path.resolve('C:/Program Files/codebase-context/dist/index.js'); + + it('Claude Code gets a project stdio config that passes the project root', () => { + const result = generateMcpConfig('claude-code', { + rootPath: projectRoot, + nodeExecutable, + serverEntrypoint + }); + + expect(result.kind).toBe('file'); + if (result.kind !== 'file') return; + expect(result.path).toBe(path.join(projectRoot, '.mcp.json')); + expect(JSON.parse(result.content)).toEqual({ + mcpServers: { + 'codebase-context': { + command: nodeExecutable, + args: [serverEntrypoint, projectRoot] + } + } + }); }); it('cursor returns a file at .cursor/mcp.json with http type', () => { const result = generateMcpConfig('cursor'); expect(result.kind).toBe('file'); if (result.kind !== 'file') return; - expect(result.path).toBe('.cursor/mcp.json'); + expect(result.path).toBe(path.resolve('.cursor/mcp.json')); const parsed = JSON.parse(result.content) as { mcpServers: { 'codebase-context': { type: string } }; }; expect(parsed.mcpServers['codebase-context'].type).toBe('http'); }); - it('codex returns a command with mcp add', () => { - const result = generateMcpConfig('codex'); - expect(result.kind).toBe('command'); - if (result.kind !== 'command') return; - expect(result.args).toContain('mcp'); - expect(result.args).toContain('add'); - expect(result.args).toContain('codebase-context'); - expect(result.args).toContain('http://127.0.0.1:3100/mcp'); + it('Codex gets a project TOML config that passes paths containing spaces safely', () => { + const result = generateMcpConfig('codex', { + rootPath: projectRoot, + nodeExecutable, + serverEntrypoint + }); + + expect(result.kind).toBe('file'); + if (result.kind !== 'file') return; + expect(result.path).toBe(path.join(projectRoot, '.codex', 'config.toml')); + expect(result.content).toBe( + `[mcp_servers.codebase-context]\ncommand = ${JSON.stringify(nodeExecutable)}\nargs = [${JSON.stringify(serverEntrypoint)}, ${JSON.stringify(projectRoot)}]\nstartup_timeout_sec = 120\n` + ); }); it('opencode returns a file at opencode.json with remote type', () => { const result = generateMcpConfig('opencode'); expect(result.kind).toBe('file'); if (result.kind !== 'file') return; - expect(result.path).toBe('opencode.json'); + expect(result.path).toBe(path.resolve('opencode.json')); const parsed = JSON.parse(result.content) as { mcp: { 'codebase-context': { type: string } }; }; @@ -69,6 +96,32 @@ describe('generateMcpConfig', () => { }); }); +describe('resolveInitRoot', () => { + it('uses explicit root before CODEBASE_ROOT and cwd', () => { + expect( + resolveInitRoot(['--root', 'C:/projects/Explicit Project'], 'C:/projects/Env', 'C:/other') + ).toBe(path.resolve('C:/projects/Explicit Project')); + }); + + it('uses CODEBASE_ROOT when no explicit root is supplied', () => { + expect(resolveInitRoot([], 'C:/projects/Env Project', 'C:/other')).toBe( + path.resolve('C:/projects/Env Project') + ); + }); + + it('uses cwd when neither root override is supplied', () => { + expect(resolveInitRoot([], undefined, 'C:/projects/Current Project')).toBe( + path.resolve('C:/projects/Current Project') + ); + }); + + it('treats an empty CODEBASE_ROOT as unset', () => { + expect(resolveInitRoot([], '', 'C:/projects/Current Project')).toBe( + path.resolve('C:/projects/Current Project') + ); + }); +}); + // --- generateInstructionBlock --- describe('generateInstructionBlock', () => { @@ -173,7 +226,11 @@ describe('_buildMergedMcpContent', () => { const generated = generateMcpConfig('cursor'); if (generated.kind !== 'file') throw new Error('expected file config'); - const merged = await _buildMergedMcpContent('/test/.cursor/mcp.json', generated.content, 'cursor'); + const merged = await _buildMergedMcpContent( + '/test/.cursor/mcp.json', + generated.content, + 'cursor' + ); const parsed = JSON.parse(merged.content) as { mcpServers: Record; someOtherKey: boolean; @@ -191,7 +248,72 @@ describe('_buildMergedMcpContent', () => { }); }); - it('preserves existing opencode mcp entries and updates codebase-context deterministically', async () => { + it('preserves existing Claude Code mcpServers entries and adds codebase-context', async () => { + readFileMock.mockResolvedValue( + JSON.stringify({ + mcpServers: { + 'existing-server': { type: 'stdio', command: 'node', args: ['server.js'] } + }, + someOtherKey: true + }) as unknown as Buffer + ); + + const generated = generateMcpConfig('claude-code', { + rootPath: path.resolve('C:/projects/app'), + nodeExecutable: 'C:/node.exe', + serverEntrypoint: path.resolve('C:/cbc/index.js') + }); + if (generated.kind !== 'file') throw new Error('expected file config'); + + const merged = await _buildMergedMcpContent( + '/test/.mcp.json', + generated.content, + 'claude-code' + ); + const parsed = JSON.parse(merged.content) as { + mcpServers: Record; + someOtherKey: boolean; + }; + + expect(merged.mergedFromExisting).toBe(true); + expect(parsed.someOtherKey).toBe(true); + expect(parsed.mcpServers['existing-server']).toEqual({ + type: 'stdio', + command: 'node', + args: ['server.js'] + }); + expect(parsed.mcpServers['codebase-context']).toEqual({ + command: 'C:/node.exe', + args: [path.resolve('C:/cbc/index.js'), path.resolve('C:/projects/app')] + }); + }); + + it('refuses to overwrite a disabled Claude Code codebase-context entry', async () => { + readFileMock.mockResolvedValue( + JSON.stringify({ + mcpServers: { + 'codebase-context': { + type: 'stdio', + command: 'node', + args: ['server.js'], + disabled: true + } + } + }) as unknown as Buffer + ); + const generated = generateMcpConfig('claude-code', { + rootPath: path.resolve('C:/projects/app'), + nodeExecutable: 'C:/node.exe', + serverEntrypoint: path.resolve('C:/cbc/index.js') + }); + if (generated.kind !== 'file') throw new Error('expected file config'); + + await expect( + _buildMergedMcpContent('/test/.mcp.json', generated.content, 'claude-code') + ).rejects.toThrow(/already contains a different codebase-context configuration/i); + }); + + it('refuses to overwrite an existing conflicting OpenCode codebase-context entry', async () => { readFileMock.mockResolvedValue( JSON.stringify({ $schema: 'https://opencode.ai/config.json', @@ -206,21 +328,276 @@ describe('_buildMergedMcpContent', () => { const generated = generateMcpConfig('opencode'); if (generated.kind !== 'file') throw new Error('expected file config'); - const merged = await _buildMergedMcpContent('/test/opencode.json', generated.content, 'opencode'); - const parsed = JSON.parse(merged.content) as { - mcp: Record; - extra: string; + await expect( + _buildMergedMcpContent('/test/opencode.json', generated.content, 'opencode') + ).rejects.toThrow(/already contains a different codebase-context configuration/i); + }); +}); + +describe('handleInitCli', () => { + const readFileMock = vi.mocked(fsMod.readFile); + const writeFileMock = vi.mocked(fsMod.writeFile); + const mkdirMock = vi.mocked(fsMod.mkdir); + const accessMock = vi.mocked(fsMod.access); + + beforeEach(() => { + vi.resetAllMocks(); + writeFileMock.mockResolvedValue(undefined); + mkdirMock.mockResolvedValue(undefined); + readFileMock.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' })); + }); + + it('writes only the selected project config in non-interactive mode', async () => { + const projectRoot = path.resolve('C:/projects/Configured App'); + + await handleInitCli(['--client', 'claude-code', '--yes', '--root', projectRoot]); + + expect(select).not.toHaveBeenCalled(); + expect(confirm).not.toHaveBeenCalled(); + expect(writeFileMock).toHaveBeenCalledOnce(); + const [writtenPath, writtenContent] = writeFileMock.mock.calls[0] as [ + string, + string, + ...unknown[] + ]; + expect(writtenPath).toBe(path.join(projectRoot, '.mcp.json')); + const parsed = JSON.parse(writtenContent) as { + mcpServers: { 'codebase-context': { args: string[] } }; }; + expect(parsed.mcpServers['codebase-context'].args.at(-1)).toBe(projectRoot); + expect(mkdirMock).toHaveBeenCalledWith(projectRoot, { recursive: true }); + }); + + it('writes a project Codex TOML file when Codex is selected non-interactively', async () => { + const projectRoot = path.resolve('C:/projects/Codex App'); + + await handleInitCli(['--client', 'codex', '--yes', '--root', projectRoot]); + + expect(writeFileMock).toHaveBeenCalledOnce(); + const [writtenPath, writtenContent] = writeFileMock.mock.calls[0] as [ + string, + string, + ...unknown[] + ]; + expect(writtenPath).toBe(path.join(projectRoot, '.codex', 'config.toml')); + expect(writtenContent).toContain('[mcp_servers.codebase-context]'); + expect(writtenContent).toContain(`args = [`); + expect(writtenContent).toContain(JSON.stringify(projectRoot)); + expect(mkdirMock).toHaveBeenCalledWith(path.join(projectRoot, '.codex'), { recursive: true }); + }); + + it('uses cwd as the project when init is run without a root override', async () => { + const previousEnvRoot = process.env.CODEBASE_ROOT; + delete process.env.CODEBASE_ROOT; + + try { + await handleInitCli(['--client', 'codex', '--yes']); + } finally { + if (previousEnvRoot === undefined) delete process.env.CODEBASE_ROOT; + else process.env.CODEBASE_ROOT = previousEnvRoot; + } + + const [writtenPath, writtenContent] = writeFileMock.mock.calls[0] as [ + string, + string, + ...unknown[] + ]; + expect(writtenPath).toBe(path.join(process.cwd(), '.codex', 'config.toml')); + expect(writtenContent).toContain(JSON.stringify(path.resolve(process.cwd()))); + }); + + it('does not overwrite a conflicting project MCP entry', async () => { + const projectRoot = path.resolve('C:/projects/Configured App'); + readFileMock.mockResolvedValue( + JSON.stringify({ + mcpServers: { + 'codebase-context': { disabled: true, command: 'old-server' } + } + }) as unknown as Buffer + ); + const previousExitCode = process.exitCode; + + await handleInitCli(['--client', 'claude-code', '--yes', '--root', projectRoot]); + + expect(writeFileMock).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + process.exitCode = previousExitCode; + }); + + it('does not create a project config when the selected root is missing', async () => { + const projectRoot = path.resolve('C:/projects/Missing App'); + accessMock.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' })); + const previousExitCode = process.exitCode; + + await handleInitCli(['--client', 'codex', '--yes', '--root', projectRoot]); + + expect(mkdirMock).not.toHaveBeenCalled(); + expect(writeFileMock).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + process.exitCode = previousExitCode; + }); + + it('shows init help without prompting or writing files', async () => { + await handleInitCli(['--help']); + + expect(select).not.toHaveBeenCalled(); + expect(confirm).not.toHaveBeenCalled(); + expect(writeFileMock).not.toHaveBeenCalled(); + }); + + it('keeps the interactive preview read-only when the user declines', async () => { + vi.mocked(select).mockResolvedValue('claude-code'); + vi.mocked(confirm).mockResolvedValue(false); + + await handleInitCli([]); + + expect(writeFileMock).not.toHaveBeenCalled(); + }); +}); + +describe('_buildMergedCodexContent', () => { + const readFileMock = vi.mocked(fsMod.readFile); + const writeFileMock = vi.mocked(fsMod.writeFile); + const mkdirMock = vi.mocked(fsMod.mkdir); + + beforeEach(() => { + vi.resetAllMocks(); + writeFileMock.mockResolvedValue(undefined); + mkdirMock.mockResolvedValue(undefined); + readFileMock.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' })); + }); + + const runCodexSetupWithExistingConfig = async (existingToml: string) => { + const projectRoot = path.resolve('C:/projects/Codex Existing'); + const initialBytes = Buffer.from(existingToml, 'utf8'); + let persistedBytes = Buffer.from(initialBytes); + readFileMock.mockResolvedValue(existingToml as unknown as Buffer); + writeFileMock.mockImplementation(async (_filePath, content) => { + persistedBytes = Buffer.from(content as string, 'utf8'); + }); + + const previousExitCode = process.exitCode; + process.exitCode = undefined; + try { + await handleInitCli(['--client', 'codex', '--yes', '--root', projectRoot]); + return { initialBytes, persistedBytes, exitCode: process.exitCode }; + } finally { + process.exitCode = previousExitCode; + } + }; + + it('preserves unrelated Codex TOML settings when adding the project server', async () => { + readFileMock.mockResolvedValue( + '[model]\nmodel = "gpt-example"\n\n[mcp_servers.other]\ncommand = "other"\n' as unknown as Buffer + ); + const generated = generateMcpConfig('codex', { + rootPath: 'C:/projects/app', + nodeExecutable: 'C:/node.exe', + serverEntrypoint: 'C:/cbc/index.js' + }); + if (generated.kind !== 'file') throw new Error('expected file config'); + + const merged = await _buildMergedCodexContent(generated.path, generated.content); expect(merged.mergedFromExisting).toBe(true); - expect(parsed.extra).toBe('value'); - expect(parsed.mcp['existing-server']).toEqual({ - type: 'remote', - url: 'http://127.0.0.1:5000/mcp' + expect(merged.content).toContain('[model]\nmodel = "gpt-example"'); + expect(merged.content).toContain('[mcp_servers.other]\ncommand = "other"'); + expect(merged.content).toContain('[mcp_servers.codebase-context]'); + }); + + it('leaves the same Codex codebase-context entry unchanged on a repeated setup', async () => { + const generated = generateMcpConfig('codex', { + rootPath: path.resolve('C:/projects/app'), + nodeExecutable: 'C:/node.exe', + serverEntrypoint: path.resolve('C:/cbc/index.js') }); - expect(parsed.mcp['codebase-context']).toEqual({ - type: 'remote', - url: 'http://127.0.0.1:3100/mcp' + if (generated.kind !== 'file') throw new Error('expected file config'); + readFileMock.mockResolvedValue(generated.content as unknown as Buffer); + + const merged = await _buildMergedCodexContent(generated.path, generated.content); + + expect(merged).toEqual({ content: generated.content, mergedFromExisting: true }); + }); + + it('treats quoted table keys as the same idempotent Codex entry', async () => { + const generated = generateMcpConfig('codex', { + rootPath: path.resolve('C:/projects/app'), + nodeExecutable: 'C:/node.exe', + serverEntrypoint: path.resolve('C:/cbc/index.js') + }); + if (generated.kind !== 'file') throw new Error('expected file config'); + const existing = generated.content.replace( + '[mcp_servers.codebase-context]', + '["mcp_servers"."codebase-context"]' + ); + readFileMock.mockResolvedValue(existing as unknown as Buffer); + + const merged = await _buildMergedCodexContent(generated.path, generated.content); + + expect(merged).toEqual({ content: existing, mergedFromExisting: true }); + }); + + it('refuses a conflicting or disabled Codex codebase-context entry', async () => { + readFileMock.mockResolvedValue( + '[mcp_servers.codebase-context]\ndisabled = true\ncommand = "old"\n' as unknown as Buffer + ); + const generated = generateMcpConfig('codex', { + rootPath: 'C:/projects/app', + nodeExecutable: 'C:/node.exe', + serverEntrypoint: 'C:/cbc/index.js' }); + if (generated.kind !== 'file') throw new Error('expected file config'); + + await expect(_buildMergedCodexContent(generated.path, generated.content)).rejects.toThrow( + /already contains a different codebase-context configuration/i + ); }); + + it.each([ + { + name: 'double-quoted parent and child keys', + toml: '["mcp_servers"."codebase-context"]\ncommand = "old"\n' + }, + { + name: 'double-quoted parent and bare child key', + toml: '["mcp_servers".codebase-context]\ncommand = "old"\n' + } + ])( + 'refuses to replace a Codex entry under $name without changing its bytes', + async ({ toml }) => { + const result = await runCodexSetupWithExistingConfig(toml); + + expect(result.exitCode).toBe(1); + expect(writeFileMock).not.toHaveBeenCalled(); + expect(result.persistedBytes).toEqual(result.initialBytes); + } + ); + + it.each([ + { + name: 'inline child in the bare parent table', + toml: '[mcp_servers]\ncodebase-context = { command = "old", args = [] }\n' + }, + { + name: 'dotted child in the double-quoted parent table', + toml: '["mcp_servers"]\ncodebase-context.command = "old"\n' + }, + { + name: 'dotted child in the single-quoted parent table', + toml: "['mcp_servers']\n'codebase-context'.command = \"old\"\n" + }, + { + name: 'inline root table under a double-quoted key', + toml: '"mcp_servers" = { "codebase-context" = { command = "old", args = [] } }\n' + } + ])( + 'refuses to replace a Codex entry under $name without changing its bytes', + async ({ toml }) => { + const result = await runCodexSetupWithExistingConfig(toml); + + expect(result.exitCode).toBe(1); + expect(writeFileMock).not.toHaveBeenCalled(); + expect(result.persistedBytes).toEqual(result.initialBytes); + } + ); }); From 7d0e65e9b4424758482c9f910454e5d5becb2f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Fri, 16 Oct 2026 20:14:42 +0200 Subject: [PATCH 098/100] docs: keep release instructions accurate and easy to follow --- README.md | 6 +++--- docs/capabilities.md | 10 +++++----- docs/client-setup.md | 4 ++-- tests/release-truth-surfaces.test.ts | 13 +++++++++++++ 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a6ee760..adab80b 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Replace `[feature]` with something you want to find. The first run downloads a l ### New in 2.4.0 -- Opt-in NestJS support from community contributor Aaron Olin. See the [analyzer options](./docs/capabilities.md#analyzers). +- NestJS support from community contributor Aaron Olin. See the [analyzer details](./docs/capabilities.md#analyzers). - Better React/Next.js detection and a fix for reranker passage scoring. - Transformers.js v4 for local models. @@ -72,7 +72,7 @@ The default connection is `stdio` (standard input/output): your client starts th ### Team patterns and examples -`get_team_patterns` shows the approaches used in the repository and points to representative files. Version `2.4.0` includes dedicated analyzers for Angular, React, Next.js and opt-in NestJS, with a generic analyzer for other stacks. +`get_team_patterns` shows the approaches used in the repository and points to representative files. Version `2.4.0` includes dedicated analyzers for Angular, React, Next.js and NestJS, with a generic analyzer for other stacks. ### Project memory @@ -157,7 +157,7 @@ The method and failures are documented so the measurements can be inspected with - Retrieval measurements describe expected-file coverage, file precision, and reported `peakPrivateGb`, not patch correctness or end-to-end coding quality. - The paired token observation covers two frozen investigation tasks and records observed agent behavior; it is not a universal token or time guarantee. - Setup checks differ by client. The detailed guide distinguishes a written config, a config recognized by the client, a local connection, and instructions checked only against official docs. -- Version `2.4.0` has dedicated Angular, React, Next.js and opt-in NestJS analyzers. Other projects use the generic analyzer and the language parsers available for that stack. +- Version `2.4.0` has dedicated Angular, React, Next.js and NestJS analyzers. Other projects use the generic analyzer and the language parsers available for that stack. - The default searchable-chunk limit is 5,000 per project. In `~/.codebase-context/config.json`, set `projects[].parsing.maxChunks` for a project that needs a higher limit. - The agent must identify its repository in tool calls when no default or unambiguous client root is available. Concurrent HTTP client isolation is not established by the stdio routing checks. diff --git a/docs/capabilities.md b/docs/capabilities.md index 59806d6..7d1dea8 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -1,6 +1,6 @@ # Capabilities Reference -Technical reference for the 2.4.0 setup preview. The public product posture is map first, find second: the bounded conventions map is the first-call surface, and search narrows to the right local example after that. For the user-facing overview, see [README.md](../README.md). +Technical reference for Codebase Context 2.4.0. Start with the repository map, then search for relevant code. For an overview, see [README.md](../README.md). **These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). @@ -29,7 +29,7 @@ Copy-pasteable client config templates are shipped in the package: - `templates/mcp/stdio/.mcp.json` — stdio setup for `.mcp.json`-style clients - `templates/mcp/http/.mcp.json` — HTTP setup for `.mcp.json`-style clients -Use Node.js 22 or newer with the 2.4.0 commands shown here. For 2.4.0 setup preview recipes and their verification limits, see the [client setup guide](./client-setup.md). +Use Node.js 22 or newer with the 2.4.0 commands shown here. For setup commands and what was checked, see the [client setup guide](./client-setup.md). ## CLI Reference @@ -52,7 +52,7 @@ For a command gallery with examples, see `docs/cli.md`. | `memory add` | `--type`, `--category`, `--memory`, `--reason` | `remember` | | `memory remove ` | — | — | -Commands that list `--json` above support raw JSON output. For MCP client registration, follow the [2.4.0 setup preview recipes](./client-setup.md). The existing `init` wizard is interactive and generates HTTP setup; see that guide for its current limitations. Errors go to stderr with exit code 1. +Commands that list `--json` above support raw JSON output. Follow the [client setup guide](./client-setup.md) to connect your client. The `init` command can set up Claude Code and Codex for the current project; the guide covers the other clients. Errors go to stderr with exit code 1. ```bash # Quick examples @@ -276,7 +276,7 @@ Impact is 2-hop transitive: direct importers (hop 1) and their importers (hop 2) - **Angular**: signals, standalone components, control flow syntax, lifecycle hooks, DI patterns, component metadata - **React**: function/class components, custom hooks, context usage, memoization, Suspense, ecosystem signal extraction - **Next.js**: App Router and Pages Router detection, route/API classification, route paths, `"use client"`, metadata exports -- **NestJS (opt-in)**: modules, controllers, providers and route handlers, contributed by Aaron Olin. The library indexer accepts `config: { analyzers: { nestjs: { enabled: true } } }`; the CLI/server project configuration does not currently expose analyzer enablement. +- **NestJS**: modules, controllers, providers and route handlers, contributed by Aaron Olin. Matching NestJS source is detected automatically. - **Generic**: 30+ have indexing/retrieval coverage including PHP, Ruby, Swift, Scala, Shell, config/markup., 10 languages have full symbol extraction (Tree-sitter: TypeScript, JavaScript, Python, Java, Kotlin, C, C++, C#, Go, Rust). Notes: @@ -305,6 +305,6 @@ Reproducible evaluation is shipped as a CLI entrypoint backed by shared scoring - **Symbol refs are not a call-graph.** `get_symbol_references` counts identifier-node occurrences in the AST (comments/strings excluded via Tree-sitter). It does not distinguish call sites from type annotations, variable assignments, or imports. Full call-site-specific analysis (`call_expression` nodes only) is a roadmap item. - **Impact is 2-hop max.** `computeImpactCandidates` walks direct importers then their importers. Full BFS reachability is on the roadmap. -- **The 2.4.0 preview source has dedicated Angular, React, Next.js and NestJS analyzers.** Other languages use the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). +- **Version 2.4.0 has dedicated Angular, React, Next.js and NestJS analyzers.** Other languages use the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). - **Default embedding model is `bge-small-en-v1.5` (512-token context).** Granite (8192 context) is opt-in via `EMBEDDING_MODEL`. OpenAI is opt-in via `EMBEDDING_PROVIDER=openai` — sends code externally. - **Patterns are file-level frequency counts.** Not semantic clustering. Rising/Declining trend is derived from git commit recency for files using each pattern, not from usage semantics. diff --git a/docs/client-setup.md b/docs/client-setup.md index c052e65..91e74e2 100644 --- a/docs/client-setup.md +++ b/docs/client-setup.md @@ -4,7 +4,7 @@ Codebase Context runs as a local MCP server: a tool your AI editor or command-line client can call while it works. Register it without a fixed folder, then let the agent supply its current repository through the `project` tool parameter. These recipes target the 2.4.0 release. -Use Node.js 22 or newer for these recipes. The package still declares Node 18+, but the built pack failed to index source on Node 18.20.4 with `ReferenceError: crypto is not defined` in UUID chunk creation. Node 22 is the lowest version used successfully in the package check; the engine declaration remains an open compatibility issue. +Use Node.js 22 or newer, as required by the package. The copyable registration commands target PowerShell or a POSIX terminal. Windows Command Prompt was not tested; OpenCode's quoted separator is shell-dependent. @@ -12,7 +12,7 @@ The copyable registration commands target PowerShell or a POSIX terminal. Window Claude Code, Codex CLI and OpenCode registration commands were checked again on October 6 without a project argument. The other clients retain their older config-shape evidence; no-folder native agent use is unverified for them. See the dated notes below. -Each client section says how far its isolated Windows check went: +Each client section says how far its earlier Windows check went. Those checks used published `2.2.0` unless another build is named; they do not verify the 2.4.0 commands: | Status | What it means | | ------------------------------ | ------------------------------------------------------------------------------------------ | diff --git a/tests/release-truth-surfaces.test.ts b/tests/release-truth-surfaces.test.ts index 49d56d4..b2186a2 100644 --- a/tests/release-truth-surfaces.test.ts +++ b/tests/release-truth-surfaces.test.ts @@ -175,4 +175,17 @@ describe('release truth surfaces', () => { expect(capabilities).not.toContain('what `codebase-context` ships today'); expect(capabilities).not.toContain('Published 2.4.0 has'); }); + + it('keeps release wording, analyzer claims and setup checks accurate', () => { + const capabilities = readText('docs/capabilities.md'); + const setup = readText('docs/client-setup.md'); + const beforeFirstClient = setup.slice(0, setup.indexOf('## Claude Code')); + + expect(capabilities).not.toMatch(/\b2\.4\.0 (?:setup )?preview\b/i); + expect(`${readme}\n${capabilities}`).not.toMatch(/opt-in NestJS|NestJS \(opt-in\)/i); + expect(capabilities).not.toContain('analyzers: { nestjs: { enabled: true } }'); + expect(capabilities).not.toContain('generates HTTP setup'); + expect(setup).not.toContain('package still declares Node 18+'); + expect(beforeFirstClient).toContain('`2.2.0`'); + }); }); From 1928b267347a165c08a21b62acf608ecc781aa32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Rossell=C3=B3=20Colom?= <74001504+PatrickSys@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:49:37 +0200 Subject: [PATCH 099/100] docs: make install commands work against published npm versions (#141) * docs: use published npm install commands * docs: keep 2.2.0 analyzers current and mark 2.4.0-only commands Addresses Codex/Greptile review on #141: Angular/React/Next.js analyzers ship in 2.2.0 (only NestJS is upcoming); init --client/--yes, codebase-context-review and parsing.maxChunks require unpublished 2.4.0, so those commands are pinned and labelled. Release-truth test allows 2.4.0 pins only next to a 'requires 2.4.0' note. --- README.md | 32 +++++------ docs/capabilities.md | 26 ++++----- docs/cli.md | 52 ++++++++--------- docs/client-setup.md | 62 ++++++++++---------- docs/demo.md | 14 ++--- docs/review-context.md | 6 +- tests/release-truth-surfaces.test.ts | 84 ++++++++++++++-------------- 7 files changed, 139 insertions(+), 137 deletions(-) diff --git a/README.md b/README.md index adab80b..aa7f416 100644 --- a/README.md +++ b/README.md @@ -6,19 +6,17 @@ Codebase Context gives coding agents a map of your repository through MCP, for d ## Set up your AI client -**These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). - -Use Node.js 22 or newer. Choose your coding tool and run its command once; the examples target the 2.4.0 release: +Use Node.js 22 or newer. These commands use the current npm release of Codebase Context: ```bash # Claude Code -claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.4.0 +claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context # Codex CLI -codex mcp add codebase-context -- npx -y codebase-context@2.4.0 +codex mcp add codebase-context -- npx -y codebase-context # OpenCode 1.x (keep the quoted separator on Windows) -opencode mcp add codebase-context '--' npx -y codebase-context@2.4.0 +opencode mcp add codebase-context '--' npx -y codebase-context ``` Start a new agent session in your project, then ask: @@ -27,7 +25,7 @@ Start a new agent session in your project, then ask: Replace `[feature]` with something you want to find. The first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. -### New in 2.4.0 +### Planned for 2.4.0 (not yet published) - NestJS support from community contributor Aaron Olin. See the [analyzer details](./docs/capabilities.md#analyzers). - Better React/Next.js detection and a fix for reranker passage scoring. @@ -42,7 +40,7 @@ For Codex Desktop, create or merge `.codex/config.toml` in the project you want ```toml [mcp_servers.codebase-context] command = "npx" -args = ["-y", "codebase-context@2.4.0"] +args = ["-y", "codebase-context"] startup_timeout_sec = 120 ``` @@ -52,10 +50,10 @@ Other clients use their own setup commands: | Client | Shortest current setup | | --------------------------- | ---------------------------------------------------------------------------- | -| Gemini CLI | `gemini mcp add --scope user codebase-context npx -y codebase-context@2.4.0` | +| Gemini CLI | `gemini mcp add --scope user codebase-context npx -y codebase-context` | | Cursor | Add `.cursor/mcp.json` | | VS Code with GitHub Copilot | Add `.vscode/mcp.json` | -| GitHub Copilot CLI | `copilot mcp add codebase-context -- npx -y codebase-context@2.4.0` | +| GitHub Copilot CLI | `copilot mcp add codebase-context -- npx -y codebase-context` | | Windsurf | Add `~/.codeium/windsurf/mcp_config.json` | Check an existing same-name entry before replacing it. To give the server a default folder, append that folder's absolute path to the `npx` arguments. The [client setup guide](./docs/client-setup.md) covers scopes, optional fixed-folder configuration and the interactive setup wizard's limitations. @@ -72,7 +70,7 @@ The default connection is `stdio` (standard input/output): your client starts th ### Team patterns and examples -`get_team_patterns` shows the approaches used in the repository and points to representative files. Version `2.4.0` includes dedicated analyzers for Angular, React, Next.js and NestJS, with a generic analyzer for other stacks. +`get_team_patterns` shows the approaches used in the repository and points to representative files. Dedicated analyzers for Angular, React, and Next.js ship in published 2.2.0; NestJS support (along with React and Next.js detection improvements and a reranker fix) is planned for unpublished 2.4.0. ### Project memory @@ -92,16 +90,16 @@ The same information is available from the terminal. Run these commands from you ```bash # Build or refresh the local index -npx -y codebase-context@2.4.0 reindex +npx -y codebase-context reindex # Repository structure, patterns, and representative files -npx -y codebase-context@2.4.0 map +npx -y codebase-context map # Ranked code search -npx -y codebase-context@2.4.0 search --query "auth middleware" +npx -y codebase-context search --query "auth middleware" # Current team patterns -npx -y codebase-context@2.4.0 patterns +npx -y codebase-context patterns ``` One stdio server can route across several repositories. Supply `project` in tool calls to select the intended repository; a successful selection becomes the default for later calls in that process. Some clients also announce workspace roots: one root can auto-select, while an ambiguous selection asks for a project instead of guessing. MCP deprecated Roots in its July 2026 revision, so explicit project selection is the documented default rather than a dependency on client discovery. @@ -157,8 +155,8 @@ The method and failures are documented so the measurements can be inspected with - Retrieval measurements describe expected-file coverage, file precision, and reported `peakPrivateGb`, not patch correctness or end-to-end coding quality. - The paired token observation covers two frozen investigation tasks and records observed agent behavior; it is not a universal token or time guarantee. - Setup checks differ by client. The detailed guide distinguishes a written config, a config recognized by the client, a local connection, and instructions checked only against official docs. -- Version `2.4.0` has dedicated Angular, React, Next.js and NestJS analyzers. Other projects use the generic analyzer and the language parsers available for that stack. -- The default searchable-chunk limit is 5,000 per project. In `~/.codebase-context/config.json`, set `projects[].parsing.maxChunks` for a project that needs a higher limit. +- Dedicated Angular, React, and Next.js analyzers ship in published 2.2.0; NestJS support is planned for unpublished 2.4.0. Other projects use the generic analyzer and the language parsers available for that stack. +- The default searchable-chunk limit is 5,000 per project. Customizing this via `projects[].parsing.maxChunks` in `~/.codebase-context/config.json` requires 2.4.0 (not yet published). - The agent must identify its repository in tool calls when no default or unambiguous client root is available. Concurrent HTTP client isolation is not established by the stdio routing checks. ## Reference diff --git a/docs/capabilities.md b/docs/capabilities.md index 7d1dea8..825c2fe 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -1,8 +1,8 @@ # Capabilities Reference -Technical reference for Codebase Context 2.4.0. Start with the repository map, then search for relevant code. For an overview, see [README.md](../README.md). +Technical reference for the current Codebase Context npm release and the upcoming 2.4.0 source. Start with the repository map, then search for relevant code. For an overview, see [README.md](../README.md). -**These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). +Commands use the current npm release. Features marked as planned for 2.4.0 are not yet available from npm; the current published version is 2.2.0. ## Transport Modes @@ -10,8 +10,8 @@ The server supports two transport modes: | Mode | Command | MCP endpoint | | ------------------- | ------------------------------------------------- | ---------------------------- | -| **stdio** (default) | `npx -y codebase-context@2.4.0` | Spawned process stdin/stdout | -| **HTTP** | `npx -y codebase-context@2.4.0 --http [--port N]` | `http://127.0.0.1:3100/mcp` | +| **stdio** (default) | `npx -y codebase-context` | Spawned process stdin/stdout | +| **HTTP** | `npx -y codebase-context --http [--port N]` | `http://127.0.0.1:3100/mcp` | HTTP defaults to `127.0.0.1:3100`. Override with `--port`, `CODEBASE_CONTEXT_PORT`, or `server.port` in `~/.codebase-context/config.json`. @@ -22,14 +22,14 @@ Per-project config overrides supported today: - `projects[].excludePatterns`: merged with the built-in exclusion set for that project at index time - `projects[].analyzerHints.analyzer`: prefers a registered analyzer by name for that project and falls back safely when the name is missing or invalid - `projects[].analyzerHints.extensions`: adds project-local source extensions for indexing and auto-refresh watching without changing defaults for other projects -- `projects[].parsing.maxChunks`: sets the maximum number of searchable chunks indexed for that project +- `projects[].parsing.maxChunks`: sets the maximum number of searchable chunks indexed for that project (requires 2.4.0, not yet published) Copy-pasteable client config templates are shipped in the package: - `templates/mcp/stdio/.mcp.json` — stdio setup for `.mcp.json`-style clients - `templates/mcp/http/.mcp.json` — HTTP setup for `.mcp.json`-style clients -Use Node.js 22 or newer with the 2.4.0 commands shown here. For setup commands and what was checked, see the [client setup guide](./client-setup.md). +Use Node.js 22 or newer with the commands shown here. For setup commands and what was checked, see the [client setup guide](./client-setup.md). ## CLI Reference @@ -56,11 +56,11 @@ Commands that list `--json` above support raw JSON output. Follow the [client se ```bash # Quick examples -npx -y codebase-context@2.4.0 status -npx -y codebase-context@2.4.0 search --query "auth middleware" --intent edit -npx -y codebase-context@2.4.0 refs --symbol "UserService" --limit 10 -npx -y codebase-context@2.4.0 cycles --scope src/features -npx -y codebase-context@2.4.0 reindex --incremental +npx -y codebase-context status +npx -y codebase-context search --query "auth middleware" --intent edit +npx -y codebase-context refs --symbol "UserService" --limit 10 +npx -y codebase-context cycles --scope src/features +npx -y codebase-context reindex --incremental ``` ## Tool Surface @@ -276,7 +276,7 @@ Impact is 2-hop transitive: direct importers (hop 1) and their importers (hop 2) - **Angular**: signals, standalone components, control flow syntax, lifecycle hooks, DI patterns, component metadata - **React**: function/class components, custom hooks, context usage, memoization, Suspense, ecosystem signal extraction - **Next.js**: App Router and Pages Router detection, route/API classification, route paths, `"use client"`, metadata exports -- **NestJS**: modules, controllers, providers and route handlers, contributed by Aaron Olin. Matching NestJS source is detected automatically. +- **NestJS** (planned for 2.4.0, not yet published): modules, controllers, providers and route handlers, contributed by Aaron Olin. Matching NestJS source is detected automatically. - **Generic**: 30+ have indexing/retrieval coverage including PHP, Ruby, Swift, Scala, Shell, config/markup., 10 languages have full symbol extraction (Tree-sitter: TypeScript, JavaScript, Python, Java, Kotlin, C, C++, C#, Go, Rust). Notes: @@ -305,6 +305,6 @@ Reproducible evaluation is shipped as a CLI entrypoint backed by shared scoring - **Symbol refs are not a call-graph.** `get_symbol_references` counts identifier-node occurrences in the AST (comments/strings excluded via Tree-sitter). It does not distinguish call sites from type annotations, variable assignments, or imports. Full call-site-specific analysis (`call_expression` nodes only) is a roadmap item. - **Impact is 2-hop max.** `computeImpactCandidates` walks direct importers then their importers. Full BFS reachability is on the roadmap. -- **Version 2.4.0 has dedicated Angular, React, Next.js and NestJS analyzers.** Other languages use the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). +- **Planned for unpublished 2.4.0:** NestJS analyzer support, React and Next.js detection improvements, and reranker fixes. Dedicated Angular, React, and Next.js analyzers already ship in published 2.2.0. Other languages use the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). - **Default embedding model is `bge-small-en-v1.5` (512-token context).** Granite (8192 context) is opt-in via `EMBEDDING_MODEL`. OpenAI is opt-in via `EMBEDDING_PROVIDER=openai` — sends code externally. - **Patterns are file-level frequency counts.** Not semantic clustering. Rising/Declining trend is derived from git commit recency for files using each pattern, not from usage semantics. diff --git a/docs/cli.md b/docs/cli.md index 88d5b96..c561ab9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -10,20 +10,20 @@ > Output depends on the repo you run it against. The examples below are illustrative (paths, counts, and detected frameworks will vary). > -> **These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). +> These commands use the current npm release. The examples are illustrative and were not captured from the current npm package. > -> These output examples come from an earlier CLI run; they were not captured from `2.4.0` and do not verify an MCP connection. +> These output examples come from an earlier CLI run and do not verify the current npm package or an MCP connection. > -> The commands below use `codebase-context@2.4.0` and require Node.js 22 or newer. The CLI is intentionally single-project per invocation and targets one root via `CODEBASE_ROOT` or the current working directory. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). +> The commands below use `codebase-context` and require Node.js 22 or newer. The CLI is intentionally single-project per invocation and targets one root via `CODEBASE_ROOT` or the current working directory. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). ## How to run ```bash # CLI only: run from a repo root, or set CODEBASE_ROOT explicitly: -CODEBASE_ROOT=/path/to/repo npx -y codebase-context@2.4.0 status +CODEBASE_ROOT=/path/to/repo npx -y codebase-context status # Commands that show --json in their help support machine output. Human mode is default. -npx -y codebase-context@2.4.0 patterns --json +npx -y codebase-context patterns --json ``` ### ASCII fallback @@ -31,7 +31,7 @@ npx -y codebase-context@2.4.0 patterns --json If your terminal doesn’t render Unicode box-drawing cleanly: ```bash -CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context@2.4.0 patterns +CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context patterns ``` ## Commands @@ -46,7 +46,7 @@ CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context@2.4.0 patterns - `reindex` — rebuild index (full or incremental) - `style-guide` — find style guide sections in docs - `memory list|add|remove` — manage team memory (stored in `.codebase-context/memory.json`) -- MCP client registration — use the 2.4.0 recipes in the [client setup guide](./client-setup.md); `init` is an interactive HTTP setup wizard with the limitations described there +- MCP client registration — use the recipes in the [client setup guide](./client-setup.md); `init` is an interactive HTTP setup wizard with the limitations described there --- @@ -55,9 +55,9 @@ CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context@2.4.0 patterns Run these CLI commands from the project root. Build or refresh the local index first; the first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. Then inspect the conventions map and search for the code you need: ```bash -npx -y codebase-context@2.4.0 reindex -npx -y codebase-context@2.4.0 map -npx -y codebase-context@2.4.0 search --query "auth middleware" +npx -y codebase-context reindex +npx -y codebase-context map +npx -y codebase-context search --query "auth middleware" ``` This is the CLI preparation flow. It is separate from MCP client registration and does not by itself show that an MCP client connected successfully. @@ -65,8 +65,8 @@ This is the CLI preparation flow. It is separate from MCP client registration an ## `map` ```bash -npx -y codebase-context@2.4.0 reindex -npx -y codebase-context@2.4.0 map +npx -y codebase-context reindex +npx -y codebase-context map ``` The conventions map - run this first on an unfamiliar repo. It shows architecture layers, active patterns with adoption rates and trend direction, and the golden files the team treats as the strongest examples. This is also what the MCP server delivers to AI agents via the `codebase://context` resource on first call, before search narrows to a specific local example. @@ -101,7 +101,7 @@ Example output (truncated): ## `metadata` ```bash -npx -y codebase-context@2.4.0 metadata +npx -y codebase-context metadata ``` Example output: @@ -122,7 +122,7 @@ Example output: ## `patterns` ```bash -npx -y codebase-context@2.4.0 patterns +npx -y codebase-context patterns ``` Example output (truncated): @@ -144,8 +144,8 @@ Example output (truncated): ## `search` ```bash -npx -y codebase-context@2.4.0 reindex -npx -y codebase-context@2.4.0 search --query "file watcher" --intent edit --limit 3 +npx -y codebase-context reindex +npx -y codebase-context search --query "file watcher" --intent edit --limit 3 ``` Example output (truncated): @@ -166,7 +166,7 @@ Example output (truncated): ## `refs` ```bash -npx -y codebase-context@2.4.0 refs --symbol "startFileWatcher" --limit 10 +npx -y codebase-context refs --symbol "startFileWatcher" --limit 10 ``` Example output (truncated): @@ -185,7 +185,7 @@ Example output (truncated): ## `cycles` ```bash -npx -y codebase-context@2.4.0 cycles --scope src +npx -y codebase-context cycles --scope src ``` Example output: @@ -201,7 +201,7 @@ Example output: ## `status` ```bash -npx -y codebase-context@2.4.0 status +npx -y codebase-context status ``` Example output: @@ -220,8 +220,8 @@ Example output: ## `reindex` ```bash -npx -y codebase-context@2.4.0 reindex -npx -y codebase-context@2.4.0 reindex --incremental --reason "changed watcher logic" +npx -y codebase-context reindex +npx -y codebase-context reindex --incremental --reason "changed watcher logic" ``` > **MCP server mode**: if you're running codebase-context as an MCP server (long-running process), the index auto-refreshes via a file watcher — you don't need to call `reindex` between edits. Use `reindex` for one-shot CLI runs or to force a full rebuild. @@ -229,7 +229,7 @@ npx -y codebase-context@2.4.0 reindex --incremental --reason "changed watcher lo ## `style-guide` ```bash -npx -y codebase-context@2.4.0 style-guide --query "naming" +npx -y codebase-context style-guide --query "naming" ``` Example output: @@ -242,14 +242,14 @@ No style guides found. ## `memory` ```bash -npx -y codebase-context@2.4.0 memory list -npx -y codebase-context@2.4.0 memory list --query "watcher" +npx -y codebase-context memory list +npx -y codebase-context memory list --query "watcher" -npx -y codebase-context@2.4.0 memory add \ +npx -y codebase-context memory add \ --type gotcha \ --category tooling \ --memory "Use pnpm, not npm" \ --reason "Workspace support and speed" -npx -y codebase-context@2.4.0 memory remove +npx -y codebase-context memory remove ``` diff --git a/docs/client-setup.md b/docs/client-setup.md index 91e74e2..4ad2c3e 100644 --- a/docs/client-setup.md +++ b/docs/client-setup.md @@ -1,18 +1,18 @@ # Client Setup -**These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). +These commands use the current npm release because the package name is unpinned. The current published version is 2.2.0; 2.4.0 is not yet published. -Codebase Context runs as a local MCP server: a tool your AI editor or command-line client can call while it works. Register it without a fixed folder, then let the agent supply its current repository through the `project` tool parameter. These recipes target the 2.4.0 release. +Codebase Context runs as a local MCP server: a tool your AI editor or command-line client can call while it works. Register it without a fixed folder, then let the agent supply its current repository through the `project` tool parameter. Use Node.js 22 or newer, as required by the package. The copyable registration commands target PowerShell or a POSIX terminal. Windows Command Prompt was not tested; OpenCode's quoted separator is shell-dependent. -## Client recipes for 2.4.0 +## Client recipes Claude Code, Codex CLI and OpenCode registration commands were checked again on October 6 without a project argument. The other clients retain their older config-shape evidence; no-folder native agent use is unverified for them. See the dated notes below. -Each client section says how far its earlier Windows check went. Those checks used published `2.2.0` unless another build is named; they do not verify the 2.4.0 commands: +Each client section says how far its earlier Windows check went. Those checks used published `2.2.0` unless another build is named; they do not verify upcoming unpublished changes: | Status | What it means | | ------------------------------ | ------------------------------------------------------------------------------------------ | @@ -27,7 +27,7 @@ After registration, start a new agent session in your project and ask: > Use Codebase Context to find [feature] in this repository. Pass this repository's absolute path as project when checking get_indexing_status and searching. Wait for indexing if needed, read codebase://context, then search_codebase and open a returned source file. Show me the relevant files. -The first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. You can prepare it separately with `npx -y codebase-context@2.4.0 reindex` from the project folder. Index readiness, client connection and useful agent behavior are separate checks. +The first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. You can prepare it separately with `npx -y codebase-context reindex` from the project folder. Index readiness, client connection and useful agent behavior are separate checks. ## Claude Code @@ -40,13 +40,13 @@ The first run downloads a local model and indexes the repository; time depends o User setup: ```bash -claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.4.0 +claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context ``` Optional shared project setup: ```bash -claude mcp add --scope project --transport stdio codebase-context -- npx -y codebase-context@2.4.0 +claude mcp add --scope project --transport stdio codebase-context -- npx -y codebase-context ``` `--scope project` writes shared `.mcp.json` and requires project-server approval. `--scope local` instead stores a private association for the current project in `~/.claude.json`; it does not write `.mcp.json`. @@ -66,7 +66,7 @@ claude mcp add --scope project --transport stdio codebase-context -- npx -y code **Scope:** The command writes user config. A trusted project can use a project `.codex/config.toml` instead. ```bash -codex mcp add codebase-context -- npx -y codebase-context@2.4.0 +codex mcp add codebase-context -- npx -y codebase-context ``` Equivalent project config: @@ -74,7 +74,7 @@ Equivalent project config: ```toml [mcp_servers.codebase-context] command = "npx" -args = ["-y", "codebase-context@2.4.0"] +args = ["-y", "codebase-context"] startup_timeout_sec = 120 ``` @@ -84,14 +84,14 @@ startup_timeout_sec = 120 **Official docs:** [Codex MCP configuration](https://learn.chatgpt.com/docs/extend/mcp?surface=cli). The supported `startup_timeout_sec` option defaults to ten seconds; the project example allows 120 seconds for initial `npx` package preparation. This does not prove Desktop startup or guarantee completion of a cold download. -## Codex Desktop: 2.4.0 recipe +## Codex Desktop recipe Create or merge `.codex/config.toml` in the project you want to search. Preserve any existing settings: ```toml [mcp_servers.codebase-context] command = "npx" -args = ["-y", "codebase-context@2.4.0"] +args = ["-y", "codebase-context"] startup_timeout_sec = 120 ``` @@ -103,33 +103,35 @@ On October 6 the exact published `2.2.0` `npx` entrypoint was tested without a f ## Interactive setup wizard +> **Requires 2.4.0 (not yet published)**: The non-interactive `--client` and `--yes` flags are not available in published 2.2.0 (which runs an interactive HTTP wizard). These commands require the unpublished 2.4.0 release and will not work with the current npm release. + Run setup from the project you want the client to search: ```bash -# Claude Code +# Claude Code (requires 2.4.0, not yet published) npx -y codebase-context@2.4.0 init --client claude-code --yes -# Codex +# Codex (requires 2.4.0, not yet published) npx -y codebase-context@2.4.0 init --client codex --yes ``` Claude Code writes `.mcp.json`; Codex writes `.codex/config.toml`. Both configs start the installed server with the selected project. Setup uses the current folder unless `--root` or `CODEBASE_ROOT` selects another one; `--root` takes priority. -`--yes` writes the config without prompts. Run `init` without flags to choose a client interactively and optionally add project instructions. Cursor and OpenCode still use HTTP and need a separately running server. +`--yes` writes the config without prompts. In 2.2.0, running `init` without flags starts the older interactive wizard. Cursor and OpenCode still use HTTP and need a separately running server. Existing settings are kept. If a different Codebase Context entry is already present, setup leaves the file unchanged and reports the conflict. Restart the client after setup and approve the server if asked. The first use builds the code index. Rerun setup if the Codebase Context installation moves, because the config uses its installed location. ## Optional fixed-folder setup and CLI preparation -To set a default project before the first tool call, append its full path to the server arguments, for example `args = ["-y", "codebase-context@2.4.0", "C:/projects/my-app"]`. Use a quoted literal shell argument in registration commands. This makes the default fixed across repositories when the registration is user-scoped; explicit tool selection can still choose another allowed project. +To set a default project before the first tool call, append its full path to the server arguments, for example `args = ["-y", "codebase-context", "C:/projects/my-app"]`. Use a quoted literal shell argument in registration commands. This makes the default fixed across repositories when the registration is user-scoped; explicit tool selection can still choose another allowed project. To prepare the index separately, run from the project folder: ```powershell -npx -y codebase-context@2.4.0 reindex +npx -y codebase-context reindex ``` -The interactive wizard limitations described above are still present in the 2.4.0 source. A successful wizard exit alone does not establish working client registration. +The interactive wizard limitations described above are still present in the current source. A successful wizard exit alone does not establish working client registration. ## Gemini CLI @@ -142,13 +144,13 @@ The interactive wizard limitations described above are still present in the 2.4. User setup: ```bash -gemini mcp add --scope user codebase-context npx -y codebase-context@2.4.0 +gemini mcp add --scope user codebase-context npx -y codebase-context ``` Project setup: ```bash -gemini mcp add --scope project codebase-context npx -y codebase-context@2.4.0 +gemini mcp add --scope project codebase-context npx -y codebase-context ``` Gemini writes an `mcpServers` entry in its settings. Claude, Codex, and Copilot CLI use `--`; Gemini does not. @@ -174,7 +176,7 @@ Create `.cursor/mcp.json`: "mcpServers": { "codebase-context": { "command": "npx", - "args": ["-y", "codebase-context@2.4.0"] + "args": ["-y", "codebase-context"] } } } @@ -199,7 +201,7 @@ Checked config shape: `.cursor/mcp.json with mcpServers.codebase-context command Official user command: ```powershell -code --add-mcp '{"name":"codebase-context","command":"npx","args":["-y","codebase-context@2.4.0"]}' +code --add-mcp '{"name":"codebase-context","command":"npx","args":["-y","codebase-context"]}' ``` Deterministic workspace config at `.vscode/mcp.json`: @@ -209,7 +211,7 @@ Deterministic workspace config at `.vscode/mcp.json`: "servers": { "codebase-context": { "command": "npx", - "args": ["-y", "codebase-context@2.4.0"] + "args": ["-y", "codebase-context"] } } } @@ -230,7 +232,7 @@ Deterministic workspace config at `.vscode/mcp.json`: **Scope:** User config. ```bash -copilot mcp add codebase-context -- npx -y codebase-context@2.4.0 +copilot mcp add codebase-context -- npx -y codebase-context ``` **What was checked:** The command ran with an isolated `--config-dir` and wrote the expected `mcp-config.json` command, arguments, and tool filter. @@ -250,7 +252,7 @@ copilot mcp add codebase-context -- npx -y codebase-context@2.4.0 Run once: ```bash -opencode mcp add codebase-context '--' npx -y codebase-context@2.4.0 +opencode mcp add codebase-context '--' npx -y codebase-context ``` Keep the quoted `'--'`: the installed Windows PowerShell npm shim consumes an unquoted separator. The quoted form also works in POSIX shells. @@ -263,7 +265,7 @@ For a project-local alternative, merge this into `opencode.json`: "mcp": { "codebase-context": { "type": "local", - "command": ["npx", "-y", "codebase-context@2.4.0"], + "command": ["npx", "-y", "codebase-context"], "enabled": true, "timeout": 120000 } @@ -296,7 +298,7 @@ Create `~/.codeium/windsurf/mcp_config.json`: "mcpServers": { "codebase-context": { "command": "npx", - "args": ["-y", "codebase-context@2.4.0"] + "args": ["-y", "codebase-context"] } } } @@ -315,11 +317,11 @@ Checked config shape: `~/.codeium/windsurf/mcp_config.json with mcpServers.codeb Use `stdio` for the simplest setup. HTTP can let several clients share one long-running local process, but support depends on the client and this path is not part of the default setup proof: ```bash -npx -y codebase-context@2.4.0 --http -npx -y codebase-context@2.4.0 --http --port 4000 +npx -y codebase-context --http +npx -y codebase-context --http --port 4000 ``` -The documented default endpoint is `http://127.0.0.1:3100/mcp`. Config-shape templates are available in [`templates/mcp/stdio/.mcp.json`](../templates/mcp/stdio/.mcp.json) and [`templates/mcp/http/.mcp.json`](../templates/mcp/http/.mcp.json); pin the published package as in the recipes above. HTTP does not discover a client's project folder; the agent still has to select its repository. +The documented default endpoint is `http://127.0.0.1:3100/mcp`. Config-shape templates are available in [`templates/mcp/stdio/.mcp.json`](../templates/mcp/stdio/.mcp.json) and [`templates/mcp/http/.mcp.json`](../templates/mcp/http/.mcp.json); the recipes use the current npm release. HTTP does not discover a client's project folder; the agent still has to select its repository. The [MCP July 28, 2026 HTTP revision](https://modelcontextprotocol.io/specification/2026-07-28/basic/transports/streamable-http) removed protocol sessions and the GET stream endpoint. This source checkout uses SDK 1.27.1 and the earlier protocol/session model. The stdio checks do not validate its concurrent HTTP project isolation or compatibility with the new revision. A protocol migration and host testing are separate from these setup recipes. @@ -335,7 +337,7 @@ Some clients also announce workspace roots. One valid root can auto-select; seve [MCP deprecated Roots on July 28, 2026](https://modelcontextprotocol.io/specification/2026-07-28/client/roots), recommending explicit directories/files in tool parameters, resource URIs or server configuration. CBC's older Roots support remains a compatibility aid rather than a required setup step. -October 6 verification: 18 focused current-source routing tests passed, including ambiguous roots, explicit selection, subsequent calls and root changes. The exact published `2.2.0` server also switched between two isolated prepared repositories A to B to A in one no-roots stdio process; search paths, context map and subsequent status calls followed each selection, and the runner host opened the expected distinct sources. This historical package check does not verify `2.4.0` or establish concurrent HTTP isolation. A native agent session across real repositories remains a validation step; stdio process-local selection does not establish concurrent HTTP isolation. +October 6 verification: 18 focused current-source routing tests passed, including ambiguous roots, explicit selection, subsequent calls and root changes. The exact published `2.2.0` server also switched between two isolated prepared repositories A to B to A in one no-roots stdio process; search paths, context map and subsequent status calls followed each selection, and the runner host opened the expected distinct sources. This historical package check does not verify newer source changes or establish concurrent HTTP isolation. A native agent session across real repositories remains a validation step; stdio process-local selection does not establish concurrent HTTP isolation. A follow-up recorded raw `search_codebase` calls containing only `query` and `mode`, with no `project`: they returned A, B and A after those explicit selections. The caller does not automatically add the full path; CBC routes omitted-project calls using its selected project. This state lasts for that stdio process. Select again after a server restart or when changing repositories. diff --git a/docs/demo.md b/docs/demo.md index 81d0021..407fd82 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -2,16 +2,16 @@ This walkthrough shows real CLI output captured from the open-source `angular-spotify` repository during a local proof run. That sample checkout is not bundled here. -**These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). +These commands use the current npm release. The output shown below is historical and does not establish behavior of that release. -To try the same CLI flow, use Node.js 22 or newer and open a terminal in the repository you want to inspect. These commands target `codebase-context@2.4.0`; the CLI uses the current directory as the project root. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). +To try the same CLI flow, use Node.js 22 or newer and open a terminal in the repository you want to inspect. These commands target `codebase-context`; the CLI uses the current directory as the project root. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). -The saved output below is from an earlier CLI run, not from `2.4.0` or the commands shown here. The machine-specific repository root is omitted from the returned file path. This capture does not verify the current package or an MCP connection. +The saved output below is from an earlier CLI run, not from the current commands. The machine-specific repository root is omitted from the returned file path. This capture does not verify the current package or an MCP connection. ## 0. Build The Local Index ```bash -npx -y codebase-context@2.4.0 reindex +npx -y codebase-context reindex ``` Run this before the first map or search in a repository. Later runs can use `reindex --incremental` after the code changes. @@ -19,7 +19,7 @@ Run this before the first map or search in a repository. Later runs can use `rei ## 1. Start With The Conventions Map ```bash -npx -y codebase-context@2.4.0 map --json +npx -y codebase-context map --json ``` Captured output excerpt: @@ -52,7 +52,7 @@ What this shows: ## 2. Search With Edit Intent ```bash -npx -y codebase-context@2.4.0 search --query "auth headers" --intent edit --limit 3 --json +npx -y codebase-context search --query "auth headers" --intent edit --limit 3 --json ``` Captured output excerpt: @@ -93,7 +93,7 @@ What this shows: ## 3. Check A Team Pattern Directly ```bash -npx -y codebase-context@2.4.0 patterns --category state --json +npx -y codebase-context patterns --category state --json ``` Captured output excerpt: diff --git a/docs/review-context.md b/docs/review-context.md index f21b3d6..964a9b4 100644 --- a/docs/review-context.md +++ b/docs/review-context.md @@ -4,7 +4,7 @@ It is deliberately **not** an AI reviewer. It does not call an LLM, decide whether code is correct, or post comments. Its job is narrower and testable: compile the changed surface, stable diff signals, related codebase context, and current conventions into a versioned input for a reviewer or evaluation harness. -**These commands use 2.4.0.** If that version is not available on npm yet, use the published 2.2.0 [setup guide](https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md). +The packaged review CLI requires the unpublished 2.4.0 release (the `codebase-context-review` executable does not exist in published 2.2.0); until then, use the source-build command shown below. ## Why this exists @@ -25,13 +25,13 @@ pnpm build node dist/review-bin.js --base origin/main --head HEAD ``` -After package publication, the package also exposes: +Requires 2.4.0 (not yet published): after package publication, the package also exposes: ```bash npm exec --yes --package=codebase-context@2.4.0 -- codebase-context-review --base origin/main --head HEAD ``` -Use `--json` for the complete machine-readable packet: +Use `--json` for the complete machine-readable packet (requires 2.4.0, not yet published): ```bash npm exec --yes --package=codebase-context@2.4.0 -- codebase-context-review \ diff --git a/tests/release-truth-surfaces.test.ts b/tests/release-truth-surfaces.test.ts index b2186a2..593c096 100644 --- a/tests/release-truth-surfaces.test.ts +++ b/tests/release-truth-surfaces.test.ts @@ -126,47 +126,49 @@ describe('release truth surfaces', () => { ); }); - it('keeps 2.4.0 setup guidance accurate before and after publication', () => { - const versionNotice = 'These commands use 2.4.0.'; - const fallbackNotice = 'If that version is not available on npm yet, use the published 2.2.0'; - const currentVersionGuide = - 'https://github.com/PatrickSys/codebase-context/blob/10ba0a635ff73cd5c5b011ffd5b31e7ed4656c65/docs/client-setup.md'; - const setupPages = [ - [ - 'README.md', - 'claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.4.0' - ], - [ - 'docs/client-setup.md', - 'claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context@2.4.0' - ], - ['docs/capabilities.md', '| **stdio** (default) |'], - ['docs/cli.md', 'CODEBASE_ROOT=/path/to/repo npx -y codebase-context@2.4.0 status'], - ['docs/demo.md', 'npx -y codebase-context@2.4.0 reindex'], - ['docs/review-context.md', 'pnpm build'] - ] as const; - - for (const [path, firstCommand] of setupPages) { - const page = readText(path); - const noticeIndex = page.indexOf(versionNotice); - const guideIndex = page.indexOf(currentVersionGuide); - const commandIndex = page.indexOf(firstCommand); - - expect( - noticeIndex, - `${path} should explain the current-version fallback` - ).toBeGreaterThanOrEqual(0); - expect(guideIndex, `${path} should link the published 2.2.0 setup`).toBeGreaterThan( - noticeIndex - ); - expect(page.slice(noticeIndex, guideIndex)).toContain(fallbackNotice); - expect(page.slice(noticeIndex, guideIndex)).toContain('[setup guide]('); - expect(page).not.toContain('**2.4.0 setup preview:**'); - expect(commandIndex, `${path} should retain its first runnable command`).toBeGreaterThan(0); - expect(noticeIndex, `${path} should qualify commands before they appear`).toBeLessThan( - commandIndex - ); - } + it('keeps pinned install commands on the published npm version', () => { + const publishedVersion = '2.2.0'; + const userFacingDocs = [ + 'README.md', + 'docs/capabilities.md', + 'docs/cli.md', + 'docs/client-setup.md', + 'docs/demo.md', + 'docs/review-context.md', + 'docs/templates.md' + ]; + const pinnedCommands = userFacingDocs.flatMap((path) => { + const page = readOptionalText(path); + if (!page) return []; + return [...page.matchAll(/codebase-context@(\d+\.\d+\.\d+)/g)] + .filter((match) => { + const version = match[1]; + if (version === publishedVersion) { + return false; + } + if ( + version === '2.4.0' && + (path === 'docs/review-context.md' || path === 'docs/client-setup.md') + ) { + const index = match.index ?? 0; + const surroundingText = page.slice( + Math.max(0, index - 300), + Math.min(page.length, index + 300) + ); + if (/requires\s+(?:the\s+unpublished\s+)?2\.4\.0/i.test(surroundingText)) { + return false; + } + } + return true; + }) + .map((match) => `${path}: ${match[0]}`); + }); + + expect(pinnedCommands).toEqual([]); + expect(readText('README.md')).toContain('npx -y codebase-context'); + expect(readText('docs/client-setup.md')).toContain('npx -y codebase-context'); + expect(readText('docs/client-setup.md')).toContain('codebase-context@2.4.0 init'); + expect(readText('docs/review-context.md')).toContain('--package=codebase-context@2.4.0'); }); it('does not describe the unpublished 2.4.0 preview as current or published', () => { From d7ea9bae75a61999e607f3122507b9d84749bd00 Mon Sep 17 00:00:00 2001 From: PatrickSys Date: Thu, 29 Oct 2026 20:41:35 +0100 Subject: [PATCH 100/100] fix(test): bound Windows fixture concurrency Default Windows test files to one worker through the existing Vitest configuration. Explicit worker flags still override the default. The full Windows suite passes with its existing assertions and timeouts. Agent-Harness: codex Agent-Session: J62-cbc-night-20261029-28964 --- vitest.config.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vitest.config.mjs b/vitest.config.mjs index 6b6958c..fd318e9 100644 --- a/vitest.config.mjs +++ b/vitest.config.mjs @@ -2,6 +2,8 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { + // File-heavy subprocess fixtures compete for native resources on Windows. + ...(process.platform === 'win32' ? { maxWorkers: 1 } : {}), include: ['tests/**/*.test.ts'], environment: 'node', testTimeout: 15000