diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index b6d1e147..6f9e0c0b 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -955,6 +955,15 @@ these models implicitly cache prefixes (no separate cache-write fee), so their ` `0.0` rather than mirroring input like most rows above them. Rates copied from the UiPath-only sibling plugin's own `pricing.py`, whose own module docstring documents the same rationale. +**`delegate_rate_key` maps the backend's id to its key (`gpt_5_6_luna` → `gpt-5-6-luna`).** The adapter +prices each turn under it, and the `check_pricing_coverage` pre-flight looks a Delegate agent's model +up under it. The out-of-tree `delegate-sdk` adapter did this `replace("_", "-")` inline, and the port to +the built-in agent lost it. Every config written in the backend's own ids (`model: gpt_5_6_luna`) then +priced the agent's tokens at `None`, so a downstream suite's run totals held only the judges' cost, and +the pre-flight warned that the model had no rate. The evalboard mirrors the rule as `rateKey` in +`evalboard/lib/pricing.ts`. A board row does not record its harness, so `resolvePricing` tries that key +only after the exact id, and an id that already has a rate keeps it. + Two things in `_DELEGATE_PRICING` look like copy errors on a skim and are not — a code review flagged both before this line existed to point at. diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index 3be51a5a..1c3c88e3 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -125,10 +125,10 @@ spent and the only way back is to run it again. So the framework warns loudly an keeps going. The warning fires up front. `check_pricing_coverage` walks every model the run pins -(subject agents and judge criteria) before the first task dispatches, and logs the -ones the card cannot price — early enough to fix the card and restart while it is -still cheap. After that the run is on its own: totals become floors, and -`tasks_cost_incomplete` says how many rows are behind that floor. +(subject agents, judge criteria and the simulated user) before the first task +dispatches, and logs the ones the card cannot price — early enough to fix the card +and restart while it is still cheap. After that the run is on its own: totals become +floors, and `tasks_cost_incomplete` says how many rows are behind that floor. Consumers should treat any cost field as a lower bound whenever `cost_complete` is false, and must not read `None` as `0.0` — "nothing could be priced" and "it was diff --git a/evalboard/lib/__tests__/pricing.test.ts b/evalboard/lib/__tests__/pricing.test.ts index 66236b06..ee40210f 100644 --- a/evalboard/lib/__tests__/pricing.test.ts +++ b/evalboard/lib/__tests__/pricing.test.ts @@ -54,6 +54,13 @@ describe("resolvePricing", () => { expect(resolvePricing("bedrock/converse/deepseek.v3.2")?.inputPerMTok).toBe(0.74); expect(resolvePricing("eu.anthropic.claude-sonnet-4-6")?.outputPerMTok).toBe(15); }); + + test("prices the underscored id the Delegate backend echoes under its hyphenated key", () => { + // Mirror of pricing.py::delegate_rate_key; without it every token bucket + // on a Delegate row rendered "—". + expect(resolvePricing("gpt_5_6_luna")).toEqual(resolvePricing("gpt-5-6-luna")); + expect(tokenBucketUsd("gpt_5_6_luna", 1_000_000, "output")).toBe(1.2); + }); }); describe("tokenBucketUsd", () => { diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts index 5629f6f5..abfd2589 100644 --- a/evalboard/lib/__tests__/runs.test.ts +++ b/evalboard/lib/__tests__/runs.test.ts @@ -730,6 +730,15 @@ describe("tallyModels", () => { }); }); + test("the Delegate backend's underscored id is the same model as its hyphenated form", () => { + const out = tallyModels([ + row("gpt_5_6_luna"), + row("gpt_5_6_luna"), + row("gpt-5-6-luna"), + ]); + expect(out).toEqual({ dominant: "gpt_5_6_luna", distinct: 1 }); + }); + test("genuinely different models still count separately", () => { // The normalization must not collapse an A/B run's real spread. const out = tallyModels([ diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index 6099dad6..573940d2 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -49,9 +49,7 @@ const _ROUTING_PREFIXES = [ "openrouter/", ]; const _REGION_PREFIXES = ["eu.", "us.", "apac.", "global."]; -// Exported so the run header's model tally groups on the same key pricing looks -// up on (lib/runs.ts::tallyModels). -export function normalizeModel(model: string): string { +function normalizeModel(model: string): string { let m = model.trim(); for (const pre of _ROUTING_PREFIXES) { if (m.startsWith(pre)) { @@ -69,6 +67,15 @@ export function normalizeModel(model: string): string { return m; } +// The Delegate backend echoes underscored ids ("gpt_5_6_luna") while the rate +// table keys the hyphenated form — mirror of pricing.py::delegate_rate_key. A row +// does not say which harness wrote it, so resolvePricing tries this key only +// after the exact id. Exported so the run header's model tally groups on it +// (lib/runs.ts::tallyModels). +export function rateKey(model: string): string { + return normalizeModel(model).replaceAll("_", "-"); +} + // Resolve pricing for a model id, tolerating routing/region prefixes and undated // aliases (the recorded model is usually the canonical id like "claude-sonnet-4-6", // but LiteLLM/Bedrock runs record it prefixed, and some carry a trailing date). @@ -85,6 +92,8 @@ export function resolvePricing(model: string | null): Pricing | null { // Try stripping a trailing -YYYYMMDD date. const undated = norm.replace(/-\d{8}$/, ""); if (Object.hasOwn(PRICING, undated)) return PRICING[undated]; + const hyphenated = rateKey(model); + if (Object.hasOwn(PRICING, hyphenated)) return PRICING[hyphenated]; return null; } diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index ccd27bdb..3c80ec0a 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -17,7 +17,7 @@ import { import { DEFAULT_VARIANT_ID, isValidVariantId } from "./variants"; import { DEFAULT_SOURCE, runsDirFor, type Source } from "./sources"; import { DELIVERABLE_KINDS, DELIVERABLE_NAMES } from "./artifact-kinds"; -import { messageCostUsd, normalizeModel } from "./pricing"; +import { messageCostUsd, rateKey } from "./pricing"; // Resolution order: // 1. EVALBOARD_LOCAL_RUNS_DIR — local mode, points at a coder_eval runs dir @@ -1001,11 +1001,12 @@ export async function readRunSummary( // but also reports the spread so the run header can say when there was more than // one instead of silently picking a winner. // -// The vote groups on the NORMALIZED id (same key pricing resolves on) because -// the recorded string varies by code path: a row that errored before the model +// The vote groups on the rate key (same key pricing resolves on) because the +// recorded string varies by code path: a row that errored before the model // resolved keeps the qualified id it was configured with -// ("eu.anthropic.claude-sonnet-5") while completed rows record the bare one, so -// counting raw strings let one errored row read as a second model. +// ("eu.anthropic.claude-sonnet-5") while completed rows record the bare one, and +// a Delegate row records the backend's "gpt_5_6_luna" for a "gpt-5-6-luna" +// config, so counting raw strings let one row read as a second model. // // Display stays the most common RAW string inside the winning group, so the chip // shows what the run recorded rather than a derived value. @@ -1017,7 +1018,7 @@ export function tallyModels(rows: RawTaskResult[]): { for (const r of rows) { const m = r.model_used; if (typeof m !== "string" || !m) continue; - const key = normalizeModel(m); + const key = rateKey(m); let raw = groups.get(key); if (!raw) { raw = new Map(); diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index 1879ae26..21440963 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -59,7 +59,7 @@ TokenUsage, TurnRecord, ) -from coder_eval.pricing import calculate_cost +from coder_eval.pricing import calculate_cost, delegate_rate_key from coder_eval.streaming.callbacks import StreamCallback, safe_emit from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( @@ -1109,7 +1109,7 @@ def _finalize_turn( usage = state.usage or TokenUsage() if not usage.is_empty(): cost = calculate_cost( - state.model_used or self.config.model or "", + delegate_rate_key(state.model_used or self.config.model or ""), uncached_input_tokens=usage.uncached_input_tokens, output_tokens=usage.output_tokens, cache_creation_tokens=usage.cache_creation_input_tokens, diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index 2087d036..ee958d6a 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -35,7 +35,7 @@ format_task_log_id, write_text_atomic, ) -from ..pricing import unpriced_models +from ..pricing import delegate_rate_key, unpriced_models from ..run_record import eval_result_to_task_dict from ..streaming.callbacks import StreamCallback from ..utils import get_version_info, looks_like_version @@ -46,16 +46,23 @@ def _run_models(resolved_tasks: list[ResolvedTask]) -> Iterator[str | None]: - """Every model id this run pins up front: subject agents and judge criteria. + """Every model id this run pins up front: subject agents, judge criteria and the simulated user. - A criterion's model is always priced from the rate card (no judge backend - reports a cost), so it matters more here than the agent's, which only falls - back to the card on a turn the backend never priced. + A criterion's model and the simulator's are always priced from the rate card + (no judge backend reports a cost, and ``simulator_cost_usd`` prices its tokens), + so they matter more here than the agent's, which only falls back to the card on + a turn the backend never priced. """ for rt in resolved_tasks: - yield rt.task.agent.model if rt.task.agent else None + agent = rt.task.agent + if agent is not None and agent.model and agent.type == AgentKind.DELEGATE: + yield delegate_rate_key(agent.model) + else: + yield agent.model if agent else None for criterion in rt.task.success_criteria: yield getattr(criterion, "model", None) + if rt.task.simulation is not None and rt.task.simulation.enabled: + yield rt.task.simulation.model def check_pricing_coverage(resolved_tasks: list[ResolvedTask]) -> list[str]: @@ -78,7 +85,7 @@ def check_pricing_coverage(resolved_tasks: list[ResolvedTask]) -> list[str]: return [] logger.warning( "No pricing rate for %s. Turns the agent's own backend prices are unaffected, but any " - + "judge call, timed-out turn or killed partial will book its tokens with no cost, so " + + "judge call, simulator turn, timed-out turn or killed partial will book its tokens with no cost, so " + "run-level totals will understate the bill (RunSummary.cost_complete reports false). " + "Add the rate to coder_eval.pricing or register it from a plugin via register_pricing().", ", ".join(repr(m) for m in missing), diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py index 0b9b1c3e..c04c110a 100644 --- a/src/coder_eval/pricing.py +++ b/src/coder_eval/pricing.py @@ -59,6 +59,11 @@ class ModelPricing: rather than a second hand-copied literal that could drift from ``_DELEGATE_PRICING``.""" +def delegate_rate_key(model: str) -> str: + """The rate-card key for a Delegate model id: the backend's ``gpt_5_6_luna`` prices as ``gpt-5-6-luna``.""" + return model.replace("_", "-") + + _PRICING: dict[str, ModelPricing] = { # Fable 5.1 (and Mythos 5.1) price cache hits at 0.025x input, not the 0.1x # every other Claude model uses. Fable 5 pays $1 on the identical $10 base. diff --git a/src/coder_eval/run_record.py b/src/coder_eval/run_record.py index 615efc07..09bcd8dd 100644 --- a/src/coder_eval/run_record.py +++ b/src/coder_eval/run_record.py @@ -16,7 +16,14 @@ from typing import Any from coder_eval.errors import truncate_crash_message -from coder_eval.models import EvaluationResult, FinalStatus, judge_cost_usd, simulator_cost_usd, sum_costs +from coder_eval.models import ( + EvaluationResult, + FinalStatus, + JudgeCriterionResult, + judge_cost_usd, + simulator_cost_usd, + sum_costs, +) from coder_eval.result_metrics import agent_wall_ms, expected_turns_overage, turn_time_buckets, visible_turn_count from coder_eval.result_metrics import has_final_reply as _has_final_reply @@ -28,28 +35,36 @@ def _cost_complete(result: EvaluationResult) -> bool: - """Whether this row's recorded agent spend accounts for everything it spent. + """Whether this row's recorded agent, judge and simulator spend accounts for everything it spent. - False means the costs on the row are a floor, not the bill. Two ways in: + False means the costs on the row are a floor, not the bill. Three ways in: - 1. A turn burned tokens the rate card could not price. The card is the fallback - for anything the backend did not price itself, so with no rate those tokens - book no money. - 2. The task was hard-killed by the task-level timeout. Keyed on the status - rather than on emptiness: the watchdog fires while the evaluation loop is - running, so a TIMEOUT row always lost an in-flight turn, even one that - completed earlier turns that do carry costs. + 1. An agent turn burned tokens the rate card could not price. The card is the + fallback for anything the backend did not price itself, so those tokens book + no money. + 2. A judge or the simulated user burned tokens with no cost. Both price from the + rate card, so an unpriced model drops that slice of the bill. + 3. The task was hard-killed by the task-level timeout. Keyed on the status, not + on emptiness: the watchdog fires while the loop runs, so a TIMEOUT row always + lost an in-flight turn, even after earlier turns that carry costs. - True for a row that burned nothing: an error before the agent ran genuinely - cost zero, and a slow setup failure is as free as a fast one. + True for a row that burned nothing: an error before the agent ran cost zero. """ if result.final_status is FinalStatus.TIMEOUT: return False - return all( - usage.total_cost_usd is not None - for t in result.iterations - if (usage := t.token_usage) is not None and not usage.is_empty() - ) + sim = result.simulation + if ( + sim is not None + and (sim.simulator_input_tokens or sim.simulator_output_tokens) + and simulator_cost_usd(result) is None + ): + return False + usages = [t.token_usage for t in result.iterations] + [ + cr.token_usage + for cr in result.success_criteria_results + result.post_failure_criteria_results + if isinstance(cr, JudgeCriterionResult) + ] + return all(usage.total_cost_usd is not None for usage in usages if usage is not None and not usage.is_empty()) def eval_result_to_task_dict( diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py index a5d1c7c4..53c8ea93 100644 --- a/tests/test_cost_accounting_paths.py +++ b/tests/test_cost_accounting_paths.py @@ -16,10 +16,12 @@ from coder_eval.models import ( ClaudeCodeAgentConfig, + DelegateAgentConfig, EvaluationResult, FinalStatus, JudgeCriterionResult, ResolvedTask, + SimulationConfig, SimulationTelemetry, TaskDefinition, TokenUsage, @@ -89,6 +91,13 @@ def test_unpinned_model_is_not_flagged(self, tmp_path): """A task deferring its model to the route can't be pre-flighted from here.""" assert check_pricing_coverage([_resolved(None, tmp_path)]) == [] + def test_delegate_backend_id_is_priced_under_its_rate_card_key(self, tmp_path, caplog): + resolved = _resolved(None, tmp_path) + resolved.task.agent = DelegateAgentConfig(type="delegate", model="gpt_5_6_luna") + with caplog.at_level(logging.WARNING): + assert check_pricing_coverage([resolved]) == [] + assert "No pricing rate" not in caplog.text + def test_judge_criterion_model_is_pre_flighted(self, tmp_path): """A judge call is ALWAYS priced from the rate card, so its model matters most. @@ -103,6 +112,19 @@ def test_default_judge_model_is_priced(self, tmp_path): judge = {"type": "llm_judge", "description": "d", "prompt": "grade it"} assert check_pricing_coverage([_resolved("claude-sonnet-5", tmp_path, [judge])]) == [] + def test_simulator_model_is_pre_flighted(self, tmp_path): + """``simulator_cost_usd`` always prices from the rate card, so an unpriced simulator books no cost.""" + resolved = _resolved("claude-sonnet-5", tmp_path) + resolved.task.simulation = SimulationConfig( + enabled=True, model="us.anthropic.claude-sonnet-99", persona="p", goal="g" + ) + assert check_pricing_coverage([resolved]) == ["us.anthropic.claude-sonnet-99"] + + def test_disabled_simulation_is_not_pre_flighted(self, tmp_path): + resolved = _resolved("claude-sonnet-5", tmp_path) + resolved.task.simulation = SimulationConfig(model="us.anthropic.claude-sonnet-99", persona="p", goal="g") + assert check_pricing_coverage([resolved]) == [] + class TestRowCostProjection: def test_cost_complete_false_when_a_turn_is_unpriced(self): @@ -239,6 +261,7 @@ def test_unpriced_judge_is_skipped_not_fatal(self): row = eval_result_to_task_dict(result) assert row["judge_cost_usd"] == pytest.approx(0.02) assert row["total_cost_usd"] == pytest.approx(0.12) + assert row["cost_complete"] is False @staticmethod def _simulated(**env) -> EvaluationResult: @@ -284,6 +307,11 @@ def test_unpriced_simulator_route_is_skipped_not_fatal(self): assert row["simulator_cost_usd"] is None # The agent's own spend still lands; only the simulator slice is missing. assert row["total_cost_usd"] == pytest.approx(0.1) + assert row["cost_complete"] is False + + def test_cost_complete_true_when_the_simulator_is_priced(self): + result = self._simulated(bedrock_model="claude-haiku-4-5-20251001") + assert eval_result_to_task_dict(result)["cost_complete"] is True class TestTotalCost: diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index 91011ff3..115275d2 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -1092,6 +1092,20 @@ async def test_cost_wired_into_record_for_configured_model(self, patch_exec, tmp assert record.token_usage is not None assert record.token_usage.total_cost_usd == pytest.approx(0.95 + 4.0) + async def test_cost_prices_the_underscored_id_the_backend_echoes(self, patch_exec, tmp_path): + events = [ + _result( + response="done", + model="gpt_5_6_luna", + usage={"input_tokens": 1_000_000, "output_tokens": 1_000_000}, + ) + ] + agent, _ = await _started_agent(patch_exec, events, tmp_path, model="gpt_5_6_luna") + record = await agent.communicate("hi") + assert record.model_used == "gpt_5_6_luna" + assert record.token_usage is not None + assert record.token_usage.total_cost_usd == pytest.approx(0.20 + 1.20) + async def test_send_error_delivers_end_events_to_stream_callback(self, patch_exec, tmp_path): events = [_line({"type": "error", "message": "network blip"})] agent, _ = await _started_agent(patch_exec, events, tmp_path)