From 60f16c871c52026ab3da7a2835956f55051e0247 Mon Sep 17 00:00:00 2001 From: abhiram-vad Date: Fri, 9 Oct 2026 16:07:23 -0700 Subject: [PATCH 1/5] fix(delegate): price the underscored model ids the Delegate backend echoes The Delegate backend names its models gpt_5_6_luna, claude_opus_5_5 and so on, and the rate card keys the hyphenated form (gpt-5-6-luna). The out-of-tree delegate-sdk adapter mapped "_" to "-" before it priced a turn; the port to the built-in agent dropped that mapping. A config written in the backend's own ids therefore priced every agent turn at None, so a run's cost held only its judges, and the pricing pre-flight warned that the model had no rate. delegate_rate_key does the mapping in one place. The adapter prices each turn under it, and check_pricing_coverage looks a Delegate agent's model up under it. Co-Authored-By: Claude Opus 5.5 --- .claude/notes/agents.md | 7 +++++++ src/coder_eval/agents/delegate_agent.py | 4 ++-- src/coder_eval/orchestration/batch.py | 8 ++++++-- src/coder_eval/pricing.py | 5 +++++ tests/test_cost_accounting_paths.py | 8 ++++++++ tests/test_delegate_agent.py | 14 ++++++++++++++ 6 files changed, 42 insertions(+), 4 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 9857d8393..0cea0f694 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -938,6 +938,13 @@ these models implicitly cache prefixes (no separate cache-write fee), so their ` `0.0` rather than mirroring input like most rows above them. Rates copied from the UiPath-only sibling plugin's own `pricing.py`, whose own module docstring documents the same rationale. +**`delegate_rate_key` maps the backend's id to its key (`gpt_5_6_luna` → `gpt-5-6-luna`).** The adapter +prices each turn under it, and the `check_pricing_coverage` pre-flight looks a Delegate agent's model +up under it. The out-of-tree `delegate-sdk` adapter did this `replace("_", "-")` inline, and the port to +the built-in agent lost it. Every config written in the backend's own ids (`model: gpt_5_6_luna`) then +priced the agent's tokens at `None`, so a downstream suite's run totals held only the judges' cost, and +the pre-flight warned that the model had no rate. + Two things in `_DELEGATE_PRICING` look like copy errors on a skim and are not — a code review flagged both before this line existed to point at. diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index 72d8b5445..ff92f6249 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -60,7 +60,7 @@ TokenUsage, TurnRecord, ) -from coder_eval.pricing import calculate_cost +from coder_eval.pricing import calculate_cost, delegate_rate_key from coder_eval.streaming.callbacks import StreamCallback, safe_emit from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( @@ -1066,7 +1066,7 @@ def _finalize_turn( usage = state.usage or TokenUsage() if not usage.is_empty(): cost = calculate_cost( - state.model_used or self.config.model or "", + delegate_rate_key(state.model_used or self.config.model or ""), uncached_input_tokens=usage.uncached_input_tokens, output_tokens=usage.output_tokens, cache_creation_tokens=usage.cache_creation_input_tokens, diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index 2087d0365..653246e8c 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -35,7 +35,7 @@ format_task_log_id, write_text_atomic, ) -from ..pricing import unpriced_models +from ..pricing import delegate_rate_key, unpriced_models from ..run_record import eval_result_to_task_dict from ..streaming.callbacks import StreamCallback from ..utils import get_version_info, looks_like_version @@ -53,7 +53,11 @@ def _run_models(resolved_tasks: list[ResolvedTask]) -> Iterator[str | None]: back to the card on a turn the backend never priced. """ for rt in resolved_tasks: - yield rt.task.agent.model if rt.task.agent else None + agent = rt.task.agent + if agent is not None and agent.model and agent.type == AgentKind.DELEGATE: + yield delegate_rate_key(agent.model) + else: + yield agent.model if agent else None for criterion in rt.task.success_criteria: yield getattr(criterion, "model", None) diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py index 0b9b1c3e5..c04c110a3 100644 --- a/src/coder_eval/pricing.py +++ b/src/coder_eval/pricing.py @@ -59,6 +59,11 @@ class ModelPricing: rather than a second hand-copied literal that could drift from ``_DELEGATE_PRICING``.""" +def delegate_rate_key(model: str) -> str: + """The rate-card key for a Delegate model id: the backend's ``gpt_5_6_luna`` prices as ``gpt-5-6-luna``.""" + return model.replace("_", "-") + + _PRICING: dict[str, ModelPricing] = { # Fable 5.1 (and Mythos 5.1) price cache hits at 0.025x input, not the 0.1x # every other Claude model uses. Fable 5 pays $1 on the identical $10 base. diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py index a5d1c7c49..1219c4edf 100644 --- a/tests/test_cost_accounting_paths.py +++ b/tests/test_cost_accounting_paths.py @@ -16,6 +16,7 @@ from coder_eval.models import ( ClaudeCodeAgentConfig, + DelegateAgentConfig, EvaluationResult, FinalStatus, JudgeCriterionResult, @@ -89,6 +90,13 @@ def test_unpinned_model_is_not_flagged(self, tmp_path): """A task deferring its model to the route can't be pre-flighted from here.""" assert check_pricing_coverage([_resolved(None, tmp_path)]) == [] + def test_delegate_backend_id_is_priced_under_its_rate_card_key(self, tmp_path, caplog): + resolved = _resolved(None, tmp_path) + resolved.task.agent = DelegateAgentConfig(type="delegate", model="gpt_5_6_luna") + with caplog.at_level(logging.WARNING): + assert check_pricing_coverage([resolved]) == [] + assert "No pricing rate" not in caplog.text + def test_judge_criterion_model_is_pre_flighted(self, tmp_path): """A judge call is ALWAYS priced from the rate card, so its model matters most. diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index f40591612..482621e84 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -1006,6 +1006,20 @@ async def test_cost_wired_into_record_for_configured_model(self, patch_exec, tmp assert record.token_usage is not None assert record.token_usage.total_cost_usd == pytest.approx(0.95 + 4.0) + async def test_cost_prices_the_underscored_id_the_backend_echoes(self, patch_exec, tmp_path): + events = [ + _result( + response="done", + model="gpt_5_6_luna", + usage={"input_tokens": 1_000_000, "output_tokens": 1_000_000}, + ) + ] + agent, _ = await _started_agent(patch_exec, events, tmp_path, model="gpt_5_6_luna") + record = await agent.communicate("hi") + assert record.model_used == "gpt_5_6_luna" + assert record.token_usage is not None + assert record.token_usage.total_cost_usd == pytest.approx(0.20 + 1.20) + async def test_send_error_delivers_end_events_to_stream_callback(self, patch_exec, tmp_path): events = [_line({"type": "error", "message": "network blip"})] agent, _ = await _started_agent(patch_exec, events, tmp_path) From 71e3f8b7c1f24c079f45df477b7b071d9b2dd9df Mon Sep 17 00:00:00 2001 From: abhiram-vad Date: Mon, 12 Oct 2026 12:19:49 -0700 Subject: [PATCH 2/5] fix(evalboard): price the underscored ids the Delegate backend echoes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The board has the same id gap as pricing.py had. resolvePricing and the run header's model tally keyed on normalizeModel, which keeps "gpt_5_6_luna" as it is, so every token bucket on a Delegate row showed "—" in the USD view, the thinking-cost simulator had no rate, and tallyModels counted gpt_5_6_luna and gpt-5-6-luna as two models. rateKey mirrors pricing.py::delegate_rate_key. A row does not record its harness, so resolvePricing tries it only after the exact id, and an id that already has a rate keeps it. tallyModels groups on it. Co-Authored-By: Claude Opus 5.5 --- .claude/notes/agents.md | 4 +++- evalboard/lib/__tests__/pricing.test.ts | 7 +++++++ evalboard/lib/__tests__/runs.test.ts | 9 +++++++++ evalboard/lib/pricing.ts | 15 ++++++++++++--- evalboard/lib/runs.ts | 13 +++++++------ 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 0cea0f694..55045934a 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -943,7 +943,9 @@ prices each turn under it, and the `check_pricing_coverage` pre-flight looks a D up under it. The out-of-tree `delegate-sdk` adapter did this `replace("_", "-")` inline, and the port to the built-in agent lost it. Every config written in the backend's own ids (`model: gpt_5_6_luna`) then priced the agent's tokens at `None`, so a downstream suite's run totals held only the judges' cost, and -the pre-flight warned that the model had no rate. +the pre-flight warned that the model had no rate. The evalboard mirrors the rule as `rateKey` in +`evalboard/lib/pricing.ts`. A board row does not record its harness, so `resolvePricing` tries that key +only after the exact id, and an id that already has a rate keeps it. Two things in `_DELEGATE_PRICING` look like copy errors on a skim and are not — a code review flagged both before this line existed to point at. diff --git a/evalboard/lib/__tests__/pricing.test.ts b/evalboard/lib/__tests__/pricing.test.ts index 66236b060..ee40210fc 100644 --- a/evalboard/lib/__tests__/pricing.test.ts +++ b/evalboard/lib/__tests__/pricing.test.ts @@ -54,6 +54,13 @@ describe("resolvePricing", () => { expect(resolvePricing("bedrock/converse/deepseek.v3.2")?.inputPerMTok).toBe(0.74); expect(resolvePricing("eu.anthropic.claude-sonnet-4-6")?.outputPerMTok).toBe(15); }); + + test("prices the underscored id the Delegate backend echoes under its hyphenated key", () => { + // Mirror of pricing.py::delegate_rate_key; without it every token bucket + // on a Delegate row rendered "—". + expect(resolvePricing("gpt_5_6_luna")).toEqual(resolvePricing("gpt-5-6-luna")); + expect(tokenBucketUsd("gpt_5_6_luna", 1_000_000, "output")).toBe(1.2); + }); }); describe("tokenBucketUsd", () => { diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts index 5629f6f54..abfd2589a 100644 --- a/evalboard/lib/__tests__/runs.test.ts +++ b/evalboard/lib/__tests__/runs.test.ts @@ -730,6 +730,15 @@ describe("tallyModels", () => { }); }); + test("the Delegate backend's underscored id is the same model as its hyphenated form", () => { + const out = tallyModels([ + row("gpt_5_6_luna"), + row("gpt_5_6_luna"), + row("gpt-5-6-luna"), + ]); + expect(out).toEqual({ dominant: "gpt_5_6_luna", distinct: 1 }); + }); + test("genuinely different models still count separately", () => { // The normalization must not collapse an A/B run's real spread. const out = tallyModels([ diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index 6099dad65..573940d2d 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -49,9 +49,7 @@ const _ROUTING_PREFIXES = [ "openrouter/", ]; const _REGION_PREFIXES = ["eu.", "us.", "apac.", "global."]; -// Exported so the run header's model tally groups on the same key pricing looks -// up on (lib/runs.ts::tallyModels). -export function normalizeModel(model: string): string { +function normalizeModel(model: string): string { let m = model.trim(); for (const pre of _ROUTING_PREFIXES) { if (m.startsWith(pre)) { @@ -69,6 +67,15 @@ export function normalizeModel(model: string): string { return m; } +// The Delegate backend echoes underscored ids ("gpt_5_6_luna") while the rate +// table keys the hyphenated form — mirror of pricing.py::delegate_rate_key. A row +// does not say which harness wrote it, so resolvePricing tries this key only +// after the exact id. Exported so the run header's model tally groups on it +// (lib/runs.ts::tallyModels). +export function rateKey(model: string): string { + return normalizeModel(model).replaceAll("_", "-"); +} + // Resolve pricing for a model id, tolerating routing/region prefixes and undated // aliases (the recorded model is usually the canonical id like "claude-sonnet-4-6", // but LiteLLM/Bedrock runs record it prefixed, and some carry a trailing date). @@ -85,6 +92,8 @@ export function resolvePricing(model: string | null): Pricing | null { // Try stripping a trailing -YYYYMMDD date. const undated = norm.replace(/-\d{8}$/, ""); if (Object.hasOwn(PRICING, undated)) return PRICING[undated]; + const hyphenated = rateKey(model); + if (Object.hasOwn(PRICING, hyphenated)) return PRICING[hyphenated]; return null; } diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index ccd27bdba..3c80ec0a0 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -17,7 +17,7 @@ import { import { DEFAULT_VARIANT_ID, isValidVariantId } from "./variants"; import { DEFAULT_SOURCE, runsDirFor, type Source } from "./sources"; import { DELIVERABLE_KINDS, DELIVERABLE_NAMES } from "./artifact-kinds"; -import { messageCostUsd, normalizeModel } from "./pricing"; +import { messageCostUsd, rateKey } from "./pricing"; // Resolution order: // 1. EVALBOARD_LOCAL_RUNS_DIR — local mode, points at a coder_eval runs dir @@ -1001,11 +1001,12 @@ export async function readRunSummary( // but also reports the spread so the run header can say when there was more than // one instead of silently picking a winner. // -// The vote groups on the NORMALIZED id (same key pricing resolves on) because -// the recorded string varies by code path: a row that errored before the model +// The vote groups on the rate key (same key pricing resolves on) because the +// recorded string varies by code path: a row that errored before the model // resolved keeps the qualified id it was configured with -// ("eu.anthropic.claude-sonnet-5") while completed rows record the bare one, so -// counting raw strings let one errored row read as a second model. +// ("eu.anthropic.claude-sonnet-5") while completed rows record the bare one, and +// a Delegate row records the backend's "gpt_5_6_luna" for a "gpt-5-6-luna" +// config, so counting raw strings let one row read as a second model. // // Display stays the most common RAW string inside the winning group, so the chip // shows what the run recorded rather than a derived value. @@ -1017,7 +1018,7 @@ export function tallyModels(rows: RawTaskResult[]): { for (const r of rows) { const m = r.model_used; if (typeof m !== "string" || !m) continue; - const key = normalizeModel(m); + const key = rateKey(m); let raw = groups.get(key); if (!raw) { raw = new Map(); From d6cac57867cae9b74e9a61f30e462e4f84921ca7 Mon Sep 17 00:00:00 2001 From: abhiram-vad Date: Mon, 12 Oct 2026 12:19:50 -0700 Subject: [PATCH 3/5] fix(pricing): pre-flight the simulated user's model simulator_cost_usd always prices the simulator's tokens from the rate card, but check_pricing_coverage did not look at simulation.model. A simulator model without a rate therefore booked no simulator cost and logged no warning: a downstream suite ran its simulator on claude-sonnet-5-5 before that rate existed and lost the cost silently. The pre-flight now adds the model of every enabled simulation. Co-Authored-By: Claude Opus 5.5 --- docs/REPORT_SCHEMA.md | 2 +- src/coder_eval/orchestration/batch.py | 11 +++++++---- tests/test_cost_accounting_paths.py | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index 3be51a5a3..220d0e174 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -125,7 +125,7 @@ spent and the only way back is to run it again. So the framework warns loudly an keeps going. The warning fires up front. `check_pricing_coverage` walks every model the run pins -(subject agents and judge criteria) before the first task dispatches, and logs the +(subject agents, judge criteria and the simulated user) before the first task dispatches, and logs the ones the card cannot price — early enough to fix the card and restart while it is still cheap. After that the run is on its own: totals become floors, and `tasks_cost_incomplete` says how many rows are behind that floor. diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index 653246e8c..b03be3b47 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -46,11 +46,12 @@ def _run_models(resolved_tasks: list[ResolvedTask]) -> Iterator[str | None]: - """Every model id this run pins up front: subject agents and judge criteria. + """Every model id this run pins up front: subject agents, judge criteria and the simulated user. - A criterion's model is always priced from the rate card (no judge backend - reports a cost), so it matters more here than the agent's, which only falls - back to the card on a turn the backend never priced. + A criterion's model and the simulator's are always priced from the rate card + (no judge backend reports a cost, and ``simulator_cost_usd`` prices its tokens), + so they matter more here than the agent's, which only falls back to the card on + a turn the backend never priced. """ for rt in resolved_tasks: agent = rt.task.agent @@ -60,6 +61,8 @@ def _run_models(resolved_tasks: list[ResolvedTask]) -> Iterator[str | None]: yield agent.model if agent else None for criterion in rt.task.success_criteria: yield getattr(criterion, "model", None) + if rt.task.simulation is not None and rt.task.simulation.enabled: + yield rt.task.simulation.model def check_pricing_coverage(resolved_tasks: list[ResolvedTask]) -> list[str]: diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py index 1219c4edf..e189ee9e6 100644 --- a/tests/test_cost_accounting_paths.py +++ b/tests/test_cost_accounting_paths.py @@ -21,6 +21,7 @@ FinalStatus, JudgeCriterionResult, ResolvedTask, + SimulationConfig, SimulationTelemetry, TaskDefinition, TokenUsage, @@ -111,6 +112,19 @@ def test_default_judge_model_is_priced(self, tmp_path): judge = {"type": "llm_judge", "description": "d", "prompt": "grade it"} assert check_pricing_coverage([_resolved("claude-sonnet-5", tmp_path, [judge])]) == [] + def test_simulator_model_is_pre_flighted(self, tmp_path): + """``simulator_cost_usd`` always prices from the rate card, so an unpriced simulator books no cost.""" + resolved = _resolved("claude-sonnet-5", tmp_path) + resolved.task.simulation = SimulationConfig( + enabled=True, model="us.anthropic.claude-sonnet-99", persona="p", goal="g" + ) + assert check_pricing_coverage([resolved]) == ["us.anthropic.claude-sonnet-99"] + + def test_disabled_simulation_is_not_pre_flighted(self, tmp_path): + resolved = _resolved("claude-sonnet-5", tmp_path) + resolved.task.simulation = SimulationConfig(model="us.anthropic.claude-sonnet-99", persona="p", goal="g") + assert check_pricing_coverage([resolved]) == [] + class TestRowCostProjection: def test_cost_complete_false_when_a_turn_is_unpriced(self): From 5a94948ab873d74bb2d98177e67b26d843c6236e Mon Sep 17 00:00:00 2001 From: abhiram-vad Date: Mon, 12 Oct 2026 12:40:54 -0700 Subject: [PATCH 4/5] docs: reflow the pricing pre-flight paragraph in REPORT_SCHEMA --- docs/REPORT_SCHEMA.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index 220d0e174..1c3c88e37 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -125,10 +125,10 @@ spent and the only way back is to run it again. So the framework warns loudly an keeps going. The warning fires up front. `check_pricing_coverage` walks every model the run pins -(subject agents, judge criteria and the simulated user) before the first task dispatches, and logs the -ones the card cannot price — early enough to fix the card and restart while it is -still cheap. After that the run is on its own: totals become floors, and -`tasks_cost_incomplete` says how many rows are behind that floor. +(subject agents, judge criteria and the simulated user) before the first task +dispatches, and logs the ones the card cannot price — early enough to fix the card +and restart while it is still cheap. After that the run is on its own: totals become +floors, and `tasks_cost_incomplete` says how many rows are behind that floor. Consumers should treat any cost field as a lower bound whenever `cost_complete` is false, and must not read `None` as `0.0` — "nothing could be priced" and "it was From d21f8ccff14d0bfb3136cc84f339ecf15c7fa45d Mon Sep 17 00:00:00 2001 From: abhiram-vad Date: Mon, 12 Oct 2026 13:51:39 -0700 Subject: [PATCH 5/5] fix(pricing): mark a row's cost incomplete when a judge or the simulator is unpriced _cost_complete looked only at the agent's turns, so a judge or simulator model with no rate left total_cost_usd a floor while cost_complete stayed true and tasks_cost_incomplete did not count the row. REPORT_SCHEMA and the pre-flight warning already said an unpriced model makes the run incomplete. The row now checks judge usage and simulator tokens too, and the warning names the simulator turn. Co-Authored-By: Claude Opus 5.5 --- src/coder_eval/orchestration/batch.py | 2 +- src/coder_eval/run_record.py | 49 +++++++++++++++++---------- tests/test_cost_accounting_paths.py | 6 ++++ 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index b03be3b47..ee958d6a0 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -85,7 +85,7 @@ def check_pricing_coverage(resolved_tasks: list[ResolvedTask]) -> list[str]: return [] logger.warning( "No pricing rate for %s. Turns the agent's own backend prices are unaffected, but any " - + "judge call, timed-out turn or killed partial will book its tokens with no cost, so " + + "judge call, simulator turn, timed-out turn or killed partial will book its tokens with no cost, so " + "run-level totals will understate the bill (RunSummary.cost_complete reports false). " + "Add the rate to coder_eval.pricing or register it from a plugin via register_pricing().", ", ".join(repr(m) for m in missing), diff --git a/src/coder_eval/run_record.py b/src/coder_eval/run_record.py index 615efc077..09bcd8dd6 100644 --- a/src/coder_eval/run_record.py +++ b/src/coder_eval/run_record.py @@ -16,7 +16,14 @@ from typing import Any from coder_eval.errors import truncate_crash_message -from coder_eval.models import EvaluationResult, FinalStatus, judge_cost_usd, simulator_cost_usd, sum_costs +from coder_eval.models import ( + EvaluationResult, + FinalStatus, + JudgeCriterionResult, + judge_cost_usd, + simulator_cost_usd, + sum_costs, +) from coder_eval.result_metrics import agent_wall_ms, expected_turns_overage, turn_time_buckets, visible_turn_count from coder_eval.result_metrics import has_final_reply as _has_final_reply @@ -28,28 +35,36 @@ def _cost_complete(result: EvaluationResult) -> bool: - """Whether this row's recorded agent spend accounts for everything it spent. + """Whether this row's recorded agent, judge and simulator spend accounts for everything it spent. - False means the costs on the row are a floor, not the bill. Two ways in: + False means the costs on the row are a floor, not the bill. Three ways in: - 1. A turn burned tokens the rate card could not price. The card is the fallback - for anything the backend did not price itself, so with no rate those tokens - book no money. - 2. The task was hard-killed by the task-level timeout. Keyed on the status - rather than on emptiness: the watchdog fires while the evaluation loop is - running, so a TIMEOUT row always lost an in-flight turn, even one that - completed earlier turns that do carry costs. + 1. An agent turn burned tokens the rate card could not price. The card is the + fallback for anything the backend did not price itself, so those tokens book + no money. + 2. A judge or the simulated user burned tokens with no cost. Both price from the + rate card, so an unpriced model drops that slice of the bill. + 3. The task was hard-killed by the task-level timeout. Keyed on the status, not + on emptiness: the watchdog fires while the loop runs, so a TIMEOUT row always + lost an in-flight turn, even after earlier turns that carry costs. - True for a row that burned nothing: an error before the agent ran genuinely - cost zero, and a slow setup failure is as free as a fast one. + True for a row that burned nothing: an error before the agent ran cost zero. """ if result.final_status is FinalStatus.TIMEOUT: return False - return all( - usage.total_cost_usd is not None - for t in result.iterations - if (usage := t.token_usage) is not None and not usage.is_empty() - ) + sim = result.simulation + if ( + sim is not None + and (sim.simulator_input_tokens or sim.simulator_output_tokens) + and simulator_cost_usd(result) is None + ): + return False + usages = [t.token_usage for t in result.iterations] + [ + cr.token_usage + for cr in result.success_criteria_results + result.post_failure_criteria_results + if isinstance(cr, JudgeCriterionResult) + ] + return all(usage.total_cost_usd is not None for usage in usages if usage is not None and not usage.is_empty()) def eval_result_to_task_dict( diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py index e189ee9e6..53c8ea938 100644 --- a/tests/test_cost_accounting_paths.py +++ b/tests/test_cost_accounting_paths.py @@ -261,6 +261,7 @@ def test_unpriced_judge_is_skipped_not_fatal(self): row = eval_result_to_task_dict(result) assert row["judge_cost_usd"] == pytest.approx(0.02) assert row["total_cost_usd"] == pytest.approx(0.12) + assert row["cost_complete"] is False @staticmethod def _simulated(**env) -> EvaluationResult: @@ -306,6 +307,11 @@ def test_unpriced_simulator_route_is_skipped_not_fatal(self): assert row["simulator_cost_usd"] is None # The agent's own spend still lands; only the simulator slice is missing. assert row["total_cost_usd"] == pytest.approx(0.1) + assert row["cost_complete"] is False + + def test_cost_complete_true_when_the_simulator_is_priced(self): + result = self._simulated(bedrock_model="claude-haiku-4-5-20251001") + assert eval_result_to_task_dict(result)["cost_complete"] is True class TestTotalCost: