fix(pricing): price Delegate's underscored model ids and flag unpriced judge/simulator cost

.claude/notes/agents.md

@@ -955,6 +955,15 @@ these models implicitly cache prefixes (no separate cache-write fee), so their `
 `0.0` rather than mirroring input like most rows above them. Rates copied from the UiPath-only sibling
 plugin's own `pricing.py`, whose own module docstring documents the same rationale.
 
+**`delegate_rate_key` maps the backend's id to its key (`gpt_5_6_luna` → `gpt-5-6-luna`).** The adapter
+prices each turn under it, and the `check_pricing_coverage` pre-flight looks a Delegate agent's model
+up under it. The out-of-tree `delegate-sdk` adapter did this `replace("_", "-")` inline, and the port to
+the built-in agent lost it. Every config written in the backend's own ids (`model: gpt_5_6_luna`) then
+priced the agent's tokens at `None`, so a downstream suite's run totals held only the judges' cost, and
+the pre-flight warned that the model had no rate. The evalboard mirrors the rule as `rateKey` in
+`evalboard/lib/pricing.ts`. A board row does not record its harness, so `resolvePricing` tries that key
+only after the exact id, and an id that already has a rate keeps it.
+
 Two things in `_DELEGATE_PRICING` look like copy errors on a skim and are not — a code review flagged
 both before this line existed to point at.
 

docs/REPORT_SCHEMA.md

@@ -125,10 +125,10 @@ spent and the only way back is to run it again. So the framework warns loudly an
 keeps going.
 
 The warning fires up front. `check_pricing_coverage` walks every model the run pins
-(subject agents and judge criteria) before the first task dispatches, and logs the
-ones the card cannot price — early enough to fix the card and restart while it is
-still cheap. After that the run is on its own: totals become floors, and
-`tasks_cost_incomplete` says how many rows are behind that floor.
+(subject agents, judge criteria and the simulated user) before the first task
+dispatches, and logs the ones the card cannot price — early enough to fix the card
+and restart while it is still cheap. After that the run is on its own: totals become
+floors, and `tasks_cost_incomplete` says how many rows are behind that floor.
 
 Consumers should treat any cost field as a lower bound whenever `cost_complete` is
 false, and must not read `None` as `0.0` — "nothing could be priced" and "it was

evalboard/lib/__tests__/pricing.test.ts

@@ -54,6 +54,13 @@ describe("resolvePricing", () => {
         expect(resolvePricing("bedrock/converse/deepseek.v3.2")?.inputPerMTok).toBe(0.74);
         expect(resolvePricing("eu.anthropic.claude-sonnet-4-6")?.outputPerMTok).toBe(15);
     });
+
+    test("prices the underscored id the Delegate backend echoes under its hyphenated key", () => {
+        // Mirror of pricing.py::delegate_rate_key; without it every token bucket
+        // on a Delegate row rendered "—".
+        expect(resolvePricing("gpt_5_6_luna")).toEqual(resolvePricing("gpt-5-6-luna"));
+        expect(tokenBucketUsd("gpt_5_6_luna", 1_000_000, "output")).toBe(1.2);
+    });
 });
 
 describe("tokenBucketUsd", () => {

evalboard/lib/__tests__/runs.test.ts

@@ -730,6 +730,15 @@ describe("tallyModels", () => {
         });
     });
 
+    test("the Delegate backend's underscored id is the same model as its hyphenated form", () => {
+        const out = tallyModels([
+            row("gpt_5_6_luna"),
+            row("gpt_5_6_luna"),
+            row("gpt-5-6-luna"),
+        ]);
+        expect(out).toEqual({ dominant: "gpt_5_6_luna", distinct: 1 });
+    });
+
     test("genuinely different models still count separately", () => {
         // The normalization must not collapse an A/B run's real spread.
         const out = tallyModels([

evalboard/lib/pricing.ts

@@ -49,9 +49,7 @@ const _ROUTING_PREFIXES = [
     "openrouter/",
 ];
 const _REGION_PREFIXES = ["eu.", "us.", "apac.", "global."];
-// Exported so the run header's model tally groups on the same key pricing looks
-// up on (lib/runs.ts::tallyModels).
-export function normalizeModel(model: string): string {
+function normalizeModel(model: string): string {
     let m = model.trim();
     for (const pre of _ROUTING_PREFIXES) {
         if (m.startsWith(pre)) {
@@ -69,6 +67,15 @@ export function normalizeModel(model: string): string {
     return m;
 }
 
+// The Delegate backend echoes underscored ids ("gpt_5_6_luna") while the rate
+// table keys the hyphenated form — mirror of pricing.py::delegate_rate_key. A row
+// does not say which harness wrote it, so resolvePricing tries this key only
+// after the exact id. Exported so the run header's model tally groups on it
+// (lib/runs.ts::tallyModels).
+export function rateKey(model: string): string {
+    return normalizeModel(model).replaceAll("_", "-");
+}
+
 // Resolve pricing for a model id, tolerating routing/region prefixes and undated
 // aliases (the recorded model is usually the canonical id like "claude-sonnet-4-6",
 // but LiteLLM/Bedrock runs record it prefixed, and some carry a trailing date).
@@ -85,6 +92,8 @@ export function resolvePricing(model: string | null): Pricing | null {
     // Try stripping a trailing -YYYYMMDD date.
     const undated = norm.replace(/-\d{8}$/, "");
     if (Object.hasOwn(PRICING, undated)) return PRICING[undated];
+    const hyphenated = rateKey(model);
+    if (Object.hasOwn(PRICING, hyphenated)) return PRICING[hyphenated];
     return null;
 }
 

evalboard/lib/runs.ts

@@ -17,7 +17,7 @@ import {
 import { DEFAULT_VARIANT_ID, isValidVariantId } from "./variants";
 import { DEFAULT_SOURCE, runsDirFor, type Source } from "./sources";
 import { DELIVERABLE_KINDS, DELIVERABLE_NAMES } from "./artifact-kinds";
-import { messageCostUsd, normalizeModel } from "./pricing";
+import { messageCostUsd, rateKey } from "./pricing";
 
 // Resolution order:
 //   1. EVALBOARD_LOCAL_RUNS_DIR — local mode, points at a coder_eval runs dir
@@ -1001,11 +1001,12 @@ export async function readRunSummary(
 // but also reports the spread so the run header can say when there was more than
 // one instead of silently picking a winner.
 //
-// The vote groups on the NORMALIZED id (same key pricing resolves on) because
-// the recorded string varies by code path: a row that errored before the model
+// The vote groups on the rate key (same key pricing resolves on) because the
+// recorded string varies by code path: a row that errored before the model
 // resolved keeps the qualified id it was configured with
-// ("eu.anthropic.claude-sonnet-5") while completed rows record the bare one, so
-// counting raw strings let one errored row read as a second model.
+// ("eu.anthropic.claude-sonnet-5") while completed rows record the bare one, and
+// a Delegate row records the backend's "gpt_5_6_luna" for a "gpt-5-6-luna"
+// config, so counting raw strings let one row read as a second model.
 //
 // Display stays the most common RAW string inside the winning group, so the chip
 // shows what the run recorded rather than a derived value.
@@ -1017,7 +1018,7 @@ export function tallyModels(rows: RawTaskResult[]): {
     for (const r of rows) {
         const m = r.model_used;
         if (typeof m !== "string" || !m) continue;
-        const key = normalizeModel(m);
+        const key = rateKey(m);
         let raw = groups.get(key);
         if (!raw) {
             raw = new Map<string, number>();

src/coder_eval/agents/delegate_agent.py

@@ -59,7 +59,7 @@
     TokenUsage,
     TurnRecord,
 )
-from coder_eval.pricing import calculate_cost
+from coder_eval.pricing import calculate_cost, delegate_rate_key
 from coder_eval.streaming.callbacks import StreamCallback, safe_emit
 from coder_eval.streaming.collector import EventCollector
 from coder_eval.streaming.events import (
@@ -1109,7 +1109,7 @@ def _finalize_turn(
         usage = state.usage or TokenUsage()
         if not usage.is_empty():
             cost = calculate_cost(
-                state.model_used or self.config.model or "",
+                delegate_rate_key(state.model_used or self.config.model or ""),
                 uncached_input_tokens=usage.uncached_input_tokens,
                 output_tokens=usage.output_tokens,
                 cache_creation_tokens=usage.cache_creation_input_tokens,

src/coder_eval/orchestration/batch.py

@@ -35,7 +35,7 @@
     format_task_log_id,
     write_text_atomic,
 )
-from ..pricing import unpriced_models
+from ..pricing import delegate_rate_key, unpriced_models
 from ..run_record import eval_result_to_task_dict
 from ..streaming.callbacks import StreamCallback
 from ..utils import get_version_info, looks_like_version
@@ -46,16 +46,23 @@
 
 
 def _run_models(resolved_tasks: list[ResolvedTask]) -> Iterator[str | None]:
-    """Every model id this run pins up front: subject agents and judge criteria.
+    """Every model id this run pins up front: subject agents, judge criteria and the simulated user.
 
-    A criterion's model is always priced from the rate card (no judge backend
-    reports a cost), so it matters more here than the agent's, which only falls
-    back to the card on a turn the backend never priced.
+    A criterion's model and the simulator's are always priced from the rate card
+    (no judge backend reports a cost, and ``simulator_cost_usd`` prices its tokens),
+    so they matter more here than the agent's, which only falls back to the card on
+    a turn the backend never priced.
     """
     for rt in resolved_tasks:
-        yield rt.task.agent.model if rt.task.agent else None
+        agent = rt.task.agent
+        if agent is not None and agent.model and agent.type == AgentKind.DELEGATE:
+            yield delegate_rate_key(agent.model)
+        else:
+            yield agent.model if agent else None
         for criterion in rt.task.success_criteria:
             yield getattr(criterion, "model", None)
+        if rt.task.simulation is not None and rt.task.simulation.enabled:
+            yield rt.task.simulation.model
 
 
 def check_pricing_coverage(resolved_tasks: list[ResolvedTask]) -> list[str]:
@@ -78,7 +85,7 @@ def check_pricing_coverage(resolved_tasks: list[ResolvedTask]) -> list[str]:
         return []
     logger.warning(
         "No pricing rate for %s. Turns the agent's own backend prices are unaffected, but any "
-        + "judge call, timed-out turn or killed partial will book its tokens with no cost, so "
+        + "judge call, simulator turn, timed-out turn or killed partial will book its tokens with no cost, so "
         + "run-level totals will understate the bill (RunSummary.cost_complete reports false). "
         + "Add the rate to coder_eval.pricing or register it from a plugin via register_pricing().",
         ", ".join(repr(m) for m in missing),

src/coder_eval/pricing.py

@@ -59,6 +59,11 @@ class ModelPricing:
 rather than a second hand-copied literal that could drift from ``_DELEGATE_PRICING``."""
 
 
+def delegate_rate_key(model: str) -> str:
+    """The rate-card key for a Delegate model id: the backend's ``gpt_5_6_luna`` prices as ``gpt-5-6-luna``."""
+    return model.replace("_", "-")
+
+
 _PRICING: dict[str, ModelPricing] = {
     # Fable 5.1 (and Mythos 5.1) price cache hits at 0.025x input, not the 0.1x
     # every other Claude model uses. Fable 5 pays $1 on the identical $10 base.

src/coder_eval/run_record.py

@@ -16,7 +16,14 @@
 from typing import Any
 
 from coder_eval.errors import truncate_crash_message
-from coder_eval.models import EvaluationResult, FinalStatus, judge_cost_usd, simulator_cost_usd, sum_costs
+from coder_eval.models import (
+    EvaluationResult,
+    FinalStatus,
+    JudgeCriterionResult,
+    judge_cost_usd,
+    simulator_cost_usd,
+    sum_costs,
+)
 from coder_eval.result_metrics import agent_wall_ms, expected_turns_overage, turn_time_buckets, visible_turn_count
 from coder_eval.result_metrics import has_final_reply as _has_final_reply
 
@@ -28,28 +35,36 @@
 
 
 def _cost_complete(result: EvaluationResult) -> bool:
-    """Whether this row's recorded agent spend accounts for everything it spent.
+    """Whether this row's recorded agent, judge and simulator spend accounts for everything it spent.
 
-    False means the costs on the row are a floor, not the bill. Two ways in:
+    False means the costs on the row are a floor, not the bill. Three ways in:
 
-    1. A turn burned tokens the rate card could not price. The card is the fallback
-       for anything the backend did not price itself, so with no rate those tokens
-       book no money.
-    2. The task was hard-killed by the task-level timeout. Keyed on the status
-       rather than on emptiness: the watchdog fires while the evaluation loop is
-       running, so a TIMEOUT row always lost an in-flight turn, even one that
-       completed earlier turns that do carry costs.
+    1. An agent turn burned tokens the rate card could not price. The card is the
+       fallback for anything the backend did not price itself, so those tokens book
+       no money.
+    2. A judge or the simulated user burned tokens with no cost. Both price from the
+       rate card, so an unpriced model drops that slice of the bill.
+    3. The task was hard-killed by the task-level timeout. Keyed on the status, not
+       on emptiness: the watchdog fires while the loop runs, so a TIMEOUT row always
+       lost an in-flight turn, even after earlier turns that carry costs.
 
-    True for a row that burned nothing: an error before the agent ran genuinely
-    cost zero, and a slow setup failure is as free as a fast one.
+    True for a row that burned nothing: an error before the agent ran cost zero.
     """
     if result.final_status is FinalStatus.TIMEOUT:
         return False
-    return all(
-        usage.total_cost_usd is not None
-        for t in result.iterations
-        if (usage := t.token_usage) is not None and not usage.is_empty()
-    )
+    sim = result.simulation
+    if (
+        sim is not None
+        and (sim.simulator_input_tokens or sim.simulator_output_tokens)
+        and simulator_cost_usd(result) is None
+    ):
+        return False
+    usages = [t.token_usage for t in result.iterations] + [
+        cr.token_usage
+        for cr in result.success_criteria_results + result.post_failure_criteria_results
+        if isinstance(cr, JudgeCriterionResult)
+    ]
+    return all(usage.total_cost_usd is not None for usage in usages if usage is not None and not usage.is_empty())
 
 
 def eval_result_to_task_dict(

tests/test_cost_accounting_paths.py

@@ -16,10 +16,12 @@
 
 from coder_eval.models import (
     ClaudeCodeAgentConfig,
+    DelegateAgentConfig,
     EvaluationResult,
     FinalStatus,
     JudgeCriterionResult,
     ResolvedTask,
+    SimulationConfig,
     SimulationTelemetry,
     TaskDefinition,
     TokenUsage,
@@ -89,6 +91,13 @@ def test_unpinned_model_is_not_flagged(self, tmp_path):
         """A task deferring its model to the route can't be pre-flighted from here."""
         assert check_pricing_coverage([_resolved(None, tmp_path)]) == []
 
+    def test_delegate_backend_id_is_priced_under_its_rate_card_key(self, tmp_path, caplog):
+        resolved = _resolved(None, tmp_path)
+        resolved.task.agent = DelegateAgentConfig(type="delegate", model="gpt_5_6_luna")
+        with caplog.at_level(logging.WARNING):
+            assert check_pricing_coverage([resolved]) == []
+        assert "No pricing rate" not in caplog.text
+
     def test_judge_criterion_model_is_pre_flighted(self, tmp_path):
         """A judge call is ALWAYS priced from the rate card, so its model matters most.
 
@@ -103,6 +112,19 @@ def test_default_judge_model_is_priced(self, tmp_path):
         judge = {"type": "llm_judge", "description": "d", "prompt": "grade it"}
         assert check_pricing_coverage([_resolved("claude-sonnet-5", tmp_path, [judge])]) == []
 
+    def test_simulator_model_is_pre_flighted(self, tmp_path):
+        """``simulator_cost_usd`` always prices from the rate card, so an unpriced simulator books no cost."""
+        resolved = _resolved("claude-sonnet-5", tmp_path)
+        resolved.task.simulation = SimulationConfig(
+            enabled=True, model="us.anthropic.claude-sonnet-99", persona="p", goal="g"
+        )
+        assert check_pricing_coverage([resolved]) == ["us.anthropic.claude-sonnet-99"]
+
+    def test_disabled_simulation_is_not_pre_flighted(self, tmp_path):
+        resolved = _resolved("claude-sonnet-5", tmp_path)
+        resolved.task.simulation = SimulationConfig(model="us.anthropic.claude-sonnet-99", persona="p", goal="g")
+        assert check_pricing_coverage([resolved]) == []
+
 
 class TestRowCostProjection:
     def test_cost_complete_false_when_a_turn_is_unpriced(self):
@@ -239,6 +261,7 @@ def test_unpriced_judge_is_skipped_not_fatal(self):
         row = eval_result_to_task_dict(result)
         assert row["judge_cost_usd"] == pytest.approx(0.02)
         assert row["total_cost_usd"] == pytest.approx(0.12)
+        assert row["cost_complete"] is False
 
     @staticmethod
     def _simulated(**env) -> EvaluationResult:
@@ -284,6 +307,11 @@ def test_unpriced_simulator_route_is_skipped_not_fatal(self):
         assert row["simulator_cost_usd"] is None
         # The agent's own spend still lands; only the simulator slice is missing.
         assert row["total_cost_usd"] == pytest.approx(0.1)
+        assert row["cost_complete"] is False
+
+    def test_cost_complete_true_when_the_simulator_is_priced(self):
+        result = self._simulated(bedrock_model="claude-haiku-4-5-20251001")
+        assert eval_result_to_task_dict(result)["cost_complete"] is True
 
 
 class TestTotalCost:

tests/test_delegate_agent.py

@@ -1092,6 +1092,20 @@ async def test_cost_wired_into_record_for_configured_model(self, patch_exec, tmp
         assert record.token_usage is not None
         assert record.token_usage.total_cost_usd == pytest.approx(0.95 + 4.0)
 
+    async def test_cost_prices_the_underscored_id_the_backend_echoes(self, patch_exec, tmp_path):
+        events = [
+            _result(
+                response="done",
+                model="gpt_5_6_luna",
+                usage={"input_tokens": 1_000_000, "output_tokens": 1_000_000},
+            )
+        ]
+        agent, _ = await _started_agent(patch_exec, events, tmp_path, model="gpt_5_6_luna")
+        record = await agent.communicate("hi")
+        assert record.model_used == "gpt_5_6_luna"
+        assert record.token_usage is not None
+        assert record.token_usage.total_cost_usd == pytest.approx(0.20 + 1.20)
+
     async def test_send_error_delivers_end_events_to_stream_callback(self, patch_exec, tmp_path):
         events = [_line({"type": "error", "message": "network blip"})]
         agent, _ = await _started_agent(patch_exec, events, tmp_path)