From f29048db2fef0f3715bcd0bcb1bb8e0b8e8c2e7d Mon Sep 17 00:00:00 2001 From: r3v3r53 Date: Fri, 18 Sep 2026 20:14:59 +0530 Subject: [PATCH] feat(cost): estimate benchmark spend and cost-per-flag OASIS records token usage on every run (RunResult.tokens) but never turns it into money, so a sweep's spend is invisible until the invoice arrives and models cannot be compared on cost-per-flag. Add src/lib/cost.ts: a pure lookup + arithmetic layer with no network calls. resolvePrice(model) -> ModelPrice | null (substring match, order-aware) estimateCost(model,tokens) -> CostEstimate estimateRunCost(run) -> CostEstimate summarizeCosts(runs) -> totals, per-model breakdown, usd-per-flag formatUsd(amount) -> sub-cent aware formatting Correct-model resolution: real runs record model: "custom" and put the concrete provider id in modelVersion. Pricing off run.model alone marked nearly every run unpriced, so estimateRunCost prefers modelVersion and falls back to model. Unknown models are reported as unpriced (usd: null + unpricedRuns count), never silently costed at zero, and models with zero flags get usdPerFlag: null rather than Infinity. Verified against 326 real run files on disk: runs 326, flags 95, unpriced 41, total .59, /bin/bash.05 per flag zai-org/GLM-5.3-Flash /bin/bash.0020 per flag (cheapest) moonshotai/Kimi-K3 /bin/bash.11 per flag 27 new tests cover pricing math, substring/ordering resolution, the modelVersion preference, unpriced accounting, per-model aggregation and formatting. Refs #77 --- src/lib/cost.ts | 235 +++++++++++++++++++++++++++++++++ tests/unit/cost.test.ts | 280 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 515 insertions(+) create mode 100644 src/lib/cost.ts create mode 100644 tests/unit/cost.test.ts diff --git a/src/lib/cost.ts b/src/lib/cost.ts new file mode 100644 index 0000000..5ea5526 --- /dev/null +++ b/src/lib/cost.ts @@ -0,0 +1,235 @@ +/** + * Cost estimation for benchmark runs. + * + * OASIS already records token usage per run (`RunResult.tokens`), but nothing + * translates those tokens into money — so a benchmark sweep's spend is invisible + * until the provider invoice arrives, and there is no way to compare models on + * cost-per-flag. + * + * This module is deliberately a pure lookup + arithmetic layer: no network + * calls, no provider SDKs. Prices are declared in a table so they are trivial to + * audit and update, and unknown models are reported as unpriced rather than + * silently guessed at zero (which is how a cost model quietly becomes fiction). + */ + +import type { RunResult, TokenUsage } from './types.js'; + +/** USD per 1,000,000 tokens. */ +export interface ModelPrice { + /** USD per 1M input (prompt) tokens. */ + input: number; + /** USD per 1M output (completion) tokens. */ + output: number; + /** Where the number came from, for auditability. */ + source: string; +} + +/** + * Price table, keyed by a lowercase substring matched against the model id. + * + * Matching is substring-based so versioned ids ("zai-org/GLM-5.3-Flash", + * "glm-5.3-flash-2026-01") resolve without an exhaustive enumeration. Order + * matters: the first matching key wins, so more specific keys must come first. + * + * NOTE: These are list prices for estimation and benchmark comparison, not a + * billing source of truth. Update as providers change pricing. + */ +export const MODEL_PRICES: ReadonlyArray = [ + // Anthropic + ['claude-opus', { input: 15, output: 75, source: 'anthropic-list' }], + ['claude-sonnet', { input: 3, output: 15, source: 'anthropic-list' }], + ['claude-haiku', { input: 0.8, output: 4, source: 'anthropic-list' }], + + // OpenAI + ['gpt-5', { input: 1.25, output: 10, source: 'openai-list' }], + ['gpt-4o-mini', { input: 0.15, output: 0.6, source: 'openai-list' }], + ['gpt-4o', { input: 2.5, output: 10, source: 'openai-list' }], + + // DeepInfra-served open models (used for the OASIS attacker roster) + ['glm-5.3-flash', { input: 0.06, output: 0.12, source: 'deepinfra-list' }], + ['glm-5.3', { input: 0.2, output: 0.6, source: 'deepinfra-list' }], + ['kimi-k3', { input: 0.3, output: 1.2, source: 'deepinfra-list' }], + ['minimax-m3', { input: 0.2, output: 1.0, source: 'deepinfra-list' }], + ['qwen3.8-27b', { input: 0.1, output: 0.3, source: 'deepinfra-list' }], + ['qwen3.8', { input: 0.25, output: 0.75, source: 'deepinfra-list' }], + ['deepseek-v4.1-flash', { input: 0.1, output: 0.3, source: 'deepinfra-list' }], + ['deepseek', { input: 0.27, output: 1.1, source: 'deepinfra-list' }], + ['inkling', { input: 0.1, output: 0.4, source: 'deepinfra-list' }], + ['llama-3.1-8b', { input: 0.03, output: 0.05, source: 'deepinfra-list' }], + ['mistral-7b', { input: 0.03, output: 0.05, source: 'deepinfra-list' }], +]; + +/** A run's estimated cost, or an explicit "unpriced" marker. */ +export interface CostEstimate { + model: string; + /** null when the model has no known price — never silently 0. */ + usd: number | null; + /** True when tokens were present but no price could be resolved. */ + unpriced: boolean; + inputTokens: number; + outputTokens: number; + price?: ModelPrice; +} + +/** Look up the price for a model id, or null if unknown. */ +export function resolvePrice(model: string): ModelPrice | null { + if (!model) return null; + const id = model.toLowerCase(); + for (const [key, price] of MODEL_PRICES) { + if (id.includes(key)) return price; + } + return null; +} + +/** Estimate the USD cost of a token count for a model. */ +export function estimateCost(model: string, tokens: TokenUsage): CostEstimate { + const price = resolvePrice(model); + const inputTokens = tokens.input ?? 0; + const outputTokens = tokens.output ?? 0; + + if (!price) { + return { + model, + usd: null, + unpriced: inputTokens + outputTokens > 0, + inputTokens, + outputTokens, + }; + } + + const usd = + (inputTokens / 1_000_000) * price.input + + (outputTokens / 1_000_000) * price.output; + + return { + model, + usd, + unpriced: false, + inputTokens, + outputTokens, + price, + }; +} + +/** + * Estimate the cost of a single run. + * + * Prefers `modelVersion` (the concrete provider model id, e.g. + * "zai-org/GLM-5.3-Flash") and falls back to `model` (a coarse provenance tag + * that is often just "custom"). Pricing off `model` alone would mark nearly + * every run unpriced. + */ +export function estimateRunCost(run: RunResult): CostEstimate { + const concrete = run.modelVersion || ''; + const coarse = run.model || ''; + + // Pick whichever identifier we can actually price; prefer the concrete one. + if (resolvePrice(concrete)) { + return estimateCost(concrete, run.tokens); + } + if (resolvePrice(coarse)) { + return estimateCost(coarse, run.tokens); + } + return estimateCost(concrete || coarse, run.tokens); +} + +/** Aggregate cost across runs, with per-model and cost-per-flag breakdowns. */ +export interface CostSummary { + totalUsd: number; + /** Runs whose model had no known price. */ + unpricedRuns: number; + runCount: number; + totalInputTokens: number; + totalOutputTokens: number; + /** Total flags captured across the runs. */ + flags: number; + /** USD per captured flag; null when no flags were captured. */ + usdPerFlag: number | null; + byModel: Array<{ + model: string; + runs: number; + usd: number; + unpriced: boolean; + inputTokens: number; + outputTokens: number; + flags: number; + usdPerFlag: number | null; + }>; +} + +/** Format a USD amount for terminal display. */ +export function formatUsd(amount: number): string { + if (amount < 0.01 && amount > 0) return `$${amount.toFixed(4)}`; + return `$${amount.toFixed(2)}`; +} + +/** + * Summarise the cost of a set of runs. + * Unpriced runs are counted and reported separately rather than assumed free. + */ +export function summarizeCosts(runs: RunResult[]): CostSummary { + const perModel = new Map(); + + let totalUsd = 0; + let unpricedRuns = 0; + let totalInputTokens = 0; + let totalOutputTokens = 0; + let flags = 0; + + for (const run of runs) { + const est = estimateRunCost(run); + const captured = run.flag != null && run.flag !== ''; + if (captured) flags += 1; + + totalInputTokens += est.inputTokens; + totalOutputTokens += est.outputTokens; + + if (est.usd == null) { + if (est.unpriced) unpricedRuns += 1; + } else { + totalUsd += est.usd; + } + + // Group by the identifier the cost was computed from, so the breakdown does + // not collapse every run into the coarse "custom" provenance tag. + const key = est.model; + let bucket = perModel.get(key); + if (!bucket) { + bucket = { + model: key, + runs: 0, + usd: 0, + unpriced: est.unpriced, + inputTokens: 0, + outputTokens: 0, + flags: 0, + usdPerFlag: null, + }; + perModel.set(key, bucket); + } + bucket.runs += 1; + bucket.inputTokens += est.inputTokens; + bucket.outputTokens += est.outputTokens; + if (captured) bucket.flags += 1; + if (est.usd != null) { + bucket.usd += est.usd; + } else if (est.unpriced) { + bucket.unpriced = true; + } + } + + const byModel = Array.from(perModel.values()) + .map(b => ({ ...b, usdPerFlag: b.flags > 0 ? b.usd / b.flags : null })) + .sort((a, b) => b.usd - a.usd); + + return { + totalUsd, + unpricedRuns, + runCount: runs.length, + totalInputTokens, + totalOutputTokens, + flags, + usdPerFlag: flags > 0 ? totalUsd / flags : null, + byModel, + }; +} diff --git a/tests/unit/cost.test.ts b/tests/unit/cost.test.ts new file mode 100644 index 0000000..304a07b --- /dev/null +++ b/tests/unit/cost.test.ts @@ -0,0 +1,280 @@ +import { describe, it, expect } from 'vitest'; +import { + resolvePrice, + estimateCost, + estimateRunCost, + summarizeCosts, + formatUsd, + MODEL_PRICES, +} from '../../src/lib/cost.js'; +import type { RunResult, TokenUsage } from '../../src/lib/types.js'; + +function tokens(input: number, output: number): TokenUsage { + return { input, output, total: input + output }; +} + +function makeRun(model: string, t: TokenUsage, flag: string | null = null): RunResult { + return { + id: 'run', + model, + modelVersion: model, + challenge: 'challenge', + startTime: new Date(0), + endTime: new Date(0), + success: flag != null, + flag, + totalTime: 1, + iterations: 1, + tokens: t, + steps: [], + techniquesUsed: [], + tacticBreakdown: {}, + methodologies: [], + toolsUsed: [], + methodologyBreakdown: {}, + }; +} + +// ============================================================================= +// resolvePrice +// ============================================================================= + +describe('resolvePrice', () => { + it('resolves exact model ids', () => { + expect(resolvePrice('claude-opus-4-8')?.input).toBe(15); + expect(resolvePrice('zai-org/GLM-5.3-Flash')?.input).toBe(0.06); + }); + + it('is case-insensitive', () => { + expect(resolvePrice('Claude-Opus-4-8')).not.toBeNull(); + expect(resolvePrice('ZAI-ORG/GLM-5.3')).not.toBeNull(); + }); + + it('prefers more specific keys (glm-5.3-flash before glm-5.3)', () => { + const flash = resolvePrice('zai-org/GLM-5.3-Flash'); + const standard = resolvePrice('zai-org/GLM-5.3'); + expect(flash?.input).toBe(0.06); + expect(standard?.input).toBe(0.2); + expect(flash?.input).not.toBe(standard?.input); + }); + + it('resolves versioned and provider-prefixed ids by substring', () => { + expect(resolvePrice('moonshotai/Kimi-K3')?.input).toBe(0.3); + expect(resolvePrice('MiniMaxAI/MiniMax-M3')?.input).toBe(0.2); + expect(resolvePrice('Qwen/Qwen3.8-27B')?.input).toBe(0.1); + }); + + it('returns null for unknown and empty models', () => { + expect(resolvePrice('some-unknown-model')).toBeNull(); + expect(resolvePrice('')).toBeNull(); + }); +}); + +// ============================================================================= +// MODEL_PRICES table sanity +// ============================================================================= + +describe('MODEL_PRICES', () => { + it('has non-negative prices with a source for every entry', () => { + for (const [key, price] of MODEL_PRICES) { + expect(key.length).toBeGreaterThan(0); + expect(price.input).toBeGreaterThanOrEqual(0); + expect(price.output).toBeGreaterThanOrEqual(0); + expect(price.source.length).toBeGreaterThan(0); + } + }); + + it('lists no duplicate keys (dup keys would shadow silently)', () => { + const keys = MODEL_PRICES.map(([k]) => k); + expect(new Set(keys).size).toBe(keys.length); + }); +}); + +// ============================================================================= +// estimateCost +// ============================================================================= + +describe('estimateCost', () => { + it('computes input + output cost correctly', () => { + // opus: $15/1M in, $75/1M out + const est = estimateCost('claude-opus-4-8', tokens(1_000_000, 1_000_000)); + expect(est.usd).toBeCloseTo(90, 6); + expect(est.unpriced).toBe(false); + }); + + it('handles zero tokens on a priced model as $0 (not unpriced)', () => { + const est = estimateCost('claude-opus-4-8', tokens(0, 0)); + expect(est.usd).toBe(0); + expect(est.unpriced).toBe(false); + }); + + it('marks an unknown model with tokens as unpriced, never 0', () => { + const est = estimateCost('who-knows', tokens(1000, 1000)); + expect(est.usd).toBeNull(); + expect(est.unpriced).toBe(true); + }); + + it('does not flag an unknown model with no tokens as unpriced', () => { + const est = estimateCost('who-knows', tokens(0, 0)); + expect(est.usd).toBeNull(); + expect(est.unpriced).toBe(false); + }); + + it('scales linearly with token count', () => { + const small = estimateCost('claude-haiku', tokens(1000, 1000)); + const large = estimateCost('claude-haiku', tokens(10_000, 10_000)); + expect(large.usd!).toBeCloseTo(small.usd! * 10, 9); + }); + + it('reports the price used, for auditability', () => { + const est = estimateCost('moonshotai/Kimi-K3', tokens(1000, 0)); + expect(est.price?.source).toBe('deepinfra-list'); + }); +}); + +// ============================================================================= +// estimateRunCost +// ============================================================================= + +describe('estimateRunCost', () => { + it('reads tokens off the run result', () => { + const run = makeRun('claude-sonnet-4', tokens(2_000_000, 1_000_000)); + // sonnet: $3/1M in, $15/1M out → 6 + 15 = 21 + expect(estimateRunCost(run).usd).toBeCloseTo(21, 6); + }); + + it('prices off modelVersion when model is the coarse "custom" tag', () => { + // Real OASIS runs record model: "custom" and put the provider id in + // modelVersion — pricing off `model` alone would mark everything unpriced. + const run: RunResult = { + ...makeRun('custom', tokens(1_000_000, 1_000_000)), + modelVersion: 'zai-org/GLM-5.3-Flash', + }; + const est = estimateRunCost(run); + expect(est.unpriced).toBe(false); + expect(est.usd).toBeCloseTo(0.18, 6); // 0.06 in + 0.12 out + expect(est.model).toBe('zai-org/GLM-5.3-Flash'); + }); + + it('falls back to model when modelVersion is unpriceable', () => { + const run: RunResult = { + ...makeRun('claude-haiku', tokens(1_000_000, 0)), + modelVersion: 'some-unknown-build', + }; + const est = estimateRunCost(run); + expect(est.unpriced).toBe(false); + expect(est.usd).toBeCloseTo(0.8, 6); + }); + + it('reports the concrete id as unpriced when neither id resolves', () => { + const run: RunResult = { + ...makeRun('custom', tokens(100, 100)), + modelVersion: 'totally-unknown', + }; + const est = estimateRunCost(run); + expect(est.usd).toBeNull(); + expect(est.unpriced).toBe(true); + }); +}); + +// ============================================================================= +// summarizeCosts +// ============================================================================= + +describe('summarizeCosts', () => { + it('returns an empty-but-valid summary for no runs', () => { + const s = summarizeCosts([]); + expect(s.totalUsd).toBe(0); + expect(s.runCount).toBe(0); + expect(s.flags).toBe(0); + expect(s.usdPerFlag).toBeNull(); + expect(s.byModel).toEqual([]); + }); + + it('totals cost and flags across runs', () => { + const runs = [ + makeRun('claude-opus-4-8', tokens(1_000_000, 1_000_000), 'FLAG{a}'), // $90 + makeRun('claude-opus-4-8', tokens(0, 0), null), // $0 + makeRun('claude-haiku', tokens(1_000_000, 0), 'FLAG{b}'), // $0.80 + ]; + const s = summarizeCosts(runs); + expect(s.totalUsd).toBeCloseTo(90.8, 6); + expect(s.flags).toBe(2); + expect(s.usdPerFlag).toBeCloseTo(45.4, 6); + expect(s.runCount).toBe(3); + }); + + it('groups by model and computes per-model cost-per-flag', () => { + const runs = [ + makeRun('claude-opus-4-8', tokens(1_000_000, 0), 'F1'), // $15 + makeRun('claude-opus-4-8', tokens(1_000_000, 0), 'F2'), // $15 + makeRun('claude-haiku', tokens(1_000_000, 0), null), // $0.80, 0 flags + ]; + const s = summarizeCosts(runs); + + const opus = s.byModel.find(m => m.model === 'claude-opus-4-8')!; + expect(opus.runs).toBe(2); + expect(opus.flags).toBe(2); + expect(opus.usd).toBeCloseTo(30, 6); + expect(opus.usdPerFlag).toBeCloseTo(15, 6); + + const haiku = s.byModel.find(m => m.model === 'claude-haiku')!; + expect(haiku.flags).toBe(0); + expect(haiku.usdPerFlag).toBeNull(); // no flags → no ratio, not Infinity + }); + + it('counts unpriced runs separately instead of pretending they are free', () => { + const runs = [ + makeRun('claude-opus-4-8', tokens(1_000_000, 0), 'F'), // $15 + makeRun('mystery-model', tokens(500_000, 500_000), 'F'), // unpriced + ]; + const s = summarizeCosts(runs); + expect(s.unpricedRuns).toBe(1); + expect(s.totalUsd).toBeCloseTo(15, 6); // mystery run contributes nothing + expect(s.byModel.find(m => m.model === 'mystery-model')?.unpriced).toBe(true); + }); + + it('sorts models by spend, descending', () => { + const runs = [ + makeRun('claude-haiku', tokens(1_000_000, 0), null), // $0.80 + makeRun('claude-opus-4-8', tokens(1_000_000, 0), null), // $15 + ]; + const s = summarizeCosts(runs); + expect(s.byModel[0].model).toBe('claude-opus-4-8'); + expect(s.byModel[1].model).toBe('claude-haiku'); + }); + + it('ignores empty-string flags when counting captures', () => { + const runs = [makeRun('claude-haiku', tokens(1000, 0), '')]; + expect(summarizeCosts(runs).flags).toBe(0); + }); + + it('totals input and output tokens across runs', () => { + const runs = [ + makeRun('claude-haiku', tokens(100, 200), null), + makeRun('claude-haiku', tokens(300, 400), null), + ]; + const s = summarizeCosts(runs); + expect(s.totalInputTokens).toBe(400); + expect(s.totalOutputTokens).toBe(600); + }); +}); + +// ============================================================================= +// formatUsd +// ============================================================================= + +describe('formatUsd', () => { + it('uses 4 decimals for sub-cent amounts so small runs are not shown as $0.00', () => { + expect(formatUsd(0.0034)).toBe('$0.0034'); + }); + + it('uses 2 decimals for normal amounts', () => { + expect(formatUsd(12.5)).toBe('$12.50'); + expect(formatUsd(90.8)).toBe('$90.80'); + }); + + it('formats zero as $0.00', () => { + expect(formatUsd(0)).toBe('$0.00'); + }); +});