diff --git a/src/lib/cost.ts b/src/lib/cost.ts new file mode 100644 index 0000000..5ea5526 --- /dev/null +++ b/src/lib/cost.ts @@ -0,0 +1,235 @@ +/** + * Cost estimation for benchmark runs. + * + * OASIS already records token usage per run (`RunResult.tokens`), but nothing + * translates those tokens into money — so a benchmark sweep's spend is invisible + * until the provider invoice arrives, and there is no way to compare models on + * cost-per-flag. + * + * This module is deliberately a pure lookup + arithmetic layer: no network + * calls, no provider SDKs. Prices are declared in a table so they are trivial to + * audit and update, and unknown models are reported as unpriced rather than + * silently guessed at zero (which is how a cost model quietly becomes fiction). + */ + +import type { RunResult, TokenUsage } from './types.js'; + +/** USD per 1,000,000 tokens. */ +export interface ModelPrice { + /** USD per 1M input (prompt) tokens. */ + input: number; + /** USD per 1M output (completion) tokens. */ + output: number; + /** Where the number came from, for auditability. */ + source: string; +} + +/** + * Price table, keyed by a lowercase substring matched against the model id. + * + * Matching is substring-based so versioned ids ("zai-org/GLM-5.3-Flash", + * "glm-5.3-flash-2026-01") resolve without an exhaustive enumeration. Order + * matters: the first matching key wins, so more specific keys must come first. + * + * NOTE: These are list prices for estimation and benchmark comparison, not a + * billing source of truth. Update as providers change pricing. + */ +export const MODEL_PRICES: ReadonlyArray = [ + // Anthropic + ['claude-opus', { input: 15, output: 75, source: 'anthropic-list' }], + ['claude-sonnet', { input: 3, output: 15, source: 'anthropic-list' }], + ['claude-haiku', { input: 0.8, output: 4, source: 'anthropic-list' }], + + // OpenAI + ['gpt-5', { input: 1.25, output: 10, source: 'openai-list' }], + ['gpt-4o-mini', { input: 0.15, output: 0.6, source: 'openai-list' }], + ['gpt-4o', { input: 2.5, output: 10, source: 'openai-list' }], + + // DeepInfra-served open models (used for the OASIS attacker roster) + ['glm-5.3-flash', { input: 0.06, output: 0.12, source: 'deepinfra-list' }], + ['glm-5.3', { input: 0.2, output: 0.6, source: 'deepinfra-list' }], + ['kimi-k3', { input: 0.3, output: 1.2, source: 'deepinfra-list' }], + ['minimax-m3', { input: 0.2, output: 1.0, source: 'deepinfra-list' }], + ['qwen3.8-27b', { input: 0.1, output: 0.3, source: 'deepinfra-list' }], + ['qwen3.8', { input: 0.25, output: 0.75, source: 'deepinfra-list' }], + ['deepseek-v4.1-flash', { input: 0.1, output: 0.3, source: 'deepinfra-list' }], + ['deepseek', { input: 0.27, output: 1.1, source: 'deepinfra-list' }], + ['inkling', { input: 0.1, output: 0.4, source: 'deepinfra-list' }], + ['llama-3.1-8b', { input: 0.03, output: 0.05, source: 'deepinfra-list' }], + ['mistral-7b', { input: 0.03, output: 0.05, source: 'deepinfra-list' }], +]; + +/** A run's estimated cost, or an explicit "unpriced" marker. */ +export interface CostEstimate { + model: string; + /** null when the model has no known price — never silently 0. */ + usd: number | null; + /** True when tokens were present but no price could be resolved. */ + unpriced: boolean; + inputTokens: number; + outputTokens: number; + price?: ModelPrice; +} + +/** Look up the price for a model id, or null if unknown. */ +export function resolvePrice(model: string): ModelPrice | null { + if (!model) return null; + const id = model.toLowerCase(); + for (const [key, price] of MODEL_PRICES) { + if (id.includes(key)) return price; + } + return null; +} + +/** Estimate the USD cost of a token count for a model. */ +export function estimateCost(model: string, tokens: TokenUsage): CostEstimate { + const price = resolvePrice(model); + const inputTokens = tokens.input ?? 0; + const outputTokens = tokens.output ?? 0; + + if (!price) { + return { + model, + usd: null, + unpriced: inputTokens + outputTokens > 0, + inputTokens, + outputTokens, + }; + } + + const usd = + (inputTokens / 1_000_000) * price.input + + (outputTokens / 1_000_000) * price.output; + + return { + model, + usd, + unpriced: false, + inputTokens, + outputTokens, + price, + }; +} + +/** + * Estimate the cost of a single run. + * + * Prefers `modelVersion` (the concrete provider model id, e.g. + * "zai-org/GLM-5.3-Flash") and falls back to `model` (a coarse provenance tag + * that is often just "custom"). Pricing off `model` alone would mark nearly + * every run unpriced. + */ +export function estimateRunCost(run: RunResult): CostEstimate { + const concrete = run.modelVersion || ''; + const coarse = run.model || ''; + + // Pick whichever identifier we can actually price; prefer the concrete one. + if (resolvePrice(concrete)) { + return estimateCost(concrete, run.tokens); + } + if (resolvePrice(coarse)) { + return estimateCost(coarse, run.tokens); + } + return estimateCost(concrete || coarse, run.tokens); +} + +/** Aggregate cost across runs, with per-model and cost-per-flag breakdowns. */ +export interface CostSummary { + totalUsd: number; + /** Runs whose model had no known price. */ + unpricedRuns: number; + runCount: number; + totalInputTokens: number; + totalOutputTokens: number; + /** Total flags captured across the runs. */ + flags: number; + /** USD per captured flag; null when no flags were captured. */ + usdPerFlag: number | null; + byModel: Array<{ + model: string; + runs: number; + usd: number; + unpriced: boolean; + inputTokens: number; + outputTokens: number; + flags: number; + usdPerFlag: number | null; + }>; +} + +/** Format a USD amount for terminal display. */ +export function formatUsd(amount: number): string { + if (amount < 0.01 && amount > 0) return `$${amount.toFixed(4)}`; + return `$${amount.toFixed(2)}`; +} + +/** + * Summarise the cost of a set of runs. + * Unpriced runs are counted and reported separately rather than assumed free. + */ +export function summarizeCosts(runs: RunResult[]): CostSummary { + const perModel = new Map(); + + let totalUsd = 0; + let unpricedRuns = 0; + let totalInputTokens = 0; + let totalOutputTokens = 0; + let flags = 0; + + for (const run of runs) { + const est = estimateRunCost(run); + const captured = run.flag != null && run.flag !== ''; + if (captured) flags += 1; + + totalInputTokens += est.inputTokens; + totalOutputTokens += est.outputTokens; + + if (est.usd == null) { + if (est.unpriced) unpricedRuns += 1; + } else { + totalUsd += est.usd; + } + + // Group by the identifier the cost was computed from, so the breakdown does + // not collapse every run into the coarse "custom" provenance tag. + const key = est.model; + let bucket = perModel.get(key); + if (!bucket) { + bucket = { + model: key, + runs: 0, + usd: 0, + unpriced: est.unpriced, + inputTokens: 0, + outputTokens: 0, + flags: 0, + usdPerFlag: null, + }; + perModel.set(key, bucket); + } + bucket.runs += 1; + bucket.inputTokens += est.inputTokens; + bucket.outputTokens += est.outputTokens; + if (captured) bucket.flags += 1; + if (est.usd != null) { + bucket.usd += est.usd; + } else if (est.unpriced) { + bucket.unpriced = true; + } + } + + const byModel = Array.from(perModel.values()) + .map(b => ({ ...b, usdPerFlag: b.flags > 0 ? b.usd / b.flags : null })) + .sort((a, b) => b.usd - a.usd); + + return { + totalUsd, + unpricedRuns, + runCount: runs.length, + totalInputTokens, + totalOutputTokens, + flags, + usdPerFlag: flags > 0 ? totalUsd / flags : null, + byModel, + }; +} diff --git a/tests/unit/cost.test.ts b/tests/unit/cost.test.ts new file mode 100644 index 0000000..304a07b --- /dev/null +++ b/tests/unit/cost.test.ts @@ -0,0 +1,280 @@ +import { describe, it, expect } from 'vitest'; +import { + resolvePrice, + estimateCost, + estimateRunCost, + summarizeCosts, + formatUsd, + MODEL_PRICES, +} from '../../src/lib/cost.js'; +import type { RunResult, TokenUsage } from '../../src/lib/types.js'; + +function tokens(input: number, output: number): TokenUsage { + return { input, output, total: input + output }; +} + +function makeRun(model: string, t: TokenUsage, flag: string | null = null): RunResult { + return { + id: 'run', + model, + modelVersion: model, + challenge: 'challenge', + startTime: new Date(0), + endTime: new Date(0), + success: flag != null, + flag, + totalTime: 1, + iterations: 1, + tokens: t, + steps: [], + techniquesUsed: [], + tacticBreakdown: {}, + methodologies: [], + toolsUsed: [], + methodologyBreakdown: {}, + }; +} + +// ============================================================================= +// resolvePrice +// ============================================================================= + +describe('resolvePrice', () => { + it('resolves exact model ids', () => { + expect(resolvePrice('claude-opus-4-8')?.input).toBe(15); + expect(resolvePrice('zai-org/GLM-5.3-Flash')?.input).toBe(0.06); + }); + + it('is case-insensitive', () => { + expect(resolvePrice('Claude-Opus-4-8')).not.toBeNull(); + expect(resolvePrice('ZAI-ORG/GLM-5.3')).not.toBeNull(); + }); + + it('prefers more specific keys (glm-5.3-flash before glm-5.3)', () => { + const flash = resolvePrice('zai-org/GLM-5.3-Flash'); + const standard = resolvePrice('zai-org/GLM-5.3'); + expect(flash?.input).toBe(0.06); + expect(standard?.input).toBe(0.2); + expect(flash?.input).not.toBe(standard?.input); + }); + + it('resolves versioned and provider-prefixed ids by substring', () => { + expect(resolvePrice('moonshotai/Kimi-K3')?.input).toBe(0.3); + expect(resolvePrice('MiniMaxAI/MiniMax-M3')?.input).toBe(0.2); + expect(resolvePrice('Qwen/Qwen3.8-27B')?.input).toBe(0.1); + }); + + it('returns null for unknown and empty models', () => { + expect(resolvePrice('some-unknown-model')).toBeNull(); + expect(resolvePrice('')).toBeNull(); + }); +}); + +// ============================================================================= +// MODEL_PRICES table sanity +// ============================================================================= + +describe('MODEL_PRICES', () => { + it('has non-negative prices with a source for every entry', () => { + for (const [key, price] of MODEL_PRICES) { + expect(key.length).toBeGreaterThan(0); + expect(price.input).toBeGreaterThanOrEqual(0); + expect(price.output).toBeGreaterThanOrEqual(0); + expect(price.source.length).toBeGreaterThan(0); + } + }); + + it('lists no duplicate keys (dup keys would shadow silently)', () => { + const keys = MODEL_PRICES.map(([k]) => k); + expect(new Set(keys).size).toBe(keys.length); + }); +}); + +// ============================================================================= +// estimateCost +// ============================================================================= + +describe('estimateCost', () => { + it('computes input + output cost correctly', () => { + // opus: $15/1M in, $75/1M out + const est = estimateCost('claude-opus-4-8', tokens(1_000_000, 1_000_000)); + expect(est.usd).toBeCloseTo(90, 6); + expect(est.unpriced).toBe(false); + }); + + it('handles zero tokens on a priced model as $0 (not unpriced)', () => { + const est = estimateCost('claude-opus-4-8', tokens(0, 0)); + expect(est.usd).toBe(0); + expect(est.unpriced).toBe(false); + }); + + it('marks an unknown model with tokens as unpriced, never 0', () => { + const est = estimateCost('who-knows', tokens(1000, 1000)); + expect(est.usd).toBeNull(); + expect(est.unpriced).toBe(true); + }); + + it('does not flag an unknown model with no tokens as unpriced', () => { + const est = estimateCost('who-knows', tokens(0, 0)); + expect(est.usd).toBeNull(); + expect(est.unpriced).toBe(false); + }); + + it('scales linearly with token count', () => { + const small = estimateCost('claude-haiku', tokens(1000, 1000)); + const large = estimateCost('claude-haiku', tokens(10_000, 10_000)); + expect(large.usd!).toBeCloseTo(small.usd! * 10, 9); + }); + + it('reports the price used, for auditability', () => { + const est = estimateCost('moonshotai/Kimi-K3', tokens(1000, 0)); + expect(est.price?.source).toBe('deepinfra-list'); + }); +}); + +// ============================================================================= +// estimateRunCost +// ============================================================================= + +describe('estimateRunCost', () => { + it('reads tokens off the run result', () => { + const run = makeRun('claude-sonnet-4', tokens(2_000_000, 1_000_000)); + // sonnet: $3/1M in, $15/1M out → 6 + 15 = 21 + expect(estimateRunCost(run).usd).toBeCloseTo(21, 6); + }); + + it('prices off modelVersion when model is the coarse "custom" tag', () => { + // Real OASIS runs record model: "custom" and put the provider id in + // modelVersion — pricing off `model` alone would mark everything unpriced. + const run: RunResult = { + ...makeRun('custom', tokens(1_000_000, 1_000_000)), + modelVersion: 'zai-org/GLM-5.3-Flash', + }; + const est = estimateRunCost(run); + expect(est.unpriced).toBe(false); + expect(est.usd).toBeCloseTo(0.18, 6); // 0.06 in + 0.12 out + expect(est.model).toBe('zai-org/GLM-5.3-Flash'); + }); + + it('falls back to model when modelVersion is unpriceable', () => { + const run: RunResult = { + ...makeRun('claude-haiku', tokens(1_000_000, 0)), + modelVersion: 'some-unknown-build', + }; + const est = estimateRunCost(run); + expect(est.unpriced).toBe(false); + expect(est.usd).toBeCloseTo(0.8, 6); + }); + + it('reports the concrete id as unpriced when neither id resolves', () => { + const run: RunResult = { + ...makeRun('custom', tokens(100, 100)), + modelVersion: 'totally-unknown', + }; + const est = estimateRunCost(run); + expect(est.usd).toBeNull(); + expect(est.unpriced).toBe(true); + }); +}); + +// ============================================================================= +// summarizeCosts +// ============================================================================= + +describe('summarizeCosts', () => { + it('returns an empty-but-valid summary for no runs', () => { + const s = summarizeCosts([]); + expect(s.totalUsd).toBe(0); + expect(s.runCount).toBe(0); + expect(s.flags).toBe(0); + expect(s.usdPerFlag).toBeNull(); + expect(s.byModel).toEqual([]); + }); + + it('totals cost and flags across runs', () => { + const runs = [ + makeRun('claude-opus-4-8', tokens(1_000_000, 1_000_000), 'FLAG{a}'), // $90 + makeRun('claude-opus-4-8', tokens(0, 0), null), // $0 + makeRun('claude-haiku', tokens(1_000_000, 0), 'FLAG{b}'), // $0.80 + ]; + const s = summarizeCosts(runs); + expect(s.totalUsd).toBeCloseTo(90.8, 6); + expect(s.flags).toBe(2); + expect(s.usdPerFlag).toBeCloseTo(45.4, 6); + expect(s.runCount).toBe(3); + }); + + it('groups by model and computes per-model cost-per-flag', () => { + const runs = [ + makeRun('claude-opus-4-8', tokens(1_000_000, 0), 'F1'), // $15 + makeRun('claude-opus-4-8', tokens(1_000_000, 0), 'F2'), // $15 + makeRun('claude-haiku', tokens(1_000_000, 0), null), // $0.80, 0 flags + ]; + const s = summarizeCosts(runs); + + const opus = s.byModel.find(m => m.model === 'claude-opus-4-8')!; + expect(opus.runs).toBe(2); + expect(opus.flags).toBe(2); + expect(opus.usd).toBeCloseTo(30, 6); + expect(opus.usdPerFlag).toBeCloseTo(15, 6); + + const haiku = s.byModel.find(m => m.model === 'claude-haiku')!; + expect(haiku.flags).toBe(0); + expect(haiku.usdPerFlag).toBeNull(); // no flags → no ratio, not Infinity + }); + + it('counts unpriced runs separately instead of pretending they are free', () => { + const runs = [ + makeRun('claude-opus-4-8', tokens(1_000_000, 0), 'F'), // $15 + makeRun('mystery-model', tokens(500_000, 500_000), 'F'), // unpriced + ]; + const s = summarizeCosts(runs); + expect(s.unpricedRuns).toBe(1); + expect(s.totalUsd).toBeCloseTo(15, 6); // mystery run contributes nothing + expect(s.byModel.find(m => m.model === 'mystery-model')?.unpriced).toBe(true); + }); + + it('sorts models by spend, descending', () => { + const runs = [ + makeRun('claude-haiku', tokens(1_000_000, 0), null), // $0.80 + makeRun('claude-opus-4-8', tokens(1_000_000, 0), null), // $15 + ]; + const s = summarizeCosts(runs); + expect(s.byModel[0].model).toBe('claude-opus-4-8'); + expect(s.byModel[1].model).toBe('claude-haiku'); + }); + + it('ignores empty-string flags when counting captures', () => { + const runs = [makeRun('claude-haiku', tokens(1000, 0), '')]; + expect(summarizeCosts(runs).flags).toBe(0); + }); + + it('totals input and output tokens across runs', () => { + const runs = [ + makeRun('claude-haiku', tokens(100, 200), null), + makeRun('claude-haiku', tokens(300, 400), null), + ]; + const s = summarizeCosts(runs); + expect(s.totalInputTokens).toBe(400); + expect(s.totalOutputTokens).toBe(600); + }); +}); + +// ============================================================================= +// formatUsd +// ============================================================================= + +describe('formatUsd', () => { + it('uses 4 decimals for sub-cent amounts so small runs are not shown as $0.00', () => { + expect(formatUsd(0.0034)).toBe('$0.0034'); + }); + + it('uses 2 decimals for normal amounts', () => { + expect(formatUsd(12.5)).toBe('$12.50'); + expect(formatUsd(90.8)).toBe('$90.80'); + }); + + it('formats zero as $0.00', () => { + expect(formatUsd(0)).toBe('$0.00'); + }); +});