diff --git a/packages/aikit/src/llm/shared.ts b/packages/aikit/src/llm/shared.ts index d5629bb8..f9f0631b 100644 --- a/packages/aikit/src/llm/shared.ts +++ b/packages/aikit/src/llm/shared.ts @@ -35,6 +35,13 @@ export type SharedOptions = Static; /** Headroom left for the prompt when sizing a response against the context window. */ const CONTEXT_SAFETY_TOKENS = 4096; +// OpenRouter counts the input with its own tokenizer and rejects input + max_tokens above the +// window before routing. It counted prose up to 12% above the provider's count. +const CONTEXT_SAFETY_RATIO = 0.15; +// The 15% margin is kept only while it leaves at least this much output. Otherwise (a nearly full +// large window, or any small window such as a 32k local model) only the fixed margin is kept, and +// output is capped at this size so an OpenRouter overcount still fits for a while before compaction. +const MIN_OUTPUT_WITH_RATIO_MARGIN = 32768; // The smallest ceiling OpenAI accepts (`max_output_tokens` >= 16). Below it an // overflowing prompt fails as an invalid parameter instead of a context overflow. const MIN_MAX_TOKENS = 16; @@ -60,7 +67,8 @@ export function applyDefaultMaxTokens, @@ -68,8 +76,13 @@ export function clampMaxTokensToContext( maxTokens: number, ): number { if (model.contextWindow <= 0) return Math.max(MIN_MAX_TOKENS, maxTokens); - const available = model.contextWindow - estimateContextTokens(context).tokens - CONTEXT_SAFETY_TOKENS; - return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, available)); + const input = estimateContextTokens(context).tokens; + const margin = Math.max(CONTEXT_SAFETY_TOKENS, Math.ceil(input * CONTEXT_SAFETY_RATIO)); + let room = model.contextWindow - input - margin; + if (room < MIN_OUTPUT_WITH_RATIO_MARGIN) { + room = Math.min(MIN_OUTPUT_WITH_RATIO_MARGIN, model.contextWindow - input - CONTEXT_SAFETY_TOKENS); + } + return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, room)); } /** Tokens always left for the answer when a thinking budget shares the response ceiling. */ diff --git a/packages/aikit/test/maxtokens.test.ts b/packages/aikit/test/maxtokens.test.ts index eed79819..057b7e41 100644 --- a/packages/aikit/test/maxtokens.test.ts +++ b/packages/aikit/test/maxtokens.test.ts @@ -1,9 +1,31 @@ import { describe, expect, it } from "vite-plus/test"; import { clampMaxTokensToContext } from "../src/llm/shared.ts"; -import type * as Message from "../src/message/message.ts"; -import { makeModel } from "./utils/fixtures.ts"; +import * as Message from "../src/message/message.ts"; +import { makeModel, makeUsage } from "./utils/fixtures.ts"; const emptyContext: Message.Context = { messages: [] }; +const WINDOW = 1_048_576; + +// The last assistant usage anchors the estimate, so the estimated input is exactly `inputTokens`. +function contextWithInput(inputTokens: number): Message.Context { + const model = makeModel(); + return { + messages: [ + Message.createUserMessage({ role: "user", parts: [{ type: "text", text: "hi" }], time: { created: 1 } }), + Message.createAssistantMessage({ + role: "assistant", + parts: [{ type: "text", text: "ok" }], + protocol: model.protocol, + provider: model.provider, + model: model.id, + thinkingLevel: "off", + usage: makeUsage({ input: inputTokens, totalTokens: inputTokens }), + stopReason: "stop", + time: { created: 2, completed: 2 }, + }), + ], + }; +} describe("clampMaxTokensToContext", () => { it("shrinks a ceiling that would not fit alongside the prompt", () => { @@ -21,4 +43,37 @@ describe("clampMaxTokensToContext", () => { const model = makeModel({ maxTokens: 8192, contextWindow: 0 }); expect(clampMaxTokensToContext(model, emptyContext, 8192)).toBe(8192); }); + + it("leaves 15% of the input unused so an OpenRouter overcount still fits", () => { + // The failing request: OpenRouter counted 105,153 input tokens where aikit estimated 100,762. + const model = makeModel({ maxTokens: WINDOW, contextWindow: WINDOW }); + const maxTokens = clampMaxTokensToContext(model, contextWithInput(100_762), WINDOW); + expect(maxTokens).toBe(WINDOW - 100_762 - 15_115); + expect(105_153 + maxTokens).toBeLessThanOrEqual(WINDOW); + }); + + it("leaves at least 4096 tokens unused for small inputs", () => { + const model = makeModel({ maxTokens: WINDOW, contextWindow: WINDOW }); + expect(clampMaxTokensToContext(model, contextWithInput(10_000), WINDOW)).toBe(WINDOW - 10_000 - 4096); + }); + + it("keeps the model output limit when it already fits", () => { + const model = makeModel({ maxTokens: 64_000, contextWindow: WINDOW }); + expect(clampMaxTokensToContext(model, contextWithInput(100_000), 64_000)).toBe(64_000); + }); + + it("sends up to 32768 tokens when the proportional margin leaves no room", () => { + const model = makeModel({ maxTokens: WINDOW, contextWindow: WINDOW }); + expect(clampMaxTokensToContext(model, contextWithInput(912_000), WINDOW)).toBe(32_768); + }); + + it("uses the real room minus 4096 when less than 32768 tokens fit", () => { + const model = makeModel({ maxTokens: WINDOW, contextWindow: WINDOW }); + expect(clampMaxTokensToContext(model, contextWithInput(1_030_000), WINDOW)).toBe(WINDOW - 1_030_000 - 4096); + }); + + it("keeps only the fixed 4096 margin for small windows", () => { + const model = makeModel({ maxTokens: 32_768, contextWindow: 32_768 }); + expect(clampMaxTokensToContext(model, contextWithInput(20_000), 32_768)).toBe(32_768 - 20_000 - 4096); + }); });