From 553df89943b94e3408c1f8b957e9822ca4349269 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Wed, 4 Nov 2026 18:13:35 +0530 Subject: [PATCH] fix(aikit): leave room for OpenRouter's input overcount when clamping max tokens OpenRouter checks input + max_tokens against the context window before routing, using its own tokenizer, which can count several percent more input than our estimate. With only 4096 tokens spare, models whose output limit spans the window were rejected with a 400 ("This endpoint's maximum context length is ...") at around 10% of the window. The output ceiling now leaves 15% of the estimated input unused (at least 4096). When that would leave less than 32768 tokens of output, only the fixed 4096 margin is kept and output is capped at 32768. Co-Authored-By: Claude Opus 5.5 --- packages/aikit/src/llm/shared.ts | 19 +++++++-- packages/aikit/test/maxtokens.test.ts | 59 ++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/packages/aikit/src/llm/shared.ts b/packages/aikit/src/llm/shared.ts index d5629bb8..f9f0631b 100644 --- a/packages/aikit/src/llm/shared.ts +++ b/packages/aikit/src/llm/shared.ts @@ -35,6 +35,13 @@ export type SharedOptions = Static; /** Headroom left for the prompt when sizing a response against the context window. */ const CONTEXT_SAFETY_TOKENS = 4096; +// OpenRouter counts the input with its own tokenizer and rejects input + max_tokens above the +// window before routing. It counted prose up to 12% above the provider's count. +const CONTEXT_SAFETY_RATIO = 0.15; +// The 15% margin is kept only while it leaves at least this much output. Otherwise (a nearly full +// large window, or any small window such as a 32k local model) only the fixed margin is kept, and +// output is capped at this size so an OpenRouter overcount still fits for a while before compaction. +const MIN_OUTPUT_WITH_RATIO_MARGIN = 32768; // The smallest ceiling OpenAI accepts (`max_output_tokens` >= 16). Below it an // overflowing prompt fails as an invalid parameter instead of a context overflow. const MIN_MAX_TOKENS = 16; @@ -60,7 +67,8 @@ export function applyDefaultMaxTokens, @@ -68,8 +76,13 @@ export function clampMaxTokensToContext( maxTokens: number, ): number { if (model.contextWindow <= 0) return Math.max(MIN_MAX_TOKENS, maxTokens); - const available = model.contextWindow - estimateContextTokens(context).tokens - CONTEXT_SAFETY_TOKENS; - return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, available)); + const input = estimateContextTokens(context).tokens; + const margin = Math.max(CONTEXT_SAFETY_TOKENS, Math.ceil(input * CONTEXT_SAFETY_RATIO)); + let room = model.contextWindow - input - margin; + if (room < MIN_OUTPUT_WITH_RATIO_MARGIN) { + room = Math.min(MIN_OUTPUT_WITH_RATIO_MARGIN, model.contextWindow - input - CONTEXT_SAFETY_TOKENS); + } + return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, room)); } /** Tokens always left for the answer when a thinking budget shares the response ceiling. */ diff --git a/packages/aikit/test/maxtokens.test.ts b/packages/aikit/test/maxtokens.test.ts index eed79819..057b7e41 100644 --- a/packages/aikit/test/maxtokens.test.ts +++ b/packages/aikit/test/maxtokens.test.ts @@ -1,9 +1,31 @@ import { describe, expect, it } from "vite-plus/test"; import { clampMaxTokensToContext } from "../src/llm/shared.ts"; -import type * as Message from "../src/message/message.ts"; -import { makeModel } from "./utils/fixtures.ts"; +import * as Message from "../src/message/message.ts"; +import { makeModel, makeUsage } from "./utils/fixtures.ts"; const emptyContext: Message.Context = { messages: [] }; +const WINDOW = 1_048_576; + +// The last assistant usage anchors the estimate, so the estimated input is exactly `inputTokens`. +function contextWithInput(inputTokens: number): Message.Context { + const model = makeModel(); + return { + messages: [ + Message.createUserMessage({ role: "user", parts: [{ type: "text", text: "hi" }], time: { created: 1 } }), + Message.createAssistantMessage({ + role: "assistant", + parts: [{ type: "text", text: "ok" }], + protocol: model.protocol, + provider: model.provider, + model: model.id, + thinkingLevel: "off", + usage: makeUsage({ input: inputTokens, totalTokens: inputTokens }), + stopReason: "stop", + time: { created: 2, completed: 2 }, + }), + ], + }; +} describe("clampMaxTokensToContext", () => { it("shrinks a ceiling that would not fit alongside the prompt", () => { @@ -21,4 +43,37 @@ describe("clampMaxTokensToContext", () => { const model = makeModel({ maxTokens: 8192, contextWindow: 0 }); expect(clampMaxTokensToContext(model, emptyContext, 8192)).toBe(8192); }); + + it("leaves 15% of the input unused so an OpenRouter overcount still fits", () => { + // The failing request: OpenRouter counted 105,153 input tokens where aikit estimated 100,762. + const model = makeModel({ maxTokens: WINDOW, contextWindow: WINDOW }); + const maxTokens = clampMaxTokensToContext(model, contextWithInput(100_762), WINDOW); + expect(maxTokens).toBe(WINDOW - 100_762 - 15_115); + expect(105_153 + maxTokens).toBeLessThanOrEqual(WINDOW); + }); + + it("leaves at least 4096 tokens unused for small inputs", () => { + const model = makeModel({ maxTokens: WINDOW, contextWindow: WINDOW }); + expect(clampMaxTokensToContext(model, contextWithInput(10_000), WINDOW)).toBe(WINDOW - 10_000 - 4096); + }); + + it("keeps the model output limit when it already fits", () => { + const model = makeModel({ maxTokens: 64_000, contextWindow: WINDOW }); + expect(clampMaxTokensToContext(model, contextWithInput(100_000), 64_000)).toBe(64_000); + }); + + it("sends up to 32768 tokens when the proportional margin leaves no room", () => { + const model = makeModel({ maxTokens: WINDOW, contextWindow: WINDOW }); + expect(clampMaxTokensToContext(model, contextWithInput(912_000), WINDOW)).toBe(32_768); + }); + + it("uses the real room minus 4096 when less than 32768 tokens fit", () => { + const model = makeModel({ maxTokens: WINDOW, contextWindow: WINDOW }); + expect(clampMaxTokensToContext(model, contextWithInput(1_030_000), WINDOW)).toBe(WINDOW - 1_030_000 - 4096); + }); + + it("keeps only the fixed 4096 margin for small windows", () => { + const model = makeModel({ maxTokens: 32_768, contextWindow: 32_768 }); + expect(clampMaxTokensToContext(model, contextWithInput(20_000), 32_768)).toBe(32_768 - 20_000 - 4096); + }); });