Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions packages/aikit/src/llm/shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@ export type SharedOptions = Static<typeof SharedOptions>;

/** Headroom left for the prompt when sizing a response against the context window. */
const CONTEXT_SAFETY_TOKENS = 4096;
// OpenRouter counts the input with its own tokenizer and rejects input + max_tokens above the
// window before routing. It counted prose up to 12% above the provider's count.
const CONTEXT_SAFETY_RATIO = 0.15;
// The 15% margin is kept only while it leaves at least this much output. Otherwise (a nearly full
// large window, or any small window such as a 32k local model) only the fixed margin is kept, and
// output is capped at this size so an OpenRouter overcount still fits for a while before compaction.
const MIN_OUTPUT_WITH_RATIO_MARGIN = 32768;
// The smallest ceiling OpenAI accepts (`max_output_tokens` >= 16). Below it an
// overflowing prompt fails as an invalid parameter instead of a context overflow.
const MIN_MAX_TOKENS = 16;
Expand All @@ -60,16 +67,22 @@ export function applyDefaultMaxTokens<TOptions extends SharedOptions = SharedOpt
* Shrink a response ceiling to what is left of the context window.
*
* Without this a long session asks for an answer that cannot fit and takes a
* provider overflow error instead of a shorter answer.
* provider overflow error instead of a shorter answer. The margin leaves room
* for the input being counted higher than estimated.
*/
export function clampMaxTokensToContext(
model: Model.TModel<Model.KnownProviderEnum>,
context: Message.Context,
maxTokens: number,
): number {
if (model.contextWindow <= 0) return Math.max(MIN_MAX_TOKENS, maxTokens);
const available = model.contextWindow - estimateContextTokens(context).tokens - CONTEXT_SAFETY_TOKENS;
return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, available));
const input = estimateContextTokens(context).tokens;
const margin = Math.max(CONTEXT_SAFETY_TOKENS, Math.ceil(input * CONTEXT_SAFETY_RATIO));
let room = model.contextWindow - input - margin;
if (room < MIN_OUTPUT_WITH_RATIO_MARGIN) {
room = Math.min(MIN_OUTPUT_WITH_RATIO_MARGIN, model.contextWindow - input - CONTEXT_SAFETY_TOKENS);
}
return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, room));
}

/** Tokens always left for the answer when a thinking budget shares the response ceiling. */
Expand Down
59 changes: 57 additions & 2 deletions packages/aikit/test/maxtokens.test.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,31 @@
import { describe, expect, it } from "vite-plus/test";
import { clampMaxTokensToContext } from "../src/llm/shared.ts";
import type * as Message from "../src/message/message.ts";
import { makeModel } from "./utils/fixtures.ts";
import * as Message from "../src/message/message.ts";
import { makeModel, makeUsage } from "./utils/fixtures.ts";

const emptyContext: Message.Context = { messages: [] };
const WINDOW = 1_048_576;

// The last assistant usage anchors the estimate, so the estimated input is exactly `inputTokens`.
function contextWithInput(inputTokens: number): Message.Context {
const model = makeModel();
return {
messages: [
Message.createUserMessage({ role: "user", parts: [{ type: "text", text: "hi" }], time: { created: 1 } }),
Message.createAssistantMessage({
role: "assistant",
parts: [{ type: "text", text: "ok" }],
protocol: model.protocol,
provider: model.provider,
model: model.id,
thinkingLevel: "off",
usage: makeUsage({ input: inputTokens, totalTokens: inputTokens }),
stopReason: "stop",
time: { created: 2, completed: 2 },
}),
],
};
}

describe("clampMaxTokensToContext", () => {
it("shrinks a ceiling that would not fit alongside the prompt", () => {
Expand All @@ -21,4 +43,37 @@ describe("clampMaxTokensToContext", () => {
const model = makeModel({ maxTokens: 8192, contextWindow: 0 });
expect(clampMaxTokensToContext(model, emptyContext, 8192)).toBe(8192);
});

it("leaves 15% of the input unused so an OpenRouter overcount still fits", () => {
// The failing request: OpenRouter counted 105,153 input tokens where aikit estimated 100,762.
const model = makeModel({ maxTokens: WINDOW, contextWindow: WINDOW });
const maxTokens = clampMaxTokensToContext(model, contextWithInput(100_762), WINDOW);
expect(maxTokens).toBe(WINDOW - 100_762 - 15_115);
expect(105_153 + maxTokens).toBeLessThanOrEqual(WINDOW);
});

it("leaves at least 4096 tokens unused for small inputs", () => {
const model = makeModel({ maxTokens: WINDOW, contextWindow: WINDOW });
expect(clampMaxTokensToContext(model, contextWithInput(10_000), WINDOW)).toBe(WINDOW - 10_000 - 4096);
});

it("keeps the model output limit when it already fits", () => {
const model = makeModel({ maxTokens: 64_000, contextWindow: WINDOW });
expect(clampMaxTokensToContext(model, contextWithInput(100_000), 64_000)).toBe(64_000);
});

it("sends up to 32768 tokens when the proportional margin leaves no room", () => {
const model = makeModel({ maxTokens: WINDOW, contextWindow: WINDOW });
expect(clampMaxTokensToContext(model, contextWithInput(912_000), WINDOW)).toBe(32_768);
});

it("uses the real room minus 4096 when less than 32768 tokens fit", () => {
const model = makeModel({ maxTokens: WINDOW, contextWindow: WINDOW });
expect(clampMaxTokensToContext(model, contextWithInput(1_030_000), WINDOW)).toBe(WINDOW - 1_030_000 - 4096);
});

it("keeps only the fixed 4096 margin for small windows", () => {
const model = makeModel({ maxTokens: 32_768, contextWindow: 32_768 });
expect(clampMaxTokensToContext(model, contextWithInput(20_000), 32_768)).toBe(32_768 - 20_000 - 4096);
});
});
Loading