From c7e9555f9521b1a185107d6c176310bc8c9a491f Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sun, 1 Nov 2026 18:38:41 +0530 Subject: [PATCH] feat(aikit): retry non-strict when Claude's strict-tool limits are exceeded Claude rejects the whole request when strict tools exceed 20 tools or 16 optional/union-typed parameters. On that 400, retry once with `prefer` tools sent non-strict; `require` tools stay strict. The retry is recorded as a `strictToolsFallback` diagnostic on the assistant message. Co-Authored-By: Claude Opus 5.5 --- packages/aikit/src/llm/failure.ts | 26 ++++ packages/aikit/src/llm/stream.ts | 121 ++++++++++----- packages/aikit/src/llm/strict.ts | 4 +- packages/aikit/src/llm/transform.ts | 9 +- packages/aikit/src/message/message.ts | 17 ++ .../e2e/__artifacts__/strict-tool-limit.json | 84 ++++++++++ .../aikit/test/e2e/strict.limit.e2e.test.ts | 145 ++++++++++++++++++ packages/aikit/test/utils/anthropic.ts | 13 +- 8 files changed, 375 insertions(+), 44 deletions(-) create mode 100644 packages/aikit/test/e2e/__artifacts__/strict-tool-limit.json create mode 100644 packages/aikit/test/e2e/strict.limit.e2e.test.ts diff --git a/packages/aikit/src/llm/failure.ts b/packages/aikit/src/llm/failure.ts index ebc74fb5..ba6a33ec 100644 --- a/packages/aikit/src/llm/failure.ts +++ b/packages/aikit/src/llm/failure.ts @@ -228,6 +228,32 @@ export function normalize(error: unknown): Failure { return { _tag: "Unknown", message: errorMessage(error), retryable: false }; } +/** + * Claude rejects a request whose strict tools exceed its limits with one of these messages. Observed limits: + * 20 strict tools, and 16 optional or union-typed parameters across all strict tools. + */ +const STRICT_TOOL_LIMIT_MESSAGES = [ + "The compiled grammar is too large", + "Too many strict tools (", + "Schemas contains too many parameters with union types (", +]; + +/** The provider's own message when it rejected the request for exceeding its strict-tool limits. */ +export function strictToolLimitMessage(error: unknown): string | undefined { + if (RetryError.isInstance(error)) return strictToolLimitMessage(error.lastError); + if (!APICallError.isInstance(error) || error.statusCode !== 400) return; + const data = record(error.data) ?? record(parseJson(error.responseBody)); + // Anthropic's body is `{ type: "error", error: { type, message } }`, so the top-level type is not the code. + const nested = record(data?.error); + if (nested?.type !== "invalid_request_error") return; + const message = nonEmptyString(nested.message) ?? error.message; + return STRICT_TOOL_LIMIT_MESSAGES.some((limit) => message.includes(limit)) ? message : undefined; +} + +export function isStrictToolLimitError(error: unknown): boolean { + return strictToolLimitMessage(error) !== undefined; +} + /** Best-effort compatibility for terminal messages written before structured failures existed. */ export function fromMessage(message: string): Failure { const normalized = normalizeMessage(message); diff --git a/packages/aikit/src/llm/stream.ts b/packages/aikit/src/llm/stream.ts index 48d9d305..38d68c1f 100644 --- a/packages/aikit/src/llm/stream.ts +++ b/packages/aikit/src/llm/stream.ts @@ -9,12 +9,14 @@ import { Options, type ProviderOptionBag, type RuntimeOptions } from "./options. import * as Pricing from "./pricing.ts"; import * as Protocol from "./protocol.ts"; import { resolveAISDKLanguageModel } from "./provider.ts"; +import { resolveMethod } from "./registry.ts"; import { formatThrownError } from "./runtime.ts"; import { applyDefaultMaxTokens } from "./shared.ts"; import * as Thinking from "./thinking.ts"; import { googleThoughtSignature, convertMessages, + isAnthropicWire, isOwnResponses, convertTools, requestTools, @@ -417,7 +419,6 @@ export const stream: Protocol.StreamFunction @@ -429,45 +430,91 @@ export const stream: Protocol.StreamFunction !tools || name in tools); - - let params: Parameters>[0] = compact({ - model: languageModel, - // Our own Responses transport receives the system prompt among the messages. - system: isOwnResponses(model, runtimeOptions.method) - ? undefined - : leadingSystemPrompt(context, model, runtimeOptions.method), - messages, - // Prompt and tool changes sent in place travel as system messages. - allowSystemInMessages: messages.some((message) => message.role === "system") || undefined, - tools, - toolChoice: runtimeOptions.toolChoice as Parameters>[0]["toolChoice"], - activeTools, - maxOutputTokens: resolveMaxOutputTokens(model, plan), - temperature: model.compat?.supportsTemperature === false ? undefined : runtimeOptions.temperature, - providerOptions, - // The exact body is where `providerThinkingLevel` is read from. - include: { requestBody: true }, - abortSignal: runtimeOptions.signal, - timeout: resolveTimeout(runtimeOptions), - maxRetries: runtimeOptions.maxRetries, - headers: runtimeOptions.headers, - // Aikit owns error normalization and emits one terminal error event. - // AI SDK's default callback logs the raw exception (including request - // payloads) before that event can reach an SDK or CLI consumer. - onError: () => {}, - }); - const payload = await runtimeOptions.onPayload?.(params, model); - if (payload !== undefined) { - params = payload as Parameters>[0]; - } + const buildParams = async (preferNonStrict: boolean) => { + const tools = convertTools( + declared.tools, + model, + runtimeOptions.method, + declared.deferred, + preferNonStrict, + ); + const sentPreferStrict = + declared.tools?.some( + (tool) => + tool.constrainedSampling && + tool.constrainedSampling.type === "json_schema" && + tool.constrainedSampling.strict === "prefer" && + tools?.[tool.name]?.strict === true, + ) ?? false; + const activeTools = runtimeOptions.activeTools?.filter((name) => !tools || name in tools); + + let params: Parameters>[0] = compact({ + model: languageModel, + // Our own Responses transport receives the system prompt among the messages. + system: isOwnResponses(model, runtimeOptions.method) + ? undefined + : leadingSystemPrompt(context, model, runtimeOptions.method), + messages, + // Prompt and tool changes sent in place travel as system messages. + allowSystemInMessages: messages.some((message) => message.role === "system") || undefined, + tools, + toolChoice: runtimeOptions.toolChoice as Parameters>[0]["toolChoice"], + activeTools, + maxOutputTokens: resolveMaxOutputTokens(model, plan), + temperature: model.compat?.supportsTemperature === false ? undefined : runtimeOptions.temperature, + providerOptions, + // The exact body is where `providerThinkingLevel` is read from. + include: { requestBody: true }, + abortSignal: runtimeOptions.signal, + timeout: resolveTimeout(runtimeOptions), + maxRetries: runtimeOptions.maxRetries, + headers: runtimeOptions.headers, + // Aikit owns error normalization and emits one terminal error event. + // AI SDK's default callback logs the raw exception (including request + // payloads) before that event can reach an SDK or CLI consumer. + onError: () => {}, + }); + + const payload = await runtimeOptions.onPayload?.(params, model); + if (payload !== undefined) { + params = payload as Parameters>[0]; + } + return { params, sentPreferStrict }; + }; - const result = streamText(params); + let preferNonStrict = false; + let built = await buildParams(preferNonStrict); stream.push({ type: "start", partial: output }); - - for await (const part of result.fullStream) { - handlePart(part, output, model, stream, pricing); + for (;;) { + try { + for await (const part of streamText(built.params).fullStream) { + handlePart(part, output, model, stream, pricing); + } + break; + } catch (error) { + // Claude refuses the whole request when strict tools exceed its limits, and `prefer` tools + // can go non-strict instead. Only a request refused before anything streamed is retried, once. + const limitMessage = Failure.strictToolLimitMessage(error); + if ( + preferNonStrict || + runtimeOptions.signal?.aborted || + output.parts.length > 0 || + !built.sentPreferStrict || + !isAnthropicWire(model, resolveMethod(model, runtimeOptions.method)) || + limitMessage === undefined + ) { + throw error; + } + (output.diagnostics ??= []).push({ + type: "strictToolsFallback", + timestamp: Date.now(), + error: { message: limitMessage }, + details: { reason: "strictToolLimits" }, + }); + preferNonStrict = true; + built = await buildParams(preferNonStrict); + } } if (runtimeOptions.signal?.aborted || output.stopReason === "aborted") { diff --git a/packages/aikit/src/llm/strict.ts b/packages/aikit/src/llm/strict.ts index 7128805f..b79f7016 100644 --- a/packages/aikit/src/llm/strict.ts +++ b/packages/aikit/src/llm/strict.ts @@ -198,14 +198,16 @@ function makeStrictJsonSchema(schema: unknown, isUnsupportedKeyword?: Unsupporte /** * The strict parameter schema to send for a JSON-schema constrained tool, or `undefined` to send it non-strict. * A `"prefer"` tool whose schema the model's strict mode cannot express falls back to non-strict; a `"require"` - * tool throws instead. + * tool throws instead. `preferNonStrict` sends every `"prefer"` tool non-strict. */ export function resolveStrictParameters( tool: Message.Tool, model?: Pick, + preferNonStrict = false, ): Record | undefined { const config = tool.constrainedSampling; if (!config || config.type !== "json_schema") return; + if (preferNonStrict && config.strict === "prefer") return; if (model?.compat?.supportsStrictMode ?? true) { const isAnthropic = diff --git a/packages/aikit/src/llm/transform.ts b/packages/aikit/src/llm/transform.ts index 167b3937..586fd148 100644 --- a/packages/aikit/src/llm/transform.ts +++ b/packages/aikit/src/llm/transform.ts @@ -102,7 +102,7 @@ type MovedImages = "anthropicError" | "chat" | "gemini"; const COPILOT = "@codeworksh/ai-sdk-github-copilot"; -function isAnthropicWire(model: Model.Info, method: Model.APIMethodEnum): boolean { +export function isAnthropicWire(model: Model.Info, method: Model.APIMethodEnum): boolean { if (packageForModel(model) === COPILOT) return method === Model.APIMethodEnum.messages; return isAnthropicMessagesModel(model); } @@ -698,8 +698,8 @@ export function grammarToolInputProperties( } /** The parameter schema, strict flag and grammar a tool goes out with on this model. */ -function toolWire(tool: Message.Tool, model?: Model.Info, method?: Model.APIMethodEnum) { - const strictParameters = resolveStrictParameters(tool, model); +function toolWire(tool: Message.Tool, model?: Model.Info, method?: Model.APIMethodEnum, preferNonStrict = false) { + const strictParameters = resolveStrictParameters(tool, model, preferNonStrict); const grammarConstraint = model && isOwnResponses(model, method) ? resolveToolGrammar(tool, model.compat) : undefined; return { @@ -715,12 +715,13 @@ export function convertTools( model?: Model.Info, method?: Model.APIMethodEnum, deferred: ReadonlySet = new Set(), + preferNonStrict = false, ): ToolSet | undefined { if (!tools || tools.length === 0) return; const result: ToolSet = {}; for (const tool of tools) { - const { inputSchema, strict, providerOptions } = toolWire(tool, model, method); + const { inputSchema, strict, providerOptions } = toolWire(tool, model, method, preferNonStrict); const key = model && deferred.has(tool.name) ? Model.optionsKey(model) : undefined; const options = key === undefined diff --git a/packages/aikit/src/message/message.ts b/packages/aikit/src/message/message.ts index 87f99629..ffe2e8e6 100644 --- a/packages/aikit/src/message/message.ts +++ b/packages/aikit/src/message/message.ts @@ -167,6 +167,22 @@ export const UserMessageSchema = Type.Object({ }); export type UserMessage = Static; +/** Something aikit did on the caller's behalf while producing a message, such as a request retried in another shape. */ +export const AssistantMessageDiagnosticSchema = Type.Object({ + type: Type.String(), + timestamp: Type.Number(), + error: Type.Optional( + Type.Object({ + name: Type.Optional(Type.String()), + message: Type.String(), + stack: Type.Optional(Type.String()), + code: Type.Optional(Type.Union([Type.String(), Type.Number()])), + }), + ), + details: Type.Optional(Type.Record(Type.String(), Type.Unknown())), +}); +export type AssistantMessageDiagnostic = Static; + export const AssistantMessageSchema = Type.Object({ role: Type.Literal("assistant"), protocol: Model.KnownProviderEnumSchema, @@ -185,6 +201,7 @@ export const AssistantMessageSchema = Type.Object({ * Depends on ai-sdk */ failure: Type.Optional(Failure.FailureSchema), + diagnostics: Type.Optional(Type.Array(AssistantMessageDiagnosticSchema)), time: Type.Object({ created: Type.Number(), completed: Type.Number(), diff --git a/packages/aikit/test/e2e/__artifacts__/strict-tool-limit.json b/packages/aikit/test/e2e/__artifacts__/strict-tool-limit.json new file mode 100644 index 00000000..20fa035f --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/strict-tool-limit.json @@ -0,0 +1,84 @@ +{ + "toolCount": { + "attempts": [ + { + "count_tool_1": true, + "count_tool_2": true, + "count_tool_3": true, + "count_tool_4": true, + "count_tool_5": true, + "count_tool_6": true, + "count_tool_7": true, + "count_tool_8": true, + "count_tool_9": true, + "count_tool_10": true, + "count_tool_11": true, + "count_tool_12": true, + "count_tool_13": true, + "count_tool_14": true, + "count_tool_15": true, + "count_tool_16": true, + "count_tool_17": true, + "count_tool_18": true, + "count_tool_19": true, + "count_tool_20": true, + "count_tool_21": true, + "required_tool": true + }, + { + "count_tool_1": false, + "count_tool_2": false, + "count_tool_3": false, + "count_tool_4": false, + "count_tool_5": false, + "count_tool_6": false, + "count_tool_7": false, + "count_tool_8": false, + "count_tool_9": false, + "count_tool_10": false, + "count_tool_11": false, + "count_tool_12": false, + "count_tool_13": false, + "count_tool_14": false, + "count_tool_15": false, + "count_tool_16": false, + "count_tool_17": false, + "count_tool_18": false, + "count_tool_19": false, + "count_tool_20": false, + "count_tool_21": false, + "required_tool": true + } + ], + "stopReason": "stop", + "diagnostics": [ + { + "type": "strictToolsFallback", + "reason": "strictToolLimits" + } + ] + }, + "unionCount": { + "attempts": [ + { + "required_tool": true, + "union_tool_1": true, + "union_tool_2": true, + "union_tool_3": true + }, + { + "required_tool": true, + "union_tool_1": false, + "union_tool_2": false, + "union_tool_3": false + } + ], + "stopReason": "stop", + "diagnostics": [ + { + "type": "strictToolsFallback", + "reason": "strictToolLimits" + } + ] + } +} \ No newline at end of file diff --git a/packages/aikit/test/e2e/strict.limit.e2e.test.ts b/packages/aikit/test/e2e/strict.limit.e2e.test.ts new file mode 100644 index 00000000..b7c86dd3 --- /dev/null +++ b/packages/aikit/test/e2e/strict.limit.e2e.test.ts @@ -0,0 +1,145 @@ +import Type, { type TSchema } from "typebox"; +import { describe, expect, it } from "vite-plus/test"; +import * as Message from "../../src/message/message.ts"; +import { complete } from "../../src/stream.ts"; +import { fakeAnthropic, textReply } from "../utils/anthropic.ts"; +import { + ANTHROPIC_E2E_MODELS, + anthropicOptions, + describeIfAnthropic, + getAnthropicModel, + type StreamableModel, +} from "../utils/llm.ts"; + +const preferTool = (name: string, parameters: TSchema): Message.Tool => ({ + name, + description: `Synthetic tool ${name}. Never call it.`, + parameters, + constrainedSampling: { type: "json_schema", strict: "prefer" }, +}); + +// Fits either limit on its own, so it stays strict once the `prefer` tools go non-strict. +const requireTool: Message.Tool = { + name: "required_tool", + description: "Synthetic tool that must stay strict. Never call it.", + parameters: Type.Object({ value: Type.String() }), + constrainedSampling: { type: "json_schema", strict: "require" }, +}; + +/** 21 `prefer` tools plus the `require` one: over the strict-tool count limit. */ +const toolCount = [ + ...Array.from({ length: 21 }, (_, index) => + preferTool(`count_tool_${index + 1}`, Type.Object({ value: Type.String() })), + ), + requireTool, +]; + +const optionalParams = (count: number) => + Type.Object( + Object.fromEntries( + Array.from({ length: count }, (_, index) => [`field_${index + 1}`, Type.Optional(Type.String())]), + ), + ); + +/** 17 optional parameters across three tools: one over the union-typed parameter limit. */ +const unionCount = [ + preferTool("union_tool_1", optionalParams(6)), + preferTool("union_tool_2", optionalParams(6)), + preferTool("union_tool_3", optionalParams(5)), + requireTool, +]; + +/** Claude's own rejections, which the recorded variant replays and the live one checks still match. */ +const CASES = { + toolCount: { + tools: toolCount, + message: + "Too many strict tools (22). The maximum number of strict tools supported is 20. Try reducing the number of tools marked as strict.", + }, + unionCount: { + tools: unionCount, + message: + "Schemas contains too many parameters with union types (17 parameters with type arrays or anyOf). This causes exponential compilation cost. Reduce the number of nullable or union-typed parameters (limit: 16 parameters with unions).", + }, +} as const; + +type Case = keyof typeof CASES; + +/** Records each request body, then hands it to `inner`. */ +function recordRequests(inner: typeof fetch) { + const bodies: Array<{ tools?: Array<{ name: string; strict?: boolean }> }> = []; + const record: typeof fetch = (input, init) => { + if (typeof init?.body === "string") bodies.push(JSON.parse(init.body)); + return inner(input, init); + }; + return { bodies, fetch: record }; +} + +async function run(model: StreamableModel, options: object, name: Case, fetch: typeof globalThis.fetch) { + const requests = recordRequests(fetch); + const message = await complete( + model, + { + systemPrompt: "Reply with the single word hi. Do not call any tool.", + messages: [ + Message.createUserMessage({ + role: "user", + parts: [{ type: "text", text: "Say hi." }], + time: { created: Date.now() }, + }), + ], + tools: CASES[name].tools, + }, + { ...options, maxRetries: 0, factoryOptions: { fetch: requests.fetch } } as never, + ); + expect(message.stopReason, message.errorMessage).toBe("stop"); + return { + message, + artifact: { + attempts: requests.bodies.map((body) => + Object.fromEntries( + (body.tools ?? []) + .map((tool) => [tool.name, tool.strict === true] as const) + .sort(([a], [b]) => a.localeCompare(b, "en", { numeric: true })), + ), + ), + stopReason: message.stopReason, + diagnostics: (message.diagnostics ?? []).map((diagnostic) => ({ + type: diagnostic.type, + reason: diagnostic.details?.reason, + })), + }, + }; +} + +async function snapshot(artifacts: Record) { + await expect(JSON.stringify(artifacts, null, 2)).toMatchFileSnapshot("./__artifacts__/strict-tool-limit.json"); +} + +describe("Strict tools over Claude's limits", () => { + it("recorded: retries once with prefer tools non-strict", async () => { + // A catalog Claude model: the SDK drops `strict` for model ids it does not know support it. + const model = await getAnthropicModel(ANTHROPIC_E2E_MODELS[0]); + const artifacts = {} as Record; + for (const name of Object.keys(CASES) as Case[]) { + const server = fakeAnthropic(textReply, { rejectFirst: CASES[name].message }); + const { message, artifact } = await run(model, { apiKey: "test-key" }, name, server.fetch); + expect(message.diagnostics?.[0]?.error?.message).toBe(CASES[name].message); + artifacts[name] = artifact; + } + await snapshot(artifacts); + }); + + describeIfAnthropic("Anthropic provider", () => { + it(`live ${ANTHROPIC_E2E_MODELS[0]}: Claude rejects, then the retry succeeds`, { timeout: 120_000 }, async () => { + const model = await getAnthropicModel(ANTHROPIC_E2E_MODELS[0]); + const artifacts = {} as Record; + for (const name of Object.keys(CASES) as Case[]) { + const { message, artifact } = await run(model, anthropicOptions({ maxTokens: 256 }), name, fetch); + expect(message.diagnostics?.[0]?.error?.message).toBe(CASES[name].message); + artifacts[name] = artifact; + } + await snapshot(artifacts); + }); + }); +}); diff --git a/packages/aikit/test/utils/anthropic.ts b/packages/aikit/test/utils/anthropic.ts index a7142e1a..4ee8ea37 100644 --- a/packages/aikit/test/utils/anthropic.ts +++ b/packages/aikit/test/utils/anthropic.ts @@ -29,13 +29,22 @@ export interface FakeAnthropic { /** * Stream `events`, then close -- or, with `stall`, keep the connection open without - * sending anything more until the SDK aborts it. + * sending anything more until the SDK aborts it. With `rejectFirst`, the first request + * is refused with a 400 `invalid_request_error` carrying that message. */ -export function fakeAnthropic(events: ReadonlyArray, options: { stall?: boolean } = {}): FakeAnthropic { +export function fakeAnthropic( + events: ReadonlyArray, + options: { stall?: boolean; rejectFirst?: string } = {}, +): FakeAnthropic { let body: unknown; + let requests = 0; const fetch: typeof globalThis.fetch = async (_input, init) => { if (typeof init?.body !== "string") throw new Error("Expected a JSON request body"); body = JSON.parse(init.body); + if (requests++ === 0 && options.rejectFirst !== undefined) { + const error = { type: "error", error: { type: "invalid_request_error", message: options.rejectFirst } }; + return new Response(JSON.stringify(error), { status: 400, headers: { "content-type": "application/json" } }); + } const stream = new ReadableStream({ start(controller) { for (const event of events) controller.enqueue(sse(event));