Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions packages/aikit/src/llm/failure.ts
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,32 @@ export function normalize(error: unknown): Failure {
return { _tag: "Unknown", message: errorMessage(error), retryable: false };
}

/**
* Claude rejects a request whose strict tools exceed its limits with one of these messages. Observed limits:
* 20 strict tools, and 16 optional or union-typed parameters across all strict tools.
*/
const STRICT_TOOL_LIMIT_MESSAGES = [
"The compiled grammar is too large",
"Too many strict tools (",
"Schemas contains too many parameters with union types (",
];

/** The provider's own message when it rejected the request for exceeding its strict-tool limits. */
export function strictToolLimitMessage(error: unknown): string | undefined {
if (RetryError.isInstance(error)) return strictToolLimitMessage(error.lastError);
if (!APICallError.isInstance(error) || error.statusCode !== 400) return;
const data = record(error.data) ?? record(parseJson(error.responseBody));
// Anthropic's body is `{ type: "error", error: { type, message } }`, so the top-level type is not the code.
const nested = record(data?.error);
if (nested?.type !== "invalid_request_error") return;
const message = nonEmptyString(nested.message) ?? error.message;
return STRICT_TOOL_LIMIT_MESSAGES.some((limit) => message.includes(limit)) ? message : undefined;
}

export function isStrictToolLimitError(error: unknown): boolean {
return strictToolLimitMessage(error) !== undefined;
}

/** Best-effort compatibility for terminal messages written before structured failures existed. */
export function fromMessage(message: string): Failure {
const normalized = normalizeMessage(message);
Expand Down
121 changes: 84 additions & 37 deletions packages/aikit/src/llm/stream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ import { Options, type ProviderOptionBag, type RuntimeOptions } from "./options.
import * as Pricing from "./pricing.ts";
import * as Protocol from "./protocol.ts";
import { resolveAISDKLanguageModel } from "./provider.ts";
import { resolveMethod } from "./registry.ts";
import { formatThrownError } from "./runtime.ts";
import { applyDefaultMaxTokens } from "./shared.ts";
import * as Thinking from "./thinking.ts";
import {
googleThoughtSignature,
convertMessages,
isAnthropicWire,
isOwnResponses,
convertTools,
requestTools,
Expand Down Expand Up @@ -417,7 +419,6 @@ export const stream: Protocol.StreamFunction<Model.KnownProviderEnum, typeof Opt
const languageModel = await resolveAISDKLanguageModel(model, runtimeOptions, plan.budget);
// Every tool callable now, so calls to tools declared in place still parse.
const declared = requestTools(context, model, runtimeOptions.method, runtimeOptions.activeTools);
const tools = convertTools(declared.tools, model, runtimeOptions.method, declared.deferred);
const messages = convertMessages(context, model, runtimeOptions.method, runtimeOptions.activeTools);
const providerOptions = resolveProviderOptions(model, context, runtimeOptions, plan) as Parameters<
typeof streamText<ToolSet>
Expand All @@ -429,45 +430,91 @@ export const stream: Protocol.StreamFunction<Model.KnownProviderEnum, typeof Opt
providerOptions as ProviderOptionBag,
),
};
const activeTools = runtimeOptions.activeTools?.filter((name) => !tools || name in tools);

let params: Parameters<typeof streamText<ToolSet>>[0] = compact({
model: languageModel,
// Our own Responses transport receives the system prompt among the messages.
system: isOwnResponses(model, runtimeOptions.method)
? undefined
: leadingSystemPrompt(context, model, runtimeOptions.method),
messages,
// Prompt and tool changes sent in place travel as system messages.
allowSystemInMessages: messages.some((message) => message.role === "system") || undefined,
tools,
toolChoice: runtimeOptions.toolChoice as Parameters<typeof streamText<ToolSet>>[0]["toolChoice"],
activeTools,
maxOutputTokens: resolveMaxOutputTokens(model, plan),
temperature: model.compat?.supportsTemperature === false ? undefined : runtimeOptions.temperature,
providerOptions,
// The exact body is where `providerThinkingLevel` is read from.
include: { requestBody: true },
abortSignal: runtimeOptions.signal,
timeout: resolveTimeout(runtimeOptions),
maxRetries: runtimeOptions.maxRetries,
headers: runtimeOptions.headers,
// Aikit owns error normalization and emits one terminal error event.
// AI SDK's default callback logs the raw exception (including request
// payloads) before that event can reach an SDK or CLI consumer.
onError: () => {},
});

const payload = await runtimeOptions.onPayload?.(params, model);
if (payload !== undefined) {
params = payload as Parameters<typeof streamText<ToolSet>>[0];
}
const buildParams = async (preferNonStrict: boolean) => {
const tools = convertTools(
declared.tools,
model,
runtimeOptions.method,
declared.deferred,
preferNonStrict,
);
const sentPreferStrict =
declared.tools?.some(
(tool) =>
tool.constrainedSampling &&
tool.constrainedSampling.type === "json_schema" &&
tool.constrainedSampling.strict === "prefer" &&
tools?.[tool.name]?.strict === true,
) ?? false;
const activeTools = runtimeOptions.activeTools?.filter((name) => !tools || name in tools);

let params: Parameters<typeof streamText<ToolSet>>[0] = compact({
model: languageModel,
// Our own Responses transport receives the system prompt among the messages.
system: isOwnResponses(model, runtimeOptions.method)
? undefined
: leadingSystemPrompt(context, model, runtimeOptions.method),
messages,
// Prompt and tool changes sent in place travel as system messages.
allowSystemInMessages: messages.some((message) => message.role === "system") || undefined,
tools,
toolChoice: runtimeOptions.toolChoice as Parameters<typeof streamText<ToolSet>>[0]["toolChoice"],
activeTools,
maxOutputTokens: resolveMaxOutputTokens(model, plan),
temperature: model.compat?.supportsTemperature === false ? undefined : runtimeOptions.temperature,
providerOptions,
// The exact body is where `providerThinkingLevel` is read from.
include: { requestBody: true },
abortSignal: runtimeOptions.signal,
timeout: resolveTimeout(runtimeOptions),
maxRetries: runtimeOptions.maxRetries,
headers: runtimeOptions.headers,
// Aikit owns error normalization and emits one terminal error event.
// AI SDK's default callback logs the raw exception (including request
// payloads) before that event can reach an SDK or CLI consumer.
onError: () => {},
});

const payload = await runtimeOptions.onPayload?.(params, model);
if (payload !== undefined) {
params = payload as Parameters<typeof streamText<ToolSet>>[0];
}
return { params, sentPreferStrict };
};

const result = streamText(params);
let preferNonStrict = false;
let built = await buildParams(preferNonStrict);
stream.push({ type: "start", partial: output });

for await (const part of result.fullStream) {
handlePart(part, output, model, stream, pricing);
for (;;) {
try {
for await (const part of streamText(built.params).fullStream) {
handlePart(part, output, model, stream, pricing);
}
break;
} catch (error) {
// Claude refuses the whole request when strict tools exceed its limits, and `prefer` tools
// can go non-strict instead. Only a request refused before anything streamed is retried, once.
const limitMessage = Failure.strictToolLimitMessage(error);
if (
preferNonStrict ||
runtimeOptions.signal?.aborted ||
output.parts.length > 0 ||
!built.sentPreferStrict ||
!isAnthropicWire(model, resolveMethod(model, runtimeOptions.method)) ||
limitMessage === undefined
) {
throw error;
}
(output.diagnostics ??= []).push({
type: "strictToolsFallback",
timestamp: Date.now(),
error: { message: limitMessage },
details: { reason: "strictToolLimits" },
});
preferNonStrict = true;
built = await buildParams(preferNonStrict);
}
}

if (runtimeOptions.signal?.aborted || output.stopReason === "aborted") {
Expand Down
4 changes: 3 additions & 1 deletion packages/aikit/src/llm/strict.ts
Original file line number Diff line number Diff line change
Expand Up @@ -198,14 +198,16 @@ function makeStrictJsonSchema(schema: unknown, isUnsupportedKeyword?: Unsupporte
/**
* The strict parameter schema to send for a JSON-schema constrained tool, or `undefined` to send it non-strict.
* A `"prefer"` tool whose schema the model's strict mode cannot express falls back to non-strict; a `"require"`
* tool throws instead.
* tool throws instead. `preferNonStrict` sends every `"prefer"` tool non-strict.
*/
export function resolveStrictParameters(
tool: Message.Tool,
model?: Pick<Model.Info, "protocol" | "compat">,
preferNonStrict = false,
): Record<string, unknown> | undefined {
const config = tool.constrainedSampling;
if (!config || config.type !== "json_schema") return;
if (preferNonStrict && config.strict === "prefer") return;

if (model?.compat?.supportsStrictMode ?? true) {
const isAnthropic =
Expand Down
9 changes: 5 additions & 4 deletions packages/aikit/src/llm/transform.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ type MovedImages = "anthropicError" | "chat" | "gemini";

const COPILOT = "@codeworksh/ai-sdk-github-copilot";

function isAnthropicWire(model: Model.Info, method: Model.APIMethodEnum): boolean {
export function isAnthropicWire(model: Model.Info, method: Model.APIMethodEnum): boolean {
if (packageForModel(model) === COPILOT) return method === Model.APIMethodEnum.messages;
return isAnthropicMessagesModel(model);
}
Expand Down Expand Up @@ -698,8 +698,8 @@ export function grammarToolInputProperties(
}

/** The parameter schema, strict flag and grammar a tool goes out with on this model. */
function toolWire(tool: Message.Tool, model?: Model.Info, method?: Model.APIMethodEnum) {
const strictParameters = resolveStrictParameters(tool, model);
function toolWire(tool: Message.Tool, model?: Model.Info, method?: Model.APIMethodEnum, preferNonStrict = false) {
const strictParameters = resolveStrictParameters(tool, model, preferNonStrict);
const grammarConstraint =
model && isOwnResponses(model, method) ? resolveToolGrammar(tool, model.compat) : undefined;
return {
Expand All @@ -715,12 +715,13 @@ export function convertTools(
model?: Model.Info,
method?: Model.APIMethodEnum,
deferred: ReadonlySet<string> = new Set(),
preferNonStrict = false,
): ToolSet | undefined {
if (!tools || tools.length === 0) return;

const result: ToolSet = {};
for (const tool of tools) {
const { inputSchema, strict, providerOptions } = toolWire(tool, model, method);
const { inputSchema, strict, providerOptions } = toolWire(tool, model, method, preferNonStrict);
const key = model && deferred.has(tool.name) ? Model.optionsKey(model) : undefined;
const options =
key === undefined
Expand Down
17 changes: 17 additions & 0 deletions packages/aikit/src/message/message.ts
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,22 @@ export const UserMessageSchema = Type.Object({
});
export type UserMessage = Static<typeof UserMessageSchema>;

/** Something aikit did on the caller's behalf while producing a message, such as a request retried in another shape. */
export const AssistantMessageDiagnosticSchema = Type.Object({
type: Type.String(),
timestamp: Type.Number(),
error: Type.Optional(
Type.Object({
name: Type.Optional(Type.String()),
message: Type.String(),
stack: Type.Optional(Type.String()),
code: Type.Optional(Type.Union([Type.String(), Type.Number()])),
}),
),
details: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
});
export type AssistantMessageDiagnostic = Static<typeof AssistantMessageDiagnosticSchema>;

export const AssistantMessageSchema = Type.Object({
role: Type.Literal("assistant"),
protocol: Model.KnownProviderEnumSchema,
Expand All @@ -185,6 +201,7 @@ export const AssistantMessageSchema = Type.Object({
* Depends on ai-sdk
*/
failure: Type.Optional(Failure.FailureSchema),
diagnostics: Type.Optional(Type.Array(AssistantMessageDiagnosticSchema)),
time: Type.Object({
created: Type.Number(),
completed: Type.Number(),
Expand Down
84 changes: 84 additions & 0 deletions packages/aikit/test/e2e/__artifacts__/strict-tool-limit.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
{
"toolCount": {
"attempts": [
{
"count_tool_1": true,
"count_tool_2": true,
"count_tool_3": true,
"count_tool_4": true,
"count_tool_5": true,
"count_tool_6": true,
"count_tool_7": true,
"count_tool_8": true,
"count_tool_9": true,
"count_tool_10": true,
"count_tool_11": true,
"count_tool_12": true,
"count_tool_13": true,
"count_tool_14": true,
"count_tool_15": true,
"count_tool_16": true,
"count_tool_17": true,
"count_tool_18": true,
"count_tool_19": true,
"count_tool_20": true,
"count_tool_21": true,
"required_tool": true
},
{
"count_tool_1": false,
"count_tool_2": false,
"count_tool_3": false,
"count_tool_4": false,
"count_tool_5": false,
"count_tool_6": false,
"count_tool_7": false,
"count_tool_8": false,
"count_tool_9": false,
"count_tool_10": false,
"count_tool_11": false,
"count_tool_12": false,
"count_tool_13": false,
"count_tool_14": false,
"count_tool_15": false,
"count_tool_16": false,
"count_tool_17": false,
"count_tool_18": false,
"count_tool_19": false,
"count_tool_20": false,
"count_tool_21": false,
"required_tool": true
}
],
"stopReason": "stop",
"diagnostics": [
{
"type": "strictToolsFallback",
"reason": "strictToolLimits"
}
]
},
"unionCount": {
"attempts": [
{
"required_tool": true,
"union_tool_1": true,
"union_tool_2": true,
"union_tool_3": true
},
{
"required_tool": true,
"union_tool_1": false,
"union_tool_2": false,
"union_tool_3": false
}
],
"stopReason": "stop",
"diagnostics": [
{
"type": "strictToolsFallback",
"reason": "strictToolLimits"
}
]
}
}
Loading
Loading