Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@ This file is the canonical source for unreleased changes and published release n

- Transcript system messages change the tools mid-conversation: `Message.createSystemMessage({ toolsAdded?, toolsRemoved?, metadata? })`. `Context.tools` is the initial set. OpenAI Codex and GitHub Copilot Responses models that support it declare every tool in place, in `additional_tools` items, and announce removals as text, so the cached prompt prefix holds across tool changes; models with tool search only get later additions as a tool search result; every other model gets the current tools top-level.
- `Transcript` helpers: `getCurrentTools`, `getDeclaredTools`, `getToolStateChanges`, `declarationsEqual`, `toToolDeclaration` and `resolveTranscriptTools`.
- Transcript system messages also change the system prompt: `content` adds instructions from that point on, and `sections` replace named sections, `null` removing one. `Context.systemPrompt` is the initial prompt. OpenAI Codex and GitHub Copilot Responses models with `supportsMidConvoSystemMessages` keep the leading prompt and get each change in place as a developer message (`Updated system prompt section "<name>": …`), so the cached prompt prefix holds; every other model gets the current prompt as its system prompt. Helpers: `getCurrentSystemPrompt`, `renderSystemPrompt`, `renderSystemPromptUpdate` and `resolveTranscriptPrompt`.
- `supportsMidConvoSystemMessages` compat flag, set by `modelgen` for the OpenAI Codex and GitHub Copilot Responses models that accept developer messages mid-conversation. In-place tool declarations (`supportsAdditionalTools`) require it.
- `filterGitHubCopilotModels` keeps the catalog models a Copilot login can reach.
- GitHub Copilot GPT models turn reasoning off with `reasoning.effort: "none"` where OpenAI accepts it, such as gpt-5.3-codex, gpt-5.6-luna and gpt-6-luna.

Expand Down
19 changes: 19 additions & 0 deletions packages/aikit/src/cli/modelgen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,21 @@ const OPENAI_CODEX_ADDITIONAL_TOOLS_MODEL_IDS = new Set([
"gpt-6-sol",
"gpt-6.1-sol",
]);
// These Responses models accept developer messages after the conversation has started (pi).
const OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS = new Set([
"gpt-5.4",
"gpt-5.4-mini",
"gpt-5.4-pro",
"gpt-5.5",
"gpt-5.6-sol",
"gpt-5.6-terra",
"gpt-5.6-luna",
"gpt-6-astra",
"gpt-6-sol",
"gpt-6-luna",
"gpt-6.1-sol",
"gpt-6.1-terra",
]);
// Copilot passes `additional_tools` items through to OpenAI for these Responses models.
const GITHUB_COPILOT_ADDITIONAL_TOOLS_MODEL_IDS = new Set([
"gpt-5.4",
Expand Down Expand Up @@ -552,6 +567,7 @@ export function openAICodexBuiltInModels(openai: Record<string, Model.Info>): Re
providerOptionsKey: OPENAI_CODEX_PROVIDER_ID,
compat: {
supportsOpenAIGrammarTools: true,
...(OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS.has(seed.id) ? { supportsMidConvoSystemMessages: true } : {}),
...(OPENAI_CODEX_ADDITIONAL_TOOLS_MODEL_IDS.has(seed.id)
? { supportsToolSearch: true, supportsAdditionalTools: true }
: {}),
Expand Down Expand Up @@ -682,6 +698,9 @@ export function githubCopilotBuiltInModels(provider: ModelsDevProvider | undefin
if (method === Model.APIMethodEnum.responses && gpt !== null && Number(gpt[1]) >= 5) {
mergeCompat(info, { supportsOpenAIGrammarTools: true });
}
if (method === Model.APIMethodEnum.responses && OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS.has(id)) {
mergeCompat(info, { supportsMidConvoSystemMessages: true });
}
if (method === Model.APIMethodEnum.responses && GITHUB_COPILOT_ADDITIONAL_TOOLS_MODEL_IDS.has(id)) {
mergeCompat(info, { supportsAdditionalTools: true });
}
Expand Down
5 changes: 3 additions & 2 deletions packages/aikit/src/llm/stream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
createAssistantMessage,
encodeOpenAIReasoningSignature,
grammarToolInputProperties,
leadingSystemPrompt,
mapFinishReason,
mapUsage,
toolCallFromPart,
Expand Down Expand Up @@ -432,9 +433,9 @@ export const stream: Protocol.StreamFunction<Model.KnownProviderEnum, typeof Opt
let params: Parameters<typeof streamText<ToolSet>>[0] = compact({
model: languageModel,
// Our own Responses transport receives the system prompt among the messages.
system: isOwnResponses(model, runtimeOptions.method) ? undefined : context.systemPrompt,
system: isOwnResponses(model, runtimeOptions.method) ? undefined : leadingSystemPrompt(context),
messages,
// Tool changes declared in place travel as system messages.
// Prompt and tool changes sent in place travel as system messages.
allowSystemInMessages: messages.some((message) => message.role === "system") || undefined,
tools,
toolChoice: runtimeOptions.toolChoice as Parameters<typeof streamText<ToolSet>>[0]["toolChoice"],
Expand Down
47 changes: 37 additions & 10 deletions packages/aikit/src/llm/transform.ts
Original file line number Diff line number Diff line change
Expand Up @@ -480,10 +480,23 @@ export function transcriptToolSupport(
restricted = false,
): Transcript.TranscriptToolSupport {
if (restricted || !isOwnResponses(model, method)) return "none";
if (model.compat?.supportsAdditionalTools) return "changes";
if (supportsMidConvoSystemMessages(model, method) && model.compat?.supportsAdditionalTools) return "changes";
return model.compat?.supportsToolSearch ? "additions" : "none";
}

/**
* Whether prompt changes go in place as developer messages. Only our own Responses transport
* sends them; the AI SDK adapters take the current prompt as the leading system prompt.
*/
export function supportsMidConvoSystemMessages(model: Model.Info, method: Model.APIMethodEnum | undefined): boolean {
return isOwnResponses(model, method) && model.compat?.supportsMidConvoSystemMessages === true;
}

/** The prompt the AI SDK adapters take as `system`: the current one, every change folded in. */
export function leadingSystemPrompt(context: Message.Context): string | undefined {
return Transcript.getCurrentSystemPrompt(context) || undefined;
}

/** A tool as the language model receives it, for declaring it inside the prompt. */
function functionTool(tool: Message.Tool, model: Model.Info, method: Model.APIMethodEnum): LanguageModelV3FunctionTool {
const { inputSchema, strict, providerOptions } = toolWire(tool, model, method);
Expand All @@ -497,22 +510,26 @@ function functionTool(tool: Message.Tool, model: Model.Info, method: Model.APIMe
};
}

function toolChangeMessage(
/** A system message sent in place: the prompt update `content`, then the tool change, if any. */
function systemChangeMessage(
content: string,
toolsAdded: readonly Message.Tool[],
toolsRemoved: readonly Message.ToolReference[],
support: Transcript.TranscriptToolSupport,
model: Model.Info,
method: Model.APIMethodEnum,
): ModelMessage | undefined {
if (toolsAdded.length === 0 && toolsRemoved.length === 0) return;
if (toolsAdded.length === 0 && toolsRemoved.length === 0) {
return content.length > 0 ? { role: "system", content: sanitizeSurrogates(content) } : undefined;
}
const toolChange: ResponsesToolChange = {
mode: support === "changes" ? "changes" : "additions",
tools: toolsAdded.map((tool) => functionTool(tool, model, method)),
removed: toolsRemoved.map((tool) => tool.name),
};
return {
role: "system",
content: "",
content: sanitizeSurrogates(content),
// Function tools are plain JSON once their schema went through `functionTool`.
providerOptions: { [Model.optionsKey(model)]: { toolChange } } as unknown as SharedV3ProviderOptions,
};
Expand All @@ -529,21 +546,31 @@ export function convertMessages(
const transformedMessages = Message.transformMessages(context.messages, model, toolCallIdNormalizer(model, method));
const support = transcriptToolSupport(model, method, activeTools !== undefined);
const transcript = Transcript.resolveTranscriptTools(context, support);
const prompt = Transcript.resolveTranscriptPrompt(context, supportsMidConvoSystemMessages(model, method));

if (transcript.initialToolsInPlace) {
const head = toolChangeMessage(context.tools ?? [], [], support, model, method);
const head = systemChangeMessage("", context.tools ?? [], [], support, model, method);
if (head) messages.push(head);
}
// Our Responses transport takes the system prompt in place, after the initial tool declaration.
if (isOwnResponses(model, method) && context.systemPrompt) {
messages.push({ role: "system", content: sanitizeSurrogates(context.systemPrompt) });
// Our Responses transport takes the leading prompt in place, after the initial tool declaration.
if (isOwnResponses(model, method) && prompt.systemPrompt) {
messages.push({ role: "system", content: sanitizeSurrogates(prompt.systemPrompt) });
}

let systemIndex = 0;
for (const msg of transformedMessages) {
if (msg.role === "system") {
if (systemIndex++ < transcript.foldedSystemMessages) continue;
const change = toolChangeMessage(msg.toolsAdded ?? [], msg.toolsRemoved ?? [], support, model, method);
const index = systemIndex++;
const update = index < prompt.foldedSystemMessages ? "" : Transcript.renderSystemPromptUpdate(msg);
const folded = index < transcript.foldedSystemMessages;
const change = systemChangeMessage(
update,
folded ? [] : (msg.toolsAdded ?? []),
folded ? [] : (msg.toolsRemoved ?? []),
support,
model,
method,
);
if (change) messages.push(change);
continue;
}
Expand Down
17 changes: 13 additions & 4 deletions packages/aikit/src/message/message.ts
Original file line number Diff line number Diff line change
Expand Up @@ -286,18 +286,27 @@ export const ToolReferenceSchema = Type.Object({
export type ToolReference = Static<typeof ToolReferenceSchema>;

/**
* A change to the tool set at one point in the transcript.
* A change to the system prompt and tool set at one point in the transcript.
*
* `Context.tools` holds the tools at the start of the conversation; each system message
* changes that set from its position on. Replaying them in order yields the current tools.
* A redefinition is a removal plus an addition of the same name.
* `Context.systemPrompt` and `Context.tools` hold the prompt and tools at the start of the
* conversation; each system message changes them from its position on. Replaying them in
* order yields the current prompt and tools. A redefinition is a removal plus an addition of
* the same name.
*/
export const SystemMessageSchema = Type.Object({
messageId: Type.String(),
role: Type.Literal("system"),
time: Type.Object({
created: Type.Number(),
}),
/** Instructions that apply from this point on, after the earlier ones. */
content: Type.Optional(Type.String()),
/**
* Named prompt sections rendered after the content. A later message replaces a section by
* name, and `null` removes it. Keep each section self-delimiting (a tag) so the model can
* relate an update to the original, and avoid integer-like names, which JSON reorders.
*/
sections: Type.Optional(Type.Record(Type.String(), Type.Union([Type.String(), Type.Null()]))),
/** Complete definitions of tools that become available at this point. */
toolsAdded: Type.Optional(Type.Array(ToolSchema)),
/** Tools that stop being available at this point. */
Expand Down
91 changes: 85 additions & 6 deletions packages/aikit/src/message/transcript.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,99 @@
/*
* @file Tool state carried by a transcript.
* @file Prompt and tool state carried by a transcript.
*
* `Context.tools` is the tool set at the start of the conversation. Each system message
* changes it from its position on, so replaying them in order yields the tools available
* now. Providers either declare those changes in place, which keeps the cached prefix, or
* fold them into the top-level tool list.
* `Context.systemPrompt` and `Context.tools` are the prompt and tool set at the start of the
* conversation. Each system message changes them from its position on, so replaying them in
* order yields the prompt and tools in effect now. Providers either send those changes in
* place, which keeps the cached prefix, or fold them into the leading prompt and the
* top-level tool list.
*/

import type { Context, Message, SystemMessage, Tool, ToolReference } from "./message.ts";

type Transcript = Pick<Context, "tools" | "messages">;
type Transcript = Pick<Context, "systemPrompt" | "tools" | "messages">;

type PromptChange = Pick<SystemMessage, "content" | "sections">;

function isSystemMessage(message: Message): message is SystemMessage {
return message.role === "system";
}

/** Render a system message as a complete prompt: its content followed by its sections. */
export function renderSystemPrompt(message: PromptChange): string {
const parts = [message.content ?? ""];
for (const text of Object.values(message.sections ?? {})) {
if (text !== null) parts.push(text);
}
return parts.filter((part) => part.length > 0).join("\n\n");
}

/**
* Render a later system message sent in place. Section changes are framed by name so the
* model can relate them to the leading prompt.
*/
export function renderSystemPromptUpdate(message: PromptChange): string {
const parts: string[] = [];
if (message.content) parts.push(message.content);
for (const [name, value] of Object.entries(message.sections ?? {})) {
parts.push(
value === null
? `Removed system prompt section "${name}".`
: `Updated system prompt section "${name}":\n\n${value}`,
);
}
return parts.join("\n\n");
}

/**
* The prompt after every transcript change in order: content accumulates, sections are
* replaced by name, and a removed section added again goes last.
*/
export function getCurrentSystemPrompt(context: Pick<Context, "systemPrompt" | "messages">): string {
const content: string[] = [];
const sections = new Map<string, string>();
const changes: PromptChange[] = [
{ content: context.systemPrompt ?? "" },
...context.messages.filter(isSystemMessage),
];
for (const message of changes) {
if (message.content) content.push(message.content);
for (const [name, value] of Object.entries(message.sections ?? {})) {
if (value === null) sections.delete(name);
else sections.set(name, value);
}
}
return renderSystemPrompt({ content: content.join("\n\n"), sections: Object.fromEntries(sections) });
}

export interface TranscriptPrompt {
/** The prompt sent at the head of the request. */
systemPrompt: string;
/**
* Number of leading transcript system messages whose prompt changes `systemPrompt` already
* holds. Every later system message sends its changes in place.
*/
foldedSystemMessages: number;
}

/**
* Split the prompt between the head of the request and in-place updates.
*
* In place, `Context.systemPrompt` leads; without one (and without initial tools), a leading
* system message is the prompt. Every other system message is an update. Otherwise the head
* holds the current prompt and no update is sent, which costs the cached prefix on a change.
*/
export function resolveTranscriptPrompt(context: Transcript, inPlace: boolean): TranscriptPrompt {
const systemMessages = context.messages.filter(isSystemMessage);
if (!inPlace) {
return { systemPrompt: getCurrentSystemPrompt(context), foldedSystemMessages: systemMessages.length };
}
const first = context.messages[0];
if (!context.systemPrompt && (context.tools ?? []).length === 0 && first && isSystemMessage(first)) {
return { systemPrompt: renderSystemPrompt(first), foldedSystemMessages: 1 };
}
return { systemPrompt: context.systemPrompt ?? "", foldedSystemMessages: 0 };
}

/** The tool changes in order, led by `Context.tools` as the initial declaration. */
function toolDeclarations(context: Transcript): Pick<SystemMessage, "toolsAdded" | "toolsRemoved">[] {
return [{ toolsAdded: context.tools ?? [] }, ...context.messages.filter(isSystemMessage)];
Expand Down
3 changes: 3 additions & 0 deletions packages/aikit/src/model/model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,10 @@ export const CompatibilitySchema = Type.Object({
/** The Google protocol accepts thinking levels rather than a token budget. */
supportsThinkingLevel: Type.Optional(Type.Boolean()),
supportsToolSearch: Type.Optional(Type.Boolean()),
/** Requires `supportsMidConvoSystemMessages`. */
supportsAdditionalTools: Type.Optional(Type.Boolean()),
/** The model accepts developer messages after the conversation has started. */
supportsMidConvoSystemMessages: Type.Optional(Type.Boolean()),
supportsStrictMode: Type.Optional(Type.Boolean()),
supportsOpenAIGrammarTools: Type.Optional(Type.Boolean()),
/** The model takes a thinking-token budget as a top-level request field. */
Expand Down
28 changes: 14 additions & 14 deletions packages/aikit/src/providers/responses/prompt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@ export type ResponsesPromptOptions = {
/** The providerOptions key this transport reads, e.g. `openai-codex`. */
provider: string;
/**
* Where system prompt text goes: the `instructions` field (the Codex backend requires
* one), or a developer message in place.
* Where the leading system prompt goes: the `instructions` field (the Codex backend
* requires one), or a developer message in place.
*/
systemPrompt: "instructions" | "developer";
grammarToolInputProperties?: ReadonlyMap<string, string>;
Expand Down Expand Up @@ -282,9 +282,9 @@ function toolRemovalNotice(names: readonly string[]): string {
}

/**
* Convert an AI SDK V3 prompt into the Responses request shape. Tool changes are
* declared in place, other system messages go where `options.systemPrompt` says,
* and everything else maps to `input` items.
* Convert an AI SDK V3 prompt into the Responses request shape. The leading system
* message goes where `options.systemPrompt` says, later ones (prompt updates and tool
* changes) go in place, and everything else maps to `input` items.
*/
export function convertToResponsesPrompt(
prompt: LanguageModelV3Prompt | LanguageModelV4Prompt,
Expand All @@ -299,7 +299,8 @@ export function convertToResponsesPrompt(
/** Tools announced as removed and not declared again since. */
const unavailableTools = new Set<string>();

const appendToolChange = (change: ResponsesToolChange): void => {
/** Declares the change's tools; returns the removal notice, which goes with the message's text. */
const appendToolChange = (change: ResponsesToolChange): string | undefined => {
const seed = toolChangeCounter++;
const tools = change.tools.map((tool) => toResponsesTool(tool, provider));
for (const tool of tools) unavailableTools.delete(tool.name);
Expand Down Expand Up @@ -331,24 +332,23 @@ export function convertToResponsesPrompt(
const added = new Set(tools.map((tool) => tool.name));
const removed = change.removed.filter((name) => !added.has(name));
if (removed.length === 0) return;
input.push({ role: "developer", content: toolRemovalNotice(removed) });
// Recorded after this change's additions, whose notice precedes this removal notice.
for (const name of removed) unavailableTools.add(name);
return toolRemovalNotice(removed);
};

for (const message of prompt) {
switch (message.role) {
case "system": {
const change = toolChange(message.providerOptions, provider);
if (change) {
appendToolChange(change);
// The first plain system message is the leading prompt; every later one is an update in place.
if (!change && options.systemPrompt === "instructions" && instructions === undefined) {
instructions = message.content;
break;
}
if (options.systemPrompt === "developer") {
if (message.content.length > 0) input.push({ role: "developer", content: message.content });
break;
}
instructions = instructions == null ? message.content : `${instructions}\n\n${message.content}`;
const removal = change ? appendToolChange(change) : undefined;
const text = [message.content, removal ?? ""].filter((part) => part.length > 0).join("\n\n");
if (text.length > 0) input.push({ role: "developer", content: text });
break;
}

Expand Down
Loading
Loading