diff --git a/CHANGELOG.md b/CHANGELOG.md index af47527c..2d892f4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,8 @@ This file is the canonical source for unreleased changes and published release n - Transcript system messages change the tools mid-conversation: `Message.createSystemMessage({ toolsAdded?, toolsRemoved?, metadata? })`. `Context.tools` is the initial set. OpenAI Codex and GitHub Copilot Responses models that support it declare every tool in place, in `additional_tools` items, and announce removals as text, so the cached prompt prefix holds across tool changes; models with tool search only get later additions as a tool search result; every other model gets the current tools top-level. - `Transcript` helpers: `getCurrentTools`, `getDeclaredTools`, `getToolStateChanges`, `declarationsEqual`, `toToolDeclaration` and `resolveTranscriptTools`. +- Transcript system messages also change the system prompt: `content` adds instructions from that point on, and `sections` replace named sections, `null` removing one. `Context.systemPrompt` is the initial prompt. OpenAI Codex and GitHub Copilot Responses models with `supportsMidConvoSystemMessages` keep the leading prompt and get each change in place as a developer message (`Updated system prompt section "": …`), so the cached prompt prefix holds; every other model gets the current prompt as its system prompt. Helpers: `getCurrentSystemPrompt`, `renderSystemPrompt`, `renderSystemPromptUpdate` and `resolveTranscriptPrompt`. +- `supportsMidConvoSystemMessages` compat flag, set by `modelgen` for the OpenAI Codex and GitHub Copilot Responses models that accept developer messages mid-conversation. In-place tool declarations (`supportsAdditionalTools`) require it. - `filterGitHubCopilotModels` keeps the catalog models a Copilot login can reach. - GitHub Copilot GPT models turn reasoning off with `reasoning.effort: "none"` where OpenAI accepts it, such as gpt-5.3-codex, gpt-5.6-luna and gpt-6-luna. diff --git a/packages/aikit/src/cli/modelgen.ts b/packages/aikit/src/cli/modelgen.ts index 842e3bbc..68fe348d 100644 --- a/packages/aikit/src/cli/modelgen.ts +++ b/packages/aikit/src/cli/modelgen.ts @@ -501,6 +501,21 @@ const OPENAI_CODEX_ADDITIONAL_TOOLS_MODEL_IDS = new Set([ "gpt-6-sol", "gpt-6.1-sol", ]); +// These Responses models accept developer messages after the conversation has started (pi). +const OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS = new Set([ + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-6-astra", + "gpt-6-sol", + "gpt-6-luna", + "gpt-6.1-sol", + "gpt-6.1-terra", +]); // Copilot passes `additional_tools` items through to OpenAI for these Responses models. const GITHUB_COPILOT_ADDITIONAL_TOOLS_MODEL_IDS = new Set([ "gpt-5.4", @@ -552,6 +567,7 @@ export function openAICodexBuiltInModels(openai: Record): Re providerOptionsKey: OPENAI_CODEX_PROVIDER_ID, compat: { supportsOpenAIGrammarTools: true, + ...(OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS.has(seed.id) ? { supportsMidConvoSystemMessages: true } : {}), ...(OPENAI_CODEX_ADDITIONAL_TOOLS_MODEL_IDS.has(seed.id) ? { supportsToolSearch: true, supportsAdditionalTools: true } : {}), @@ -682,6 +698,9 @@ export function githubCopilotBuiltInModels(provider: ModelsDevProvider | undefin if (method === Model.APIMethodEnum.responses && gpt !== null && Number(gpt[1]) >= 5) { mergeCompat(info, { supportsOpenAIGrammarTools: true }); } + if (method === Model.APIMethodEnum.responses && OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS.has(id)) { + mergeCompat(info, { supportsMidConvoSystemMessages: true }); + } if (method === Model.APIMethodEnum.responses && GITHUB_COPILOT_ADDITIONAL_TOOLS_MODEL_IDS.has(id)) { mergeCompat(info, { supportsAdditionalTools: true }); } diff --git a/packages/aikit/src/llm/stream.ts b/packages/aikit/src/llm/stream.ts index 525dcf7d..b3116ca0 100644 --- a/packages/aikit/src/llm/stream.ts +++ b/packages/aikit/src/llm/stream.ts @@ -21,6 +21,7 @@ import { createAssistantMessage, encodeOpenAIReasoningSignature, grammarToolInputProperties, + leadingSystemPrompt, mapFinishReason, mapUsage, toolCallFromPart, @@ -432,9 +433,9 @@ export const stream: Protocol.StreamFunction>[0] = compact({ model: languageModel, // Our own Responses transport receives the system prompt among the messages. - system: isOwnResponses(model, runtimeOptions.method) ? undefined : context.systemPrompt, + system: isOwnResponses(model, runtimeOptions.method) ? undefined : leadingSystemPrompt(context), messages, - // Tool changes declared in place travel as system messages. + // Prompt and tool changes sent in place travel as system messages. allowSystemInMessages: messages.some((message) => message.role === "system") || undefined, tools, toolChoice: runtimeOptions.toolChoice as Parameters>[0]["toolChoice"], diff --git a/packages/aikit/src/llm/transform.ts b/packages/aikit/src/llm/transform.ts index 45e5a53e..bf161bc6 100644 --- a/packages/aikit/src/llm/transform.ts +++ b/packages/aikit/src/llm/transform.ts @@ -480,10 +480,23 @@ export function transcriptToolSupport( restricted = false, ): Transcript.TranscriptToolSupport { if (restricted || !isOwnResponses(model, method)) return "none"; - if (model.compat?.supportsAdditionalTools) return "changes"; + if (supportsMidConvoSystemMessages(model, method) && model.compat?.supportsAdditionalTools) return "changes"; return model.compat?.supportsToolSearch ? "additions" : "none"; } +/** + * Whether prompt changes go in place as developer messages. Only our own Responses transport + * sends them; the AI SDK adapters take the current prompt as the leading system prompt. + */ +export function supportsMidConvoSystemMessages(model: Model.Info, method: Model.APIMethodEnum | undefined): boolean { + return isOwnResponses(model, method) && model.compat?.supportsMidConvoSystemMessages === true; +} + +/** The prompt the AI SDK adapters take as `system`: the current one, every change folded in. */ +export function leadingSystemPrompt(context: Message.Context): string | undefined { + return Transcript.getCurrentSystemPrompt(context) || undefined; +} + /** A tool as the language model receives it, for declaring it inside the prompt. */ function functionTool(tool: Message.Tool, model: Model.Info, method: Model.APIMethodEnum): LanguageModelV3FunctionTool { const { inputSchema, strict, providerOptions } = toolWire(tool, model, method); @@ -497,14 +510,18 @@ function functionTool(tool: Message.Tool, model: Model.Info, method: Model.APIMe }; } -function toolChangeMessage( +/** A system message sent in place: the prompt update `content`, then the tool change, if any. */ +function systemChangeMessage( + content: string, toolsAdded: readonly Message.Tool[], toolsRemoved: readonly Message.ToolReference[], support: Transcript.TranscriptToolSupport, model: Model.Info, method: Model.APIMethodEnum, ): ModelMessage | undefined { - if (toolsAdded.length === 0 && toolsRemoved.length === 0) return; + if (toolsAdded.length === 0 && toolsRemoved.length === 0) { + return content.length > 0 ? { role: "system", content: sanitizeSurrogates(content) } : undefined; + } const toolChange: ResponsesToolChange = { mode: support === "changes" ? "changes" : "additions", tools: toolsAdded.map((tool) => functionTool(tool, model, method)), @@ -512,7 +529,7 @@ function toolChangeMessage( }; return { role: "system", - content: "", + content: sanitizeSurrogates(content), // Function tools are plain JSON once their schema went through `functionTool`. providerOptions: { [Model.optionsKey(model)]: { toolChange } } as unknown as SharedV3ProviderOptions, }; @@ -529,21 +546,31 @@ export function convertMessages( const transformedMessages = Message.transformMessages(context.messages, model, toolCallIdNormalizer(model, method)); const support = transcriptToolSupport(model, method, activeTools !== undefined); const transcript = Transcript.resolveTranscriptTools(context, support); + const prompt = Transcript.resolveTranscriptPrompt(context, supportsMidConvoSystemMessages(model, method)); if (transcript.initialToolsInPlace) { - const head = toolChangeMessage(context.tools ?? [], [], support, model, method); + const head = systemChangeMessage("", context.tools ?? [], [], support, model, method); if (head) messages.push(head); } - // Our Responses transport takes the system prompt in place, after the initial tool declaration. - if (isOwnResponses(model, method) && context.systemPrompt) { - messages.push({ role: "system", content: sanitizeSurrogates(context.systemPrompt) }); + // Our Responses transport takes the leading prompt in place, after the initial tool declaration. + if (isOwnResponses(model, method) && prompt.systemPrompt) { + messages.push({ role: "system", content: sanitizeSurrogates(prompt.systemPrompt) }); } let systemIndex = 0; for (const msg of transformedMessages) { if (msg.role === "system") { - if (systemIndex++ < transcript.foldedSystemMessages) continue; - const change = toolChangeMessage(msg.toolsAdded ?? [], msg.toolsRemoved ?? [], support, model, method); + const index = systemIndex++; + const update = index < prompt.foldedSystemMessages ? "" : Transcript.renderSystemPromptUpdate(msg); + const folded = index < transcript.foldedSystemMessages; + const change = systemChangeMessage( + update, + folded ? [] : (msg.toolsAdded ?? []), + folded ? [] : (msg.toolsRemoved ?? []), + support, + model, + method, + ); if (change) messages.push(change); continue; } diff --git a/packages/aikit/src/message/message.ts b/packages/aikit/src/message/message.ts index 08cd4e58..026000a1 100644 --- a/packages/aikit/src/message/message.ts +++ b/packages/aikit/src/message/message.ts @@ -286,11 +286,12 @@ export const ToolReferenceSchema = Type.Object({ export type ToolReference = Static; /** - * A change to the tool set at one point in the transcript. + * A change to the system prompt and tool set at one point in the transcript. * - * `Context.tools` holds the tools at the start of the conversation; each system message - * changes that set from its position on. Replaying them in order yields the current tools. - * A redefinition is a removal plus an addition of the same name. + * `Context.systemPrompt` and `Context.tools` hold the prompt and tools at the start of the + * conversation; each system message changes them from its position on. Replaying them in + * order yields the current prompt and tools. A redefinition is a removal plus an addition of + * the same name. */ export const SystemMessageSchema = Type.Object({ messageId: Type.String(), @@ -298,6 +299,14 @@ export const SystemMessageSchema = Type.Object({ time: Type.Object({ created: Type.Number(), }), + /** Instructions that apply from this point on, after the earlier ones. */ + content: Type.Optional(Type.String()), + /** + * Named prompt sections rendered after the content. A later message replaces a section by + * name, and `null` removes it. Keep each section self-delimiting (a tag) so the model can + * relate an update to the original, and avoid integer-like names, which JSON reorders. + */ + sections: Type.Optional(Type.Record(Type.String(), Type.Union([Type.String(), Type.Null()]))), /** Complete definitions of tools that become available at this point. */ toolsAdded: Type.Optional(Type.Array(ToolSchema)), /** Tools that stop being available at this point. */ diff --git a/packages/aikit/src/message/transcript.ts b/packages/aikit/src/message/transcript.ts index c7e7c0e9..0902408d 100644 --- a/packages/aikit/src/message/transcript.ts +++ b/packages/aikit/src/message/transcript.ts @@ -1,20 +1,99 @@ /* - * @file Tool state carried by a transcript. + * @file Prompt and tool state carried by a transcript. * - * `Context.tools` is the tool set at the start of the conversation. Each system message - * changes it from its position on, so replaying them in order yields the tools available - * now. Providers either declare those changes in place, which keeps the cached prefix, or - * fold them into the top-level tool list. + * `Context.systemPrompt` and `Context.tools` are the prompt and tool set at the start of the + * conversation. Each system message changes them from its position on, so replaying them in + * order yields the prompt and tools in effect now. Providers either send those changes in + * place, which keeps the cached prefix, or fold them into the leading prompt and the + * top-level tool list. */ import type { Context, Message, SystemMessage, Tool, ToolReference } from "./message.ts"; -type Transcript = Pick; +type Transcript = Pick; + +type PromptChange = Pick; function isSystemMessage(message: Message): message is SystemMessage { return message.role === "system"; } +/** Render a system message as a complete prompt: its content followed by its sections. */ +export function renderSystemPrompt(message: PromptChange): string { + const parts = [message.content ?? ""]; + for (const text of Object.values(message.sections ?? {})) { + if (text !== null) parts.push(text); + } + return parts.filter((part) => part.length > 0).join("\n\n"); +} + +/** + * Render a later system message sent in place. Section changes are framed by name so the + * model can relate them to the leading prompt. + */ +export function renderSystemPromptUpdate(message: PromptChange): string { + const parts: string[] = []; + if (message.content) parts.push(message.content); + for (const [name, value] of Object.entries(message.sections ?? {})) { + parts.push( + value === null + ? `Removed system prompt section "${name}".` + : `Updated system prompt section "${name}":\n\n${value}`, + ); + } + return parts.join("\n\n"); +} + +/** + * The prompt after every transcript change in order: content accumulates, sections are + * replaced by name, and a removed section added again goes last. + */ +export function getCurrentSystemPrompt(context: Pick): string { + const content: string[] = []; + const sections = new Map(); + const changes: PromptChange[] = [ + { content: context.systemPrompt ?? "" }, + ...context.messages.filter(isSystemMessage), + ]; + for (const message of changes) { + if (message.content) content.push(message.content); + for (const [name, value] of Object.entries(message.sections ?? {})) { + if (value === null) sections.delete(name); + else sections.set(name, value); + } + } + return renderSystemPrompt({ content: content.join("\n\n"), sections: Object.fromEntries(sections) }); +} + +export interface TranscriptPrompt { + /** The prompt sent at the head of the request. */ + systemPrompt: string; + /** + * Number of leading transcript system messages whose prompt changes `systemPrompt` already + * holds. Every later system message sends its changes in place. + */ + foldedSystemMessages: number; +} + +/** + * Split the prompt between the head of the request and in-place updates. + * + * In place, `Context.systemPrompt` leads; without one (and without initial tools), a leading + * system message is the prompt. Every other system message is an update. Otherwise the head + * holds the current prompt and no update is sent, which costs the cached prefix on a change. + */ +export function resolveTranscriptPrompt(context: Transcript, inPlace: boolean): TranscriptPrompt { + const systemMessages = context.messages.filter(isSystemMessage); + if (!inPlace) { + return { systemPrompt: getCurrentSystemPrompt(context), foldedSystemMessages: systemMessages.length }; + } + const first = context.messages[0]; + if (!context.systemPrompt && (context.tools ?? []).length === 0 && first && isSystemMessage(first)) { + return { systemPrompt: renderSystemPrompt(first), foldedSystemMessages: 1 }; + } + return { systemPrompt: context.systemPrompt ?? "", foldedSystemMessages: 0 }; +} + /** The tool changes in order, led by `Context.tools` as the initial declaration. */ function toolDeclarations(context: Transcript): Pick[] { return [{ toolsAdded: context.tools ?? [] }, ...context.messages.filter(isSystemMessage)]; diff --git a/packages/aikit/src/model/model.ts b/packages/aikit/src/model/model.ts index f59ee50d..879a06cd 100644 --- a/packages/aikit/src/model/model.ts +++ b/packages/aikit/src/model/model.ts @@ -160,7 +160,10 @@ export const CompatibilitySchema = Type.Object({ /** The Google protocol accepts thinking levels rather than a token budget. */ supportsThinkingLevel: Type.Optional(Type.Boolean()), supportsToolSearch: Type.Optional(Type.Boolean()), + /** Requires `supportsMidConvoSystemMessages`. */ supportsAdditionalTools: Type.Optional(Type.Boolean()), + /** The model accepts developer messages after the conversation has started. */ + supportsMidConvoSystemMessages: Type.Optional(Type.Boolean()), supportsStrictMode: Type.Optional(Type.Boolean()), supportsOpenAIGrammarTools: Type.Optional(Type.Boolean()), /** The model takes a thinking-token budget as a top-level request field. */ diff --git a/packages/aikit/src/providers/responses/prompt.ts b/packages/aikit/src/providers/responses/prompt.ts index efc175a8..8cbb339f 100644 --- a/packages/aikit/src/providers/responses/prompt.ts +++ b/packages/aikit/src/providers/responses/prompt.ts @@ -95,8 +95,8 @@ export type ResponsesPromptOptions = { /** The providerOptions key this transport reads, e.g. `openai-codex`. */ provider: string; /** - * Where system prompt text goes: the `instructions` field (the Codex backend requires - * one), or a developer message in place. + * Where the leading system prompt goes: the `instructions` field (the Codex backend + * requires one), or a developer message in place. */ systemPrompt: "instructions" | "developer"; grammarToolInputProperties?: ReadonlyMap; @@ -282,9 +282,9 @@ function toolRemovalNotice(names: readonly string[]): string { } /** - * Convert an AI SDK V3 prompt into the Responses request shape. Tool changes are - * declared in place, other system messages go where `options.systemPrompt` says, - * and everything else maps to `input` items. + * Convert an AI SDK V3 prompt into the Responses request shape. The leading system + * message goes where `options.systemPrompt` says, later ones (prompt updates and tool + * changes) go in place, and everything else maps to `input` items. */ export function convertToResponsesPrompt( prompt: LanguageModelV3Prompt | LanguageModelV4Prompt, @@ -299,7 +299,8 @@ export function convertToResponsesPrompt( /** Tools announced as removed and not declared again since. */ const unavailableTools = new Set(); - const appendToolChange = (change: ResponsesToolChange): void => { + /** Declares the change's tools; returns the removal notice, which goes with the message's text. */ + const appendToolChange = (change: ResponsesToolChange): string | undefined => { const seed = toolChangeCounter++; const tools = change.tools.map((tool) => toResponsesTool(tool, provider)); for (const tool of tools) unavailableTools.delete(tool.name); @@ -331,24 +332,23 @@ export function convertToResponsesPrompt( const added = new Set(tools.map((tool) => tool.name)); const removed = change.removed.filter((name) => !added.has(name)); if (removed.length === 0) return; - input.push({ role: "developer", content: toolRemovalNotice(removed) }); // Recorded after this change's additions, whose notice precedes this removal notice. for (const name of removed) unavailableTools.add(name); + return toolRemovalNotice(removed); }; for (const message of prompt) { switch (message.role) { case "system": { const change = toolChange(message.providerOptions, provider); - if (change) { - appendToolChange(change); + // The first plain system message is the leading prompt; every later one is an update in place. + if (!change && options.systemPrompt === "instructions" && instructions === undefined) { + instructions = message.content; break; } - if (options.systemPrompt === "developer") { - if (message.content.length > 0) input.push({ role: "developer", content: message.content }); - break; - } - instructions = instructions == null ? message.content : `${instructions}\n\n${message.content}`; + const removal = change ? appendToolChange(change) : undefined; + const text = [message.content, removal ?? ""].filter((part) => part.length > 0).join("\n\n"); + if (text.length > 0) input.push({ role: "developer", content: text }); break; } diff --git a/packages/aikit/src/utils/estimate.ts b/packages/aikit/src/utils/estimate.ts index 9712c6fe..4b395aa3 100644 --- a/packages/aikit/src/utils/estimate.ts +++ b/packages/aikit/src/utils/estimate.ts @@ -9,6 +9,7 @@ */ import type * as Message from "../message/message.ts"; +import * as Transcript from "../message/transcript.ts"; /** Rough characters-per-token ratio across the tokenizers in use. */ const CHARS_PER_TOKEN = 4; @@ -65,7 +66,9 @@ function toolResultChars(part: Message.ToolCall): number { } export function estimateMessageTokens(message: Message.Message): number { - if (message.role === "system") return estimateToolsTokens(message.toolsAdded); + if (message.role === "system") { + return estimateTextTokens(Transcript.renderSystemPromptUpdate(message)) + estimateToolsTokens(message.toolsAdded); + } if (message.role === "user") return estimateContentTokens(message.parts); let chars = 0; @@ -148,8 +151,8 @@ export function estimateContextTokens(context: Message.Context): ContextUsageEst /* * Reported usage already includes the system prompt and the tools that were - * live at the time. Tools added since arrive as system messages after it, - * which the trailing estimate already counts. + * live at the time. Prompt and tool changes since arrive as system messages + * after it, which the trailing estimate already counts. */ if (estimate.lastUsageIndex !== null) return estimate; diff --git a/packages/aikit/test/__artifacts__/modelgen.json b/packages/aikit/test/__artifacts__/modelgen.json index 6335bef4..5bbf7420 100644 --- a/packages/aikit/test/__artifacts__/modelgen.json +++ b/packages/aikit/test/__artifacts__/modelgen.json @@ -2782,6 +2782,7 @@ "baseUrl": "https://chatgpt.com/backend-api", "compat": { "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsOpenAIGrammarTools": true, "supportsToolSearch": true }, @@ -2843,6 +2844,7 @@ "baseUrl": "https://chatgpt.com/backend-api", "compat": { "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsOpenAIGrammarTools": true, "supportsToolSearch": true }, @@ -2904,6 +2906,7 @@ "baseUrl": "https://chatgpt.com/backend-api", "compat": { "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsOpenAIGrammarTools": true, "supportsToolSearch": true }, @@ -2965,6 +2968,7 @@ "baseUrl": "https://chatgpt.com/backend-api", "compat": { "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsOpenAIGrammarTools": true, "supportsToolSearch": true }, @@ -3027,6 +3031,7 @@ "baseUrl": "https://chatgpt.com/backend-api", "compat": { "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsOpenAIGrammarTools": true, "supportsToolSearch": true }, @@ -3088,6 +3093,7 @@ "baseUrl": "https://chatgpt.com/backend-api", "compat": { "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsOpenAIGrammarTools": true, "supportsToolSearch": true }, @@ -3149,6 +3155,7 @@ "baseUrl": "https://chatgpt.com/backend-api", "compat": { "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsOpenAIGrammarTools": true, "supportsToolSearch": true }, diff --git a/packages/aikit/test/e2e/__artifacts__/sections.anthropic.json b/packages/aikit/test/e2e/__artifacts__/sections.anthropic.json new file mode 100644 index 00000000..9afb7e9c --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/sections.anthropic.json @@ -0,0 +1,5 @@ +{ + "promptChange": "folded", + "earlierSectionSent": false, + "answer": "bravo" +} \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/sections.github-copilot-responses.json b/packages/aikit/test/e2e/__artifacts__/sections.github-copilot-responses.json new file mode 100644 index 00000000..30a4f390 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/sections.github-copilot-responses.json @@ -0,0 +1,5 @@ +{ + "promptChange": "in place", + "earlierSectionSent": true, + "answer": "bravo" +} \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/sections.google.json b/packages/aikit/test/e2e/__artifacts__/sections.google.json new file mode 100644 index 00000000..9afb7e9c --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/sections.google.json @@ -0,0 +1,5 @@ +{ + "promptChange": "folded", + "earlierSectionSent": false, + "answer": "bravo" +} \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/sections.openai-codex.json b/packages/aikit/test/e2e/__artifacts__/sections.openai-codex.json new file mode 100644 index 00000000..30a4f390 --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/sections.openai-codex.json @@ -0,0 +1,5 @@ +{ + "promptChange": "in place", + "earlierSectionSent": true, + "answer": "bravo" +} \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/sections.openai.json b/packages/aikit/test/e2e/__artifacts__/sections.openai.json new file mode 100644 index 00000000..9afb7e9c --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/sections.openai.json @@ -0,0 +1,5 @@ +{ + "promptChange": "folded", + "earlierSectionSent": false, + "answer": "bravo" +} \ No newline at end of file diff --git a/packages/aikit/test/e2e/__artifacts__/sections.openrouter.json b/packages/aikit/test/e2e/__artifacts__/sections.openrouter.json new file mode 100644 index 00000000..9afb7e9c --- /dev/null +++ b/packages/aikit/test/e2e/__artifacts__/sections.openrouter.json @@ -0,0 +1,5 @@ +{ + "promptChange": "folded", + "earlierSectionSent": false, + "answer": "bravo" +} \ No newline at end of file diff --git a/packages/aikit/test/e2e/sections.e2e.test.ts b/packages/aikit/test/e2e/sections.e2e.test.ts new file mode 100644 index 00000000..bc6be59d --- /dev/null +++ b/packages/aikit/test/e2e/sections.e2e.test.ts @@ -0,0 +1,153 @@ +import { zstdDecompressSync } from "node:zlib"; +import { describe, expect, it } from "vite-plus/test"; +import { llm } from "../../src/llm.ts"; +import * as Message from "../../src/message/message.ts"; +import { + getGitHubCopilotApiKey, + GitHubCopilotOAuthClient, + gitHubCopilotBaseUrl, +} from "../../src/oauth/github/copilot.ts"; +import { complete } from "../../src/stream.ts"; +import { + ANTHROPIC_E2E_MODELS, + anthropicOptions, + describeIfAnthropic, + describeIfGoogle, + describeIfOpenAI, + describeIfOpenAICodex, + describeIfOpenRouter, + getAnthropicModel, + getGoogleModel, + getOpenAICodexModel, + getOpenAIModel, + getOpenRouterModel, + googleOptions, + openaiCodexOptions, + openaiOptions, + openrouterOptions, + type StreamableModel, +} from "../utils/llm.ts"; + +const copilotApiKey = await getGitHubCopilotApiKey(); +const copilotBaseUrl = gitHubCopilotBaseUrl((await new GitHubCopilotOAuthClient().getCredentials()) ?? undefined); +const describeIfGitHubCopilot = copilotApiKey ? describe : describe.skip; + +async function getCopilotModel(modelId: string): Promise { + const model = await llm("github-copilot", modelId); + if (!model) throw new Error(`expected github-copilot model ${modelId}`); + return model as StreamableModel; +} + +/** Records each request body the provider sends (Codex zstd-compresses its own), then forwards it. */ +function recordRequests() { + const bodies: Array> = []; + const record: typeof fetch = (input, init) => { + const body = init?.body; + if (typeof body === "string") bodies.push(JSON.parse(body)); + else if (body instanceof Uint8Array) bodies.push(JSON.parse(zstdDecompressSync(body).toString())); + return fetch(input, init); + }; + return { bodies, fetch: record }; +} + +const codeword = (word: string) => + Message.createSystemMessage({ sections: { codeword: `${word}` } }); + +const ask = (text: string) => + Message.createUserMessage({ role: "user", parts: [{ type: "text", text }], time: { created: Date.now() } }); + +const answer = (message: Message.AssistantMessage) => + message.parts + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join("") + .trim() + .toLowerCase(); + +/* + * A prompt section changed mid-conversation reaches the model on the next request. The first + * turn has the codeword "alpha"; a system message then replaces the section with "bravo", and + * the second turn must answer "bravo". The artifact records whether the change went in place + * (the earlier prompt still sent) or was folded into the leading prompt. + */ +async function answerChangedSection(model: StreamableModel, options: object, name: string) { + const context: Message.Context = { + systemPrompt: "Answer with the codeword from the codeword section when asked. Reply with the codeword only.", + messages: [codeword("alpha"), ask("What is the codeword?")], + }; + + const first = await complete(model, context, options as never); + expect(first.stopReason, first.errorMessage).toBe("stop"); + expect(answer(first)).toContain("alpha"); + context.messages.push(first, codeword("bravo"), ask("What is the codeword now?")); + + const requests = recordRequests(); + const second = await complete(model, context, { + ...options, + factoryOptions: { ...(options as { factoryOptions?: object }).factoryOptions, fetch: requests.fetch }, + } as never); + expect(second.stopReason, second.errorMessage).toBe("stop"); + expect(answer(second)).toContain("bravo"); + + const body = JSON.stringify(requests.bodies.at(-1)); + const artifact = { + promptChange: body.includes('Updated system prompt section \\"codeword\\"') ? "in place" : "folded", + earlierSectionSent: body.includes("alpha"), + answer: "bravo", + }; + await expect(JSON.stringify(artifact, null, "\t")).toMatchFileSnapshot(`./__artifacts__/sections.${name}.json`); + return artifact; +} + +describe("Prompt sections changed mid-conversation", () => { + describeIfOpenAICodex("OpenAI Codex provider", () => { + it("gpt-6-luna sends the change in place", { retry: 2, timeout: 120_000 }, async () => { + const model = await getOpenAICodexModel("gpt-6-luna"); + const artifact = await answerChangedSection(model, openaiCodexOptions(), "openai-codex"); + expect(artifact.promptChange).toBe("in place"); + }); + }); + + describeIfAnthropic("Anthropic provider", () => { + it(ANTHROPIC_E2E_MODELS[0], { retry: 2, timeout: 120_000 }, async () => { + const model = await getAnthropicModel(ANTHROPIC_E2E_MODELS[0]); + const artifact = await answerChangedSection(model, anthropicOptions({ maxTokens: 512 }), "anthropic"); + expect(artifact.promptChange).toBe("folded"); + }); + }); + + describeIfOpenAI("OpenAI provider", () => { + it("gpt-6-luna", { retry: 2, timeout: 120_000 }, async () => { + const model = await getOpenAIModel("gpt-6-luna"); + const artifact = await answerChangedSection(model, openaiOptions(), "openai"); + expect(artifact.promptChange).toBe("folded"); + }); + }); + + describeIfGoogle("Google provider", () => { + it("gemini-3.5-flash", { retry: 2, timeout: 120_000 }, async () => { + const model = await getGoogleModel(); + const artifact = await answerChangedSection(model, googleOptions(), "google"); + expect(artifact.promptChange).toBe("folded"); + }); + }); + + describeIfGitHubCopilot("GitHub Copilot provider", () => { + it("gpt-6-luna sends the change in place on /responses", { retry: 2, timeout: 120_000 }, async () => { + const model = await getCopilotModel("gpt-6-luna"); + const artifact = await answerChangedSection( + model, + { apiKey: copilotApiKey!, baseURL: copilotBaseUrl, reasoning: "low" }, + "github-copilot-responses", + ); + expect(artifact.promptChange).toBe("in place"); + }); + }); + + describeIfOpenRouter("OpenRouter provider", () => { + it("deepseek/deepseek-v4.1-flash", { retry: 2, timeout: 120_000 }, async () => { + const model = await getOpenRouterModel("deepseek/deepseek-v4.1-flash"); + const artifact = await answerChangedSection(model, openrouterOptions(), "openrouter"); + expect(artifact.promptChange).toBe("folded"); + }); + }); +}); diff --git a/packages/aikit/test/transcript.test.ts b/packages/aikit/test/transcript.test.ts index 2d221ea7..13216232 100644 --- a/packages/aikit/test/transcript.test.ts +++ b/packages/aikit/test/transcript.test.ts @@ -1,4 +1,4 @@ -/** Tool changes carried by transcript system messages, from the context down to the Responses wire. */ +/** Prompt and tool changes carried by transcript system messages, from the context down to the Responses wire. */ import Type from "typebox"; import { describe, expect, it } from "vite-plus/test"; import * as Message from "../src/message/message.ts"; @@ -15,6 +15,7 @@ import { } from "./utils/openai-codex.ts"; type ResponsesBody = { + instructions?: string; tools?: Array<{ name: string; description?: string }>; input: Array<{ type?: string; @@ -32,7 +33,7 @@ const baseTool = tool("base_tool"); const dropTool = tool("drop_tool"); const lateTool = tool("late_tool"); -const system = (change: Pick) => +const system = (change: Omit) => Message.createSystemMessage(change); const changesModel = () => openAICodexModels()["gpt-6-luna"]!; @@ -180,7 +181,122 @@ describe("transcript tool changes on the Responses wire", () => { }); }); +describe("transcript prompt changes on the Responses wire", () => { + const update = system({ sections: { codeword: "bravo" } }); + const steps = (model: Model.Info) => + Promise.all( + [[makeUserMessage("first")], [makeUserMessage("first"), update, makeUserMessage("second")]].map((messages) => + capture(model, { + systemPrompt: "base prompt", + messages: [system({ sections: { codeword: "alpha" } }), ...messages], + }), + ), + ); + + it("keeps the leading prompt and sends a section change in place", async () => { + const [before, after] = await steps(changesModel()); + + expect(before?.instructions).toBe("base prompt"); + expect(after?.instructions).toBe("base prompt"); + expectGrowingPrefix([before?.input ?? [], after?.input ?? []]); + expect(after?.input).toMatchObject([ + { role: "developer", content: 'Updated system prompt section "codeword":\n\nalpha' }, + { role: "user" }, + { role: "developer", content: 'Updated system prompt section "codeword":\n\nbravo' }, + { role: "user" }, + ]); + }); + + it("folds every change into the leading prompt on a model without mid-conversation system messages", async () => { + const [, after] = await steps(openAICodexModels()["gpt-5.3-codex-spark"]!); + + expect(after?.instructions).toBe("base prompt\n\nbravo"); + expect(after?.input.map((item) => item.role)).toEqual(["user", "user"]); + }); + + it("sends a prompt update and a tool removal of one message as one developer message", async () => { + const body = await capture(changesModel(), { + systemPrompt: "base prompt", + tools: [baseTool, dropTool], + messages: [ + makeUserMessage("first"), + system({ content: "Be brief.", toolsRemoved: [{ name: "drop_tool" }] }), + makeUserMessage("second"), + ], + }); + + expect(body.input.slice(-2)).toMatchObject([ + { + role: "developer", + content: "Be brief.\n\nThe following tools are no longer available. Do not call them:\n- drop_tool", + }, + { role: "user" }, + ]); + }); +}); + describe("Transcript", () => { + const context: Message.Context = { + systemPrompt: "base", + messages: [ + system({ sections: { a: "1", b: "1" } }), + makeUserMessage("hello"), + system({ content: "also do this" }), + system({ sections: { a: "2", b: null, c: "1" } }), + ], + }; + + it("replays content and sections into the current prompt", () => { + expect(Transcript.getCurrentSystemPrompt(context)).toBe("base\n\nalso do this\n\n2\n\n1"); + expect(Transcript.getCurrentSystemPrompt({ messages: [makeUserMessage("hi")] })).toBe(""); + }); + + it("replays a late full patch on a transcript without a prompt as the prompt", () => { + const late = { messages: [makeUserMessage("old"), system({ sections: { preamble: "You are pi." } })] }; + expect(Transcript.getCurrentSystemPrompt(late)).toBe("You are pi."); + }); + + it("appends a removed section added again", () => { + const readded = { + messages: [ + system({ sections: { a: "", b: "" } }), + system({ sections: { a: null } }), + system({ sections: { a: "again" } }), + ], + }; + expect(Transcript.getCurrentSystemPrompt(readded)).toBe("\n\nagain"); + }); + + it("renders complete prompts and framed updates", () => { + expect(Transcript.renderSystemPrompt({ content: "base", sections: { a: "1", b: null } })).toBe( + "base\n\n1", + ); + expect(Transcript.renderSystemPromptUpdate({ sections: { a: "2", b: null } })).toBe( + 'Updated system prompt section "a":\n\n2\n\nRemoved system prompt section "b".', + ); + }); + + it("resolves the leading prompt per transport", () => { + expect(Transcript.resolveTranscriptPrompt(context, false)).toEqual({ + systemPrompt: "base\n\nalso do this\n\n2\n\n1", + foldedSystemMessages: 3, + }); + expect(Transcript.resolveTranscriptPrompt(context, true)).toEqual({ + systemPrompt: "base", + foldedSystemMessages: 0, + }); + // Without a prompt or initial tools, a leading system message is the prompt. + const { systemPrompt: _, ...headless } = context; + expect(Transcript.resolveTranscriptPrompt(headless, true)).toEqual({ + systemPrompt: "1\n\n1", + foldedSystemMessages: 1, + }); + expect(Transcript.resolveTranscriptPrompt({ ...headless, tools: [baseTool] }, true)).toEqual({ + systemPrompt: "", + foldedSystemMessages: 0, + }); + }); + it("resolves the top-level tool boundary per transport", () => { const messages = [ system({ toolsRemoved: [{ name: "drop_tool" }] }), diff --git a/packages/aikit/test/transform.test.ts b/packages/aikit/test/transform.test.ts index e1f43787..fa542960 100644 --- a/packages/aikit/test/transform.test.ts +++ b/packages/aikit/test/transform.test.ts @@ -696,7 +696,9 @@ describe("convertMessages", () => { }); it("attaches Codex namespace metadata and declares later tools in place", () => { - const codexModel = makeCodexModel({ compat: { supportsAdditionalTools: true } }); + const codexModel = makeCodexModel({ + compat: { supportsMidConvoSystemMessages: true, supportsAdditionalTools: true }, + }); const toolCall: Message.ToolCallCompletedPart = { ...makeCompletedToolCall("call-1|fc-1", "search", [{ type: "text", text: "found" }], { query: "x" }), namespace: "workspace",