diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d892f4d..44b8865d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ This file is the canonical source for unreleased changes and published release n - MCP OAuth: servers sign in with discovery, client registration and PKCE, tokens stored in `~/.codework/mcpauth.json` by server name and URL and refreshed under a lock shared across processes. A server waiting for sign-in reconnects on the next message once its sign-in changes. `auth: { provider }` sends a model provider's credential instead. - The ephemeral `mcp.server.status` event reports each MCP server state change (`connecting`, `connected`, `disconnected`, `failed`, `authenticate`, `disabled`) to every thread using the server. A tool the MCP SDK rejects is skipped instead of failing its server, and listed as `skipped` on the `connected` status. - `toolsearch` turns itself on when a thread has deferred tools or an enabled MCP server with `deferred` exposure; a settings entry still turns it off or on. A search waits up to 10 seconds for such servers still connecting. +- A thread can have its own prompt foundation: `configure({ foundation })` records it in the thread's `agent` document, in place of the one prompt plugins render, and `foundation: null` goes back to that. Other threads keep theirs; a fork starts with the foundation of its cut, and the choice survives a restart. Each change leads the thread's next request with the new foundation, a cache miss once. `Session.create` takes `foundation` for the main thread, and `Session.ThreadInfo.foundation` reports a thread's choice. +- The deferred tools of the catalog reach every request as `deferredTools`, so Anthropic declares them up front and loading one through `toolsearch` keeps the cached prefix. +- Prompt sections reach the model as transcript system messages. A request sends the foundation as its system prompt; the first request of a thread appends a `system` entry declaring every tagged section (``, ``, ``, plugin sections), and a later request appends one only for the sections that changed, a removed section as `null`, together with any tool change. Resume and fork replay these entries, so the model sees what it saw before, and a model that takes system messages in place (OpenAI Codex, OpenAI and GitHub Copilot Responses, Anthropic and GitHub Copilot Claude) keeps its cached prefix when a section changes. #### Breaking Changes @@ -50,6 +53,7 @@ This file is the canonical source for unreleased changes and published release n - Migrations add `session_input.cancelled_seq`, and `session.execution` and `session.restarts`. - Stored context messages are user and assistant messages only; a stored system message fails to decode with "system messages are not stored as entries". - Tests use `gpt-6-luna` in place of the deprecated `gpt-5.5`. +- `session.thread.tools.changed` (`ToolsChanged`) is now `session.thread.system.changed` (`SystemChanged`), and its `system` entry data has an optional `sections` patch. `State.Snapshot.systemPrompt` is now `prompt: { foundation, sections }`. ### @codeworksh/cli @@ -60,6 +64,7 @@ This file is the canonical source for unreleased changes and published release n - Added the `thread.update` RPC (`{ sessionId, threadId?, messageId, delivery?, text? }`), which moves a queued prompt to the other lane or rewrites its text, keeping its place. `delivery: "steer"` also wakes the thread. `updated` is `false` when the prompt was already taken in. - `codework serve` resumes the sessions a crash or a shutdown cut off when it starts. Thread info carries `parked: true` for a thread that restarted too often to resume on its own; its next prompt runs it again. +- `thread.configure` and `session.create` take `runtime.foundation`, a thread's own prompt foundation; `null` on `thread.configure` goes back to the rendered one. `ThreadInfo.foundation` reports it. - `codework run --thread ` continues one thread of `--session`. - `codework thread list ` lists a session's threads with their model, usage and title. `codework thread fork --at ` or `--before ` starts a new thread and prints its id, then for `--before` the text of the prompt it left out. `codework session relink --thread ` moves one thread to another checkout. All three take `--json`. - `ThreadInfo.usage` reports a thread's own cost and tokens. `event.subscribe` and `session.log` take `threadId` to keep only that thread's events. @@ -89,8 +94,10 @@ This file is the canonical source for unreleased changes and published release n - Transcript system messages change the tools mid-conversation: `Message.createSystemMessage({ toolsAdded?, toolsRemoved?, metadata? })`. `Context.tools` is the initial set. OpenAI Codex and GitHub Copilot Responses models that support it declare every tool in place, in `additional_tools` items, and announce removals as text, so the cached prompt prefix holds across tool changes; models with tool search only get later additions as a tool search result; every other model gets the current tools top-level. - `Transcript` helpers: `getCurrentTools`, `getDeclaredTools`, `getToolStateChanges`, `declarationsEqual`, `toToolDeclaration` and `resolveTranscriptTools`. -- Transcript system messages also change the system prompt: `content` adds instructions from that point on, and `sections` replace named sections, `null` removing one. `Context.systemPrompt` is the initial prompt. OpenAI Codex and GitHub Copilot Responses models with `supportsMidConvoSystemMessages` keep the leading prompt and get each change in place as a developer message (`Updated system prompt section "": …`), so the cached prompt prefix holds; every other model gets the current prompt as its system prompt. Helpers: `getCurrentSystemPrompt`, `renderSystemPrompt`, `renderSystemPromptUpdate` and `resolveTranscriptPrompt`. -- `supportsMidConvoSystemMessages` compat flag, set by `modelgen` for the OpenAI Codex and GitHub Copilot Responses models that accept developer messages mid-conversation. In-place tool declarations (`supportsAdditionalTools`) require it. +- Transcript system messages also change the system prompt: `content` adds instructions from that point on, and `sections` replace named sections, `null` removing one. `Context.systemPrompt` is the initial prompt. Models with `supportsMidConvoSystemMessages` keep the leading prompt and get each change in place as a system or developer message (`Updated system prompt section "": …`), so the cached prompt prefix holds; every other model gets the current prompt as its system prompt. On Anthropic a change is sent before the next assistant message or at the end, since the Messages API takes a system message only after a user message and never between a tool call and its result; one before the first reply joins the top-level system prompt, where it caches. Helpers: `getCurrentSystemPrompt`, `renderSystemPrompt`, `renderSystemPromptUpdate` and `resolveTranscriptPrompt`. +- `supportsMidConvoSystemMessages` compat flag, set by `modelgen` for the models that accept system messages mid-conversation: OpenAI Codex, OpenAI and GitHub Copilot Responses (gpt-5.4 and later), and Anthropic and GitHub Copilot Claude (claude-opus-4.8, -5 and -5.5, claude-sonnet-5.5, claude-haiku-5.5, claude-fable-5 and -5.1, claude-mythos-5 and -5.1). The baseline catalog carries it. In-place tool declarations (`supportsAdditionalTools`) require it. +- Anthropic models take tool changes in place (`supportsAdditionalTools`, set for direct Anthropic only, as Copilot rejects them): the request declares its initial tools, then every other tool the thread can reach deferred (`defer_loading`), the catalog passed as the new `Context.deferredTools` and every tool the transcript declared, and a later system message adds or removes one by reference. Loading a deferred tool then keeps the cached prompt prefix. +- `Transcript.getCurrentSections`: the sections a transcript declares after replaying its system messages. `Transcript.diffSections(previous, current)`: the patch between two section sets, `null` removing one. - `filterGitHubCopilotModels` keeps the catalog models a Copilot login can reach. - GitHub Copilot GPT models turn reasoning off with `reasoning.effort: "none"` where OpenAI accepts it, such as gpt-5.3-codex, gpt-5.6-luna and gpt-6-luna. diff --git a/packages/aikit/src/cli/modelgen.ts b/packages/aikit/src/cli/modelgen.ts index 68fe348d..86475aac 100644 --- a/packages/aikit/src/cli/modelgen.ts +++ b/packages/aikit/src/cli/modelgen.ts @@ -353,6 +353,9 @@ function applyOpenAiMetadata(model: Model.Info, source: ModelsDevModel | undefin } // models.dev reports the input sub-limit as gpt-5-pro's output. if (id === "gpt-5-pro") model.maxTokens = OPENAI_MAX_TOKENS; + if (model.api?.method === Model.APIMethodEnum.responses && OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS.has(id)) { + mergeCompat(model, { supportsMidConvoSystemMessages: true }); + } } model.cost.serviceTierMultipliers = serviceTierMultipliers(model, source); @@ -391,6 +394,9 @@ function isAnthropicMessages(model: Model.Info): boolean { // The Messages API rejects `temperature` on these. const ANTHROPIC_NO_TEMPERATURE = /opus-4[-.][78]|opus-5|(sonnet|haiku)-5[-.]5/; +// These take system messages inside the conversation, verified live against the Messages API. +const ANTHROPIC_MID_CONVO_SYSTEM_MESSAGES = + /^claude-(?:opus-(?:4[.-]8|5(?:[.-]5)?)|(?:sonnet|haiku)-5[.-]5|(?:fable|mythos)-5(?:[.-]1)?)(?:-\d{8})?$/; // Checked live against the Messages API: effort levels each adaptive family // accepts, the 5.5 and Fable models that reject `thinking: {type: "disabled"}`, and @@ -422,6 +428,10 @@ function applyAnthropicMetadata(model: Model.Info): void { mergeThinkingLevelMap(model, { off: null }); } if (ANTHROPIC_NO_TEMPERATURE.test(id)) mergeCompat(model, { supportsTemperature: false }); + // Copilot passes these system messages through but rejects tool changes in them. + if (model.provider.id === "anthropic" && ANTHROPIC_MID_CONVO_SYSTEM_MESSAGES.test(id)) { + mergeCompat(model, { supportsMidConvoSystemMessages: true, supportsAdditionalTools: true }); + } } // @@ -701,6 +711,9 @@ export function githubCopilotBuiltInModels(provider: ModelsDevProvider | undefin if (method === Model.APIMethodEnum.responses && OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS.has(id)) { mergeCompat(info, { supportsMidConvoSystemMessages: true }); } + if (method === Model.APIMethodEnum.messages && ANTHROPIC_MID_CONVO_SYSTEM_MESSAGES.test(id)) { + mergeCompat(info, { supportsMidConvoSystemMessages: true }); + } if (method === Model.APIMethodEnum.responses && GITHUB_COPILOT_ADDITIONAL_TOOLS_MODEL_IDS.has(id)) { mergeCompat(info, { supportsAdditionalTools: true }); } diff --git a/packages/aikit/src/llm/stream.ts b/packages/aikit/src/llm/stream.ts index b3116ca0..48d9d305 100644 --- a/packages/aikit/src/llm/stream.ts +++ b/packages/aikit/src/llm/stream.ts @@ -1,7 +1,6 @@ import { APICallError } from "@ai-sdk/provider"; import { RetryError, streamText, type TextStreamPart, type ToolSet } from "ai"; import * as Message from "../message/message.ts"; -import * as Transcript from "../message/transcript.ts"; import * as Model from "../model/model.ts"; import { AssistantMessageEventStream } from "../utils/eventstream.ts"; import { compact } from "../utils/helpers.ts"; @@ -18,6 +17,7 @@ import { convertMessages, isOwnResponses, convertTools, + requestTools, createAssistantMessage, encodeOpenAIReasoningSignature, grammarToolInputProperties, @@ -416,7 +416,8 @@ export const stream: Protocol.StreamFunction @@ -433,7 +434,9 @@ export const stream: Protocol.StreamFunction>[0] = compact({ model: languageModel, // Our own Responses transport receives the system prompt among the messages. - system: isOwnResponses(model, runtimeOptions.method) ? undefined : leadingSystemPrompt(context), + system: isOwnResponses(model, runtimeOptions.method) + ? undefined + : leadingSystemPrompt(context, model, runtimeOptions.method), messages, // Prompt and tool changes sent in place travel as system messages. allowSystemInMessages: messages.some((message) => message.role === "system") || undefined, diff --git a/packages/aikit/src/llm/transform.ts b/packages/aikit/src/llm/transform.ts index bf161bc6..167b3937 100644 --- a/packages/aikit/src/llm/transform.ts +++ b/packages/aikit/src/llm/transform.ts @@ -485,16 +485,85 @@ export function transcriptToolSupport( } /** - * Whether prompt changes go in place as developer messages. Only our own Responses transport - * sends them; the AI SDK adapters take the current prompt as the leading system prompt. + * Whether prompt changes go in place as system messages. Our own Responses transport and the + * AI SDK's OpenAI Responses and Anthropic Messages adapters keep them where they stand; Chat + * Completions takes the current prompt as the leading system prompt. */ export function supportsMidConvoSystemMessages(model: Model.Info, method: Model.APIMethodEnum | undefined): boolean { - return isOwnResponses(model, method) && model.compat?.supportsMidConvoSystemMessages === true; + if (model.compat?.supportsMidConvoSystemMessages !== true) return false; + return isOwnResponses(model, method) || !isChatCompletions(model, resolveMethod(model, method)); } -/** The prompt the AI SDK adapters take as `system`: the current one, every change folded in. */ -export function leadingSystemPrompt(context: Message.Context): string | undefined { - return Transcript.getCurrentSystemPrompt(context) || undefined; +/** + * Anthropic with tool changes in place: every tool the thread can reach is declared up front, the + * ones beyond its initial set deferred, and later changes name them by reference. Adding a tool + * to the top-level list costs the cached prefix; referencing a deferred one does not. + */ +function anthropicToolReferences(model: Model.Info, method: Model.APIMethodEnum, restricted: boolean): boolean { + return ( + !restricted && + isAnthropicWire(model, method) && + supportsMidConvoSystemMessages(model, method) && + model.compat?.supportsAdditionalTools === true + ); +} + +/** Messages before the first reply: on Anthropic their system messages join the leading prompt and tools. */ +function beforeFirstReply(messages: readonly Message.Message[]): Message.Message[] { + const reply = messages.findIndex((message) => message.role === "assistant"); + return reply === -1 ? [...messages] : messages.slice(0, reply); +} + +/** + * The tools the request declares top-level, and which of them are deferred. Usually every tool + * callable now. With Anthropic tool references, the initial set, then the deferred catalog and + * every other tool the transcript declared, deferred, in an order that holds across requests. + */ +export function requestTools( + context: Message.Context, + model: Model.Info, + methodOverride?: Model.APIMethodEnum, + activeTools?: readonly string[], +): { tools: Message.Tool[]; deferred: ReadonlySet } { + if (!anthropicToolReferences(model, resolveMethod(model, methodOverride), activeTools !== undefined)) { + return { tools: Transcript.getCurrentTools(context), deferred: new Set() }; + } + const initial = Transcript.getCurrentTools({ ...context, messages: beforeFirstReply(context.messages) }); + const latest = new Map(Transcript.getDeclaredTools(context).map((tool) => [tool.name, tool])); + const tools = new Map(initial.map((tool) => [tool.name, tool])); + const deferred = new Set(); + for (const tool of [...(context.deferredTools ?? []), ...latest.values()]) { + if (tools.has(tool.name)) continue; + tools.set(tool.name, latest.get(tool.name) ?? tool); + deferred.add(tool.name); + } + return { tools: [...tools.values()], deferred }; +} + +/** A later system message's tool changes as Anthropic tool references; a redefinition needs no removal. */ +function toolReferences(message: Message.SystemMessage) { + const added = new Set((message.toolsAdded ?? []).map((tool) => tool.name)); + return [ + ...(message.toolsRemoved ?? []) + .filter((tool) => !added.has(tool.name)) + .map((tool) => ({ type: "tool_removal" as const, toolName: tool.name })), + ...[...added].map((name) => ({ type: "tool_addition" as const, toolName: name })), + ]; +} + +/** + * The prompt the AI SDK adapters take as `system`: the leading one when changes go in place, + * else the current one, every change folded in. + */ +export function leadingSystemPrompt( + context: Message.Context, + model: Model.Info, + method: Model.APIMethodEnum | undefined, +): string | undefined { + return ( + Transcript.resolveTranscriptPrompt(context, supportsMidConvoSystemMessages(model, method)).systemPrompt || + undefined + ); } /** A tool as the language model receives it, for declaring it inside the prompt. */ @@ -557,11 +626,32 @@ export function convertMessages( messages.push({ role: "system", content: sanitizeSurrogates(prompt.systemPrompt) }); } + // Every system message goes as an update, the first declaration too: a section a model read + // whole does not yield to a later update of it, one it read as an update does. On Anthropic, + // one before the first reply joins the leading prompt, where it caches: a trailing system + // message's cache entry is never read once a reply follows it. Anthropic takes a later one + // only after a user message, and never between a tool call and its result: each is held for + // the next assistant message, or the end. + const anthropic = isAnthropicWire(model, method); + const references = anthropicToolReferences(model, method, activeTools !== undefined); + const leading: ModelMessage[] = []; + const held: ModelMessage[] = []; + let replied = false; let systemIndex = 0; for (const msg of transformedMessages) { if (msg.role === "system") { const index = systemIndex++; const update = index < prompt.foldedSystemMessages ? "" : Transcript.renderSystemPromptUpdate(msg); + // The tools of one before the first reply belong to the initial set. + const toolChanges = references && replied ? toolReferences(msg) : []; + if (toolChanges.length > 0) { + held.push({ + role: "system", + content: sanitizeSurrogates(update), + providerOptions: { [Model.optionsKey(model)]: { toolChanges } }, + }); + continue; + } const folded = index < transcript.foldedSystemMessages; const change = systemChangeMessage( update, @@ -571,17 +661,22 @@ export function convertMessages( model, method, ); - if (change) messages.push(change); + if (change) (!anthropic ? messages : replied ? held : leading).push(change); continue; } if (msg.role === "user") { messages.push(...userContent(msg.parts)); continue; } - messages.push(...assistantMessages(msg, model, method)); + const converted = assistantMessages(msg, model, method); + if (converted.length > 0) { + replied = true; + messages.push(...held.splice(0)); + } + messages.push(...converted); } - return messages; + return [...leading, ...messages, ...held]; } /** @@ -619,17 +714,23 @@ export function convertTools( tools?: Message.Tool[], model?: Model.Info, method?: Model.APIMethodEnum, + deferred: ReadonlySet = new Set(), ): ToolSet | undefined { if (!tools || tools.length === 0) return; const result: ToolSet = {}; for (const tool of tools) { const { inputSchema, strict, providerOptions } = toolWire(tool, model, method); + const key = model && deferred.has(tool.name) ? Model.optionsKey(model) : undefined; + const options = + key === undefined + ? providerOptions + : { ...providerOptions, [key]: { ...providerOptions?.[key], deferLoading: true } }; result[tool.name] = { description: tool.description, inputSchema: jsonSchema(inputSchema as any), ...(strict ? { strict } : {}), - ...(providerOptions ? { providerOptions } : {}), + ...(options ? { providerOptions: options } : {}), }; } return result; diff --git a/packages/aikit/src/message/message.ts b/packages/aikit/src/message/message.ts index 026000a1..87f99629 100644 --- a/packages/aikit/src/message/message.ts +++ b/packages/aikit/src/message/message.ts @@ -333,6 +333,8 @@ export const ContextSchema = Type.Object({ systemPrompt: Type.Optional(Type.String()), messages: Type.Array(MessageSchema), tools: Type.Optional(Type.Array(ToolSchema)), + /** Tools a later system message may add, for transports that must declare every tool up front. */ + deferredTools: Type.Optional(Type.Array(ToolSchema)), }); export type Context = Static; diff --git a/packages/aikit/src/message/transcript.ts b/packages/aikit/src/message/transcript.ts index 0902408d..a604d6db 100644 --- a/packages/aikit/src/message/transcript.ts +++ b/packages/aikit/src/message/transcript.ts @@ -45,24 +45,43 @@ export function renderSystemPromptUpdate(message: PromptChange): string { } /** - * The prompt after every transcript change in order: content accumulates, sections are - * replaced by name, and a removed section added again goes last. + * The sections after every transcript change in order: replaced by name, removed by `null`, and + * a removed section added again goes last. */ -export function getCurrentSystemPrompt(context: Pick): string { - const content: string[] = []; +export function getCurrentSections(messages: readonly Message[]): Record { const sections = new Map(); - const changes: PromptChange[] = [ - { content: context.systemPrompt ?? "" }, - ...context.messages.filter(isSystemMessage), - ]; - for (const message of changes) { - if (message.content) content.push(message.content); + for (const message of messages) { + if (!isSystemMessage(message)) continue; for (const [name, value] of Object.entries(message.sections ?? {})) { if (value === null) sections.delete(name); else sections.set(name, value); } } - return renderSystemPrompt({ content: content.join("\n\n"), sections: Object.fromEntries(sections) }); + return Object.fromEntries(sections); +} + +/** + * The patch that brings `previous` sections to `current`, `null` removing one; `undefined` when + * they match. + */ +export function diffSections( + previous: Readonly>, + current: Readonly>, +): Record | undefined { + const patch: Record = {}; + for (const [name, text] of Object.entries(current)) if (previous[name] !== text) patch[name] = text; + for (const name of Object.keys(previous)) if (!Object.hasOwn(current, name)) patch[name] = null; + return Object.keys(patch).length === 0 ? undefined : patch; +} + +/** The prompt after every transcript change in order: content accumulates, sections as {@link getCurrentSections}. */ +export function getCurrentSystemPrompt(context: Pick): string { + const content: string[] = [context.systemPrompt ?? ""]; + for (const message of context.messages) if (isSystemMessage(message)) content.push(message.content ?? ""); + return renderSystemPrompt({ + content: content.filter((part) => part.length > 0).join("\n\n"), + sections: getCurrentSections(context.messages), + }); } export interface TranscriptPrompt { diff --git a/packages/aikit/src/model/baseline.json b/packages/aikit/src/model/baseline.json index d925d91f..a5ca950f 100644 --- a/packages/aikit/src/model/baseline.json +++ b/packages/aikit/src/model/baseline.json @@ -8,7 +8,9 @@ }, "baseUrl": "https://api.anthropic.com/v1", "compat": { - "forceAdaptiveThinking": true + "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true }, "contextWindow": 1000000, "cost": { @@ -50,7 +52,9 @@ }, "baseUrl": "https://api.anthropic.com/v1", "compat": { - "forceAdaptiveThinking": true + "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true }, "contextWindow": 1000000, "cost": { @@ -161,6 +165,8 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -367,6 +373,8 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -409,6 +417,8 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -451,6 +461,8 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -647,6 +659,8 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -1348,53 +1362,6 @@ "xhigh": "xhigh" } }, - "gpt-5.2-chat-latest": { - "api": { - "id": "gpt-5.2-chat-latest", - "method": "responses", - "url": "https://api.openai.com/v1" - }, - "baseUrl": "https://api.openai.com/v1", - "contextWindow": 128000, - "cost": { - "cacheRead": 0.175, - "cacheWrite": 0, - "input": 1.75, - "output": 14, - "serviceTierMultipliers": { - "flex": 0.5, - "priority": 2 - } - }, - "id": "gpt-5.2-chat-latest", - "input": [ - "text", - "image" - ], - "maxTokens": 16384, - "name": "GPT-5.2 Chat", - "npm": "@ai-sdk/openai", - "protocol": "openai", - "provider": { - "env": [ - "OPENAI_API_KEY" - ], - "id": "openai", - "name": "OpenAI", - "source": "api" - }, - "providerOptionsKey": "openai", - "reasoning": true, - "thinkingLevelMap": { - "high": null, - "low": null, - "max": null, - "medium": "medium", - "minimal": null, - "off": null, - "xhigh": "xhigh" - } - }, "gpt-5.2-pro": { "api": { "id": "gpt-5.2-pro", @@ -1442,48 +1409,6 @@ "xhigh": "xhigh" } }, - "gpt-5.3-chat-latest": { - "api": { - "id": "gpt-5.3-chat-latest", - "method": "responses", - "url": "https://api.openai.com/v1" - }, - "baseUrl": "https://api.openai.com/v1", - "contextWindow": 128000, - "cost": { - "cacheRead": 0.175, - "cacheWrite": 0, - "input": 1.75, - "output": 14, - "serviceTierMultipliers": { - "flex": 0.5, - "priority": 2 - } - }, - "id": "gpt-5.3-chat-latest", - "input": [ - "text", - "image" - ], - "maxTokens": 16384, - "name": "GPT-5.3 Chat (latest)", - "npm": "@ai-sdk/openai", - "protocol": "openai", - "provider": { - "env": [ - "OPENAI_API_KEY" - ], - "id": "openai", - "name": "OpenAI", - "source": "api" - }, - "providerOptionsKey": "openai", - "reasoning": false, - "thinkingLevelMap": { - "off": null, - "xhigh": "xhigh" - } - }, "gpt-5.3-codex": { "api": { "id": "gpt-5.3-codex", @@ -1585,6 +1510,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.25, @@ -1641,6 +1569,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 400000, "cost": { "cacheRead": 0.075, @@ -1735,6 +1666,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 1050000, "cost": { "cacheRead": 0, @@ -1791,6 +1725,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.5, @@ -1903,6 +1840,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.02, @@ -1959,6 +1899,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.4, @@ -2015,6 +1958,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.2, @@ -2071,6 +2017,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 1, @@ -2128,6 +2077,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.01, @@ -2184,6 +2136,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.2, @@ -2240,6 +2195,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.1, @@ -2290,6 +2248,65 @@ "xhigh": "xhigh" } }, + "gpt-6.1-terra": { + "api": { + "id": "gpt-6.1-terra", + "method": "responses", + "url": "https://api.openai.com/v1" + }, + "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, + "contextWindow": 1050000, + "cost": { + "cacheRead": 0.05, + "cacheWrite": 1.25, + "input": 1, + "output": 5, + "serviceTierMultipliers": { + "flex": 0.5, + "priority": 2 + }, + "tiers": [ + { + "cacheRead": 0.1, + "cacheWrite": 2.5, + "input": 2, + "inputTokensAbove": 272000, + "output": 7.5 + } + ] + }, + "id": "gpt-6.1-terra", + "input": [ + "text", + "image" + ], + "maxTokens": 128000, + "name": "GPT-6.1 Terra", + "npm": "@ai-sdk/openai", + "protocol": "openai", + "provider": { + "env": [ + "OPENAI_API_KEY" + ], + "id": "openai", + "name": "OpenAI", + "source": "api" + }, + "providerOptionsKey": "openai", + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": "max", + "medium": "medium", + "minimal": null, + "off": "none", + "xhigh": "xhigh" + } + }, "gpt-daybreak-blue-latest": { "api": { "id": "gpt-daybreak-blue-latest", @@ -3711,42 +3728,6 @@ "xhigh": "xhigh" } }, - "arcee-ai/trinity-large-thinking": { - "api": { - "id": "arcee-ai/trinity-large-thinking", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 262144, - "cost": { - "cacheRead": 0.06, - "cacheWrite": 0, - "input": 0.25, - "output": 0.8 - }, - "id": "arcee-ai/trinity-large-thinking", - "input": [ - "text" - ], - "maxTokens": 80000, - "name": "Arcee AI: Trinity Large Thinking", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "off": null - } - }, "bytedance-seed/seed-1.6": { "api": { "id": "bytedance-seed/seed-1.6", @@ -4163,18 +4144,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 163840, + "contextWindow": 128000, "cost": { "cacheRead": 0, "cacheWrite": 0, - "input": 0.32, - "output": 0.89 + "input": 0.2574, + "output": 1.0287 }, "id": "deepseek/deepseek-chat", "input": [ "text" ], - "maxTokens": 16384, + "maxTokens": 16000, "name": "DeepSeek: DeepSeek V3", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -4240,7 +4221,7 @@ "input": [ "text" ], - "maxTokens": 32768, + "maxTokens": 147456, "name": "DeepSeek: DeepSeek V3.1", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -4255,42 +4236,6 @@ "providerOptionsKey": "openrouter", "reasoning": true }, - "deepseek/deepseek-r1": { - "api": { - "id": "deepseek/deepseek-r1", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 64000, - "cost": { - "cacheRead": 0, - "cacheWrite": 0, - "input": 0.7, - "output": 2.5 - }, - "id": "deepseek/deepseek-r1", - "input": [ - "text" - ], - "maxTokens": 16000, - "name": "DeepSeek: R1", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "off": null - } - }, "deepseek/deepseek-r1-0528": { "api": { "id": "deepseek/deepseek-r1-0528", @@ -4367,18 +4312,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 131072, + "contextWindow": 163840, "cost": { - "cacheRead": 0.028, + "cacheRead": 0.13, "cacheWrite": 0, - "input": 0.28, - "output": 0.42 + "input": 0.26, + "output": 0.38 }, "id": "deepseek/deepseek-v3.2", "input": [ "text" ], - "maxTokens": 65536, + "maxTokens": 147456, "name": "DeepSeek: DeepSeek V3.2", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -4477,10 +4422,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.042, + "cacheRead": 0.0238, "cacheWrite": 0, - "input": 0.15, - "output": 0.15 + "input": 0.119, + "output": 0.238 }, "id": "deepseek/deepseek-v4-flash-0731", "input": [ @@ -4560,18 +4505,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 1024000, + "contextWindow": 1048576, "cost": { - "cacheRead": 0.079605, + "cacheRead": 0.135, "cacheWrite": 0, - "input": 0.95526, - "output": 1.91052 + "input": 1.6, + "output": 3.2 }, "id": "deepseek/deepseek-v4-pro", "input": [ "text" ], - "maxTokens": 384000, + "maxTokens": 393216, "name": "DeepSeek: DeepSeek V4 Pro 0423", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -4602,7 +4547,7 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 1024000, + "contextWindow": 1048576, "cost": { "cacheRead": 0.022, "cacheWrite": 0, @@ -4613,7 +4558,7 @@ "input": [ "text" ], - "maxTokens": 384000, + "maxTokens": 393216, "name": "DeepSeek: DeepSeek V4 Pro 0813", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -4646,10 +4591,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.0016, + "cacheRead": 0.006, "cacheWrite": 0, - "input": 0.0464, - "output": 0.08 + "input": 0.045, + "output": 0.5 }, "id": "deepseek/deepseek-v4.1-flash", "input": [ @@ -5781,7 +5726,7 @@ "cacheRead": 0.015, "cacheWrite": 0, "input": 0.06, - "output": 0.25 + "output": 0.15 }, "id": "ibm-granite/granite-4.2-8b", "input": [ @@ -5895,25 +5840,67 @@ "xhigh": null } }, - "inclusionai/ling-3.0-flash": { + "inception/mercury-3": { "api": { - "id": "inclusionai/ling-3.0-flash", + "id": "inception/mercury-3", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 262144, + "contextWindow": 1000000, "cost": { - "cacheRead": 0.0042, + "cacheRead": 0.01, "cacheWrite": 0, - "input": 0.021, - "output": 0.063 + "input": 0.1, + "output": 0.35 }, - "id": "inclusionai/ling-3.0-flash", + "id": "inception/mercury-3", "input": [ "text" ], - "maxTokens": 32768, + "maxTokens": 131072, + "name": "Inception: Mercury 3", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": null, + "medium": "medium", + "minimal": null, + "off": "none", + "xhigh": null + } + }, + "inclusionai/ling-3.0-flash": { + "api": { + "id": "inclusionai/ling-3.0-flash", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 262144, + "cost": { + "cacheRead": 0.0042, + "cacheWrite": 0, + "input": 0.021, + "output": 0.063 + }, + "id": "inclusionai/ling-3.0-flash", + "input": [ + "text" + ], + "maxTokens": 32768, "name": "inclusionAI: Ling 3.0 Flash", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -5969,7 +5956,7 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 262144, + "contextWindow": 1000000, "cost": { "cacheRead": 0.015, "cacheWrite": 0, @@ -5980,7 +5967,7 @@ "input": [ "text" ], - "maxTokens": 235929, + "maxTokens": 262144, "name": "inclusionAI: Ling 3.1 Flash", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -5995,6 +5982,42 @@ "providerOptionsKey": "openrouter", "reasoning": true }, + "liquid/lfm-2.5-2.6b": { + "api": { + "id": "liquid/lfm-2.5-2.6b", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 128000, + "cost": { + "cacheRead": 0.04, + "cacheWrite": 0, + "input": 0.35, + "output": 0.5 + }, + "id": "liquid/lfm-2.5-2.6b", + "input": [ + "text" + ], + "maxTokens": 8192, + "name": "LiquidAI: LFM2.5-2.6B", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "off": null + } + }, "liquid/lfm-2.5-2.6b:free": { "api": { "id": "liquid/lfm-2.5-2.6b:free", @@ -6544,7 +6567,7 @@ "text", "image" ], - "maxTokens": 943718, + "maxTokens": 131072, "name": "Meta: Muse Spark Open", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -6822,6 +6845,49 @@ "providerOptionsKey": "openrouter", "reasoning": true }, + "minimax/minimax-m3.1-flash": { + "api": { + "id": "minimax/minimax-m3.1-flash", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 1048576, + "cost": { + "cacheRead": 0.015, + "cacheWrite": 0, + "input": 0.15, + "output": 0.6 + }, + "id": "minimax/minimax-m3.1-flash", + "input": [ + "text", + "image" + ], + "maxTokens": 524288, + "name": "MiniMax: MiniMax M3.1 Flash", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": "max", + "medium": "medium", + "minimal": null, + "off": null, + "xhigh": "xhigh" + } + }, "mistralai/codestral-2508": { "api": { "id": "mistralai/codestral-2508", @@ -7101,8 +7167,8 @@ "cost": { "cacheRead": 0.07, "cacheWrite": 0, - "input": 0.68, - "output": 2.09 + "input": 0.7, + "output": 2.1 }, "id": "mistralai/mistral-large-4-0", "input": [ @@ -7394,12 +7460,12 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 256000, + "contextWindow": 128000, "cost": { "cacheRead": 0, "cacheWrite": 0, - "input": 0.09375, - "output": 0.25 + "input": 0.075, + "output": 0.2 }, "id": "mistralai/mistral-small-3.2-24b-instruct", "input": [ @@ -7666,10 +7732,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0.18, + "cacheRead": 0.15, "cacheWrite": 0, - "input": 0.6712, - "output": 3.35 + "input": 0.6, + "output": 3.5 }, "id": "moonshotai/kimi-k2.7-code", "input": [ @@ -7703,10 +7769,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.21, + "cacheRead": 0.182, "cacheWrite": 0, - "input": 0.239, - "output": 11.936 + "input": 0.192, + "output": 15 }, "id": "moonshotai/kimi-k3", "input": [ @@ -7737,6 +7803,49 @@ "xhigh": null } }, + "moonshotai/kimi-k3.1-pro": { + "api": { + "id": "moonshotai/kimi-k3.1-pro", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 1048576, + "cost": { + "cacheRead": 0.2, + "cacheWrite": 0, + "input": 2, + "output": 10 + }, + "id": "moonshotai/kimi-k3.1-pro", + "input": [ + "text", + "image" + ], + "maxTokens": 943718, + "name": "MoonshotAI: Kimi K3.1 Pro", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": "max", + "medium": null, + "minimal": null, + "off": "none", + "xhigh": null + } + }, "nex-agi/nex-n2.5-pro": { "api": { "id": "nex-agi/nex-n2.5-pro", @@ -7789,7 +7898,7 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0, + "cacheRead": 0.012, "cacheWrite": 0, "input": 0.06, "output": 0.24 @@ -7854,18 +7963,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 262144, + "contextWindow": 1048576, "cost": { - "cacheRead": 0, + "cacheRead": 0.15, "cacheWrite": 0, - "input": 0.08, - "output": 0.45 + "input": 0.3, + "output": 2.4 }, "id": "nvidia/nemotron-3-super-120b-a12b", "input": [ "text" ], - "maxTokens": 235929, + "maxTokens": 262144, "name": "NVIDIA: Nemotron 3 Super", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -7889,48 +7998,6 @@ "xhigh": null } }, - "nvidia/nemotron-3-super-120b-a12b:free": { - "api": { - "id": "nvidia/nemotron-3-super-120b-a12b:free", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 262144, - "cost": { - "cacheRead": 0, - "cacheWrite": 0, - "input": 0, - "output": 0 - }, - "id": "nvidia/nemotron-3-super-120b-a12b:free", - "input": [ - "text" - ], - "maxTokens": 235929, - "name": "NVIDIA: Nemotron 3 Super (free)", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "high": null, - "low": "low", - "max": null, - "medium": "medium", - "minimal": null, - "off": "none", - "xhigh": null - } - }, "nvidia/nemotron-3-ultra-550b-a55b": { "api": { "id": "nvidia/nemotron-3-ultra-550b-a55b", @@ -8090,10 +8157,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0.08, + "cacheRead": 0.02, "cacheWrite": 0, - "input": 0.1, - "output": 0.4 + "input": 0.15, + "output": 0.5 }, "id": "nvidia/nemotron-3.5-super-vl", "input": [ @@ -8149,39 +8216,6 @@ "providerOptionsKey": "openrouter", "reasoning": true }, - "openai/gpt-3.5-turbo": { - "api": { - "id": "openai/gpt-3.5-turbo", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 16385, - "cost": { - "cacheRead": 0, - "cacheWrite": 0, - "input": 0.5, - "output": 1.5 - }, - "id": "openai/gpt-3.5-turbo", - "input": [ - "text" - ], - "maxTokens": 4096, - "name": "OpenAI: GPT-3.5 Turbo", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": false - }, "openai/gpt-3.5-turbo-0613": { "api": { "id": "openai/gpt-3.5-turbo-0613", @@ -8281,48 +8315,14 @@ "providerOptionsKey": "openrouter", "reasoning": false }, - "openai/gpt-4-turbo": { + "openai/gpt-4.1": { "api": { - "id": "openai/gpt-4-turbo", + "id": "openai/gpt-4.1", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 128000, - "cost": { - "cacheRead": 0, - "cacheWrite": 0, - "input": 10, - "output": 30 - }, - "id": "openai/gpt-4-turbo", - "input": [ - "text", - "image" - ], - "maxTokens": 4096, - "name": "OpenAI: GPT-4 Turbo", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": false - }, - "openai/gpt-4.1": { - "api": { - "id": "openai/gpt-4.1", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 1047576, + "contextWindow": 1047576, "cost": { "cacheRead": 0.5, "cacheWrite": 0, @@ -8392,7 +8392,7 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1047576, "cost": { - "cacheRead": 0.025, + "cacheRead": 0.03, "cacheWrite": 0, "input": 0.1, "output": 0.4 @@ -8402,7 +8402,7 @@ "image", "text" ], - "maxTokens": 32768, + "maxTokens": 942818, "name": "OpenAI: GPT-4.1 Nano", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -10193,6 +10193,110 @@ "xhigh": "xhigh" } }, + "openai/gpt-6.1-terra": { + "api": { + "id": "openai/gpt-6.1-terra", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 1050000, + "cost": { + "cacheRead": 0.05, + "cacheWrite": 1.25, + "input": 1, + "output": 5, + "tiers": [ + { + "cacheRead": 0.1, + "cacheWrite": 2.5, + "input": 2, + "inputTokensAbove": 272000, + "output": 7.5 + } + ] + }, + "id": "openai/gpt-6.1-terra", + "input": [ + "image", + "text" + ], + "maxTokens": 128000, + "name": "OpenAI: GPT-6.1 Terra", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": "max", + "medium": "medium", + "minimal": null, + "off": "none", + "xhigh": "xhigh" + } + }, + "openai/gpt-6.1-terra-pro": { + "api": { + "id": "openai/gpt-6.1-terra-pro", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 1050000, + "cost": { + "cacheRead": 0.05, + "cacheWrite": 1.25, + "input": 1, + "output": 5, + "tiers": [ + { + "cacheRead": 0.1, + "cacheWrite": 2.5, + "input": 2, + "inputTokensAbove": 272000, + "output": 7.5 + } + ] + }, + "id": "openai/gpt-6.1-terra-pro", + "input": [ + "image", + "text" + ], + "maxTokens": 128000, + "name": "OpenAI: GPT-6.1 Terra Pro", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": "max", + "medium": "medium", + "minimal": null, + "off": null, + "xhigh": "xhigh" + } + }, "openai/gpt-audio": { "api": { "id": "openai/gpt-audio", @@ -10413,43 +10517,6 @@ "off": null } }, - "openai/o1": { - "api": { - "id": "openai/o1", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, - "cost": { - "cacheRead": 7.5, - "cacheWrite": 0, - "input": 15, - "output": 60 - }, - "id": "openai/o1", - "input": [ - "text", - "image" - ], - "maxTokens": 100000, - "name": "OpenAI: o1", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "off": null - } - }, "openai/o3": { "api": { "id": "openai/o3", @@ -10487,84 +10554,6 @@ "off": null } }, - "openai/o3-mini": { - "api": { - "id": "openai/o3-mini", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, - "cost": { - "cacheRead": 0.55, - "cacheWrite": 0, - "input": 1.1, - "output": 4.4 - }, - "id": "openai/o3-mini", - "input": [ - "text" - ], - "maxTokens": 100000, - "name": "OpenAI: o3 Mini", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "off": null - } - }, - "openai/o3-mini-high": { - "api": { - "id": "openai/o3-mini-high", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, - "cost": { - "cacheRead": 0.55, - "cacheWrite": 0, - "input": 1.1, - "output": 4.4 - }, - "id": "openai/o3-mini-high", - "input": [ - "text" - ], - "maxTokens": 100000, - "name": "OpenAI: o3 Mini High", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "high": "high", - "low": null, - "max": null, - "medium": null, - "minimal": null, - "off": null, - "xhigh": null - } - }, "openai/o3-pro": { "api": { "id": "openai/o3-pro", @@ -10602,64 +10591,27 @@ "off": null } }, - "openai/o4-mini": { - "api": { - "id": "openai/o4-mini", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, - "cost": { - "cacheRead": 0.275, - "cacheWrite": 0, - "input": 1.1, - "output": 4.4 - }, - "id": "openai/o4-mini", - "input": [ - "image", - "text" - ], - "maxTokens": 100000, - "name": "OpenAI: o4 Mini", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "off": null - } - }, - "openai/o4-mini-high": { + "openrouter/auto": { "api": { - "id": "openai/o4-mini-high", + "id": "openrouter/auto", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, + "contextWindow": 2000000, "cost": { - "cacheRead": 0.275, + "cacheRead": 0, "cacheWrite": 0, - "input": 1.1, - "output": 4.4 + "input": 0, + "output": 0 }, - "id": "openai/o4-mini-high", + "id": "openrouter/auto", "input": [ - "image", - "text" + "text", + "image" ], - "maxTokens": 100000, - "name": "OpenAI: o4 Mini High", + "maxTokens": 4096, + "name": "Auto Router", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", "provider": { @@ -10671,20 +10623,11 @@ "source": "api" }, "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "high": "high", - "low": null, - "max": null, - "medium": null, - "minimal": null, - "off": null, - "xhigh": null - } + "reasoning": true }, - "openrouter/auto": { + "openrouter/auto-beta": { "api": { - "id": "openrouter/auto", + "id": "openrouter/auto-beta", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, @@ -10696,13 +10639,13 @@ "input": 0, "output": 0 }, - "id": "openrouter/auto", + "id": "openrouter/auto-beta", "input": [ "text", "image" ], - "maxTokens": 4096, - "name": "Auto Router", + "maxTokens": 65536, + "name": "Auto Router (Beta)", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", "provider": { @@ -10716,27 +10659,27 @@ "providerOptionsKey": "openrouter", "reasoning": true }, - "openrouter/auto-beta": { + "openrouter/free": { "api": { - "id": "openrouter/auto-beta", + "id": "openrouter/free", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 2000000, + "contextWindow": 200000, "cost": { "cacheRead": 0, "cacheWrite": 0, "input": 0, "output": 0 }, - "id": "openrouter/auto-beta", + "id": "openrouter/free", "input": [ "text", "image" ], "maxTokens": 4096, - "name": "Auto Router (Beta)", + "name": "Free Models Router", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", "provider": { @@ -10750,27 +10693,26 @@ "providerOptionsKey": "openrouter", "reasoning": true }, - "openrouter/free": { + "openrouter/pareto-code": { "api": { - "id": "openrouter/free", + "id": "openrouter/pareto-code", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, + "contextWindow": 2000000, "cost": { "cacheRead": 0, "cacheWrite": 0, "input": 0, "output": 0 }, - "id": "openrouter/free", + "id": "openrouter/pareto-code", "input": [ - "text", - "image" + "text" ], "maxTokens": 4096, - "name": "Free Models Router", + "name": "Pareto Code Router", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", "provider": { @@ -10784,26 +10726,27 @@ "providerOptionsKey": "openrouter", "reasoning": true }, - "openrouter/pareto-code": { + "particle-ai/particle-1": { "api": { - "id": "openrouter/pareto-code", + "id": "particle-ai/particle-1", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 2000000, + "contextWindow": 1048576, "cost": { - "cacheRead": 0, + "cacheRead": 0.003, "cacheWrite": 0, - "input": 0, - "output": 0 + "input": 0.05, + "output": 0.5 }, - "id": "openrouter/pareto-code", + "id": "particle-ai/particle-1", "input": [ - "text" + "text", + "image" ], - "maxTokens": 4096, - "name": "Pareto Code Router", + "maxTokens": 131072, + "name": "Particle AI: Particle-1", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", "provider": { @@ -10815,7 +10758,16 @@ "source": "api" }, "providerOptionsKey": "openrouter", - "reasoning": true + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": "max", + "medium": "medium", + "minimal": "minimal", + "off": "none", + "xhigh": "xhigh" + } }, "perceptron/perceptron-mk1.5": { "api": { @@ -11035,6 +10987,40 @@ "xhigh": "xhigh" } }, + "prism-ml/ternary-bonsai-2-9b": { + "api": { + "id": "prism-ml/ternary-bonsai-2-9b", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 262144, + "cost": { + "cacheRead": 0.04, + "cacheWrite": 0, + "input": 0.08, + "output": 0.13 + }, + "id": "prism-ml/ternary-bonsai-2-9b", + "input": [ + "text", + "image" + ], + "maxTokens": 65536, + "name": "PrismML: Ternary Bonsai 2 9B", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true + }, "qwen/qwen-2.5-72b-instruct": { "api": { "id": "qwen/qwen-2.5-72b-instruct", @@ -11287,9 +11273,9 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0, + "cacheRead": 0.1, "cacheWrite": 0, - "input": 0.09, + "input": 0.1, "output": 0.3 }, "id": "qwen/qwen3-30b-a3b-instruct-2507", @@ -11525,42 +11511,6 @@ "providerOptionsKey": "openrouter", "reasoning": false }, - "qwen/qwen3-next-80b-a3b-thinking": { - "api": { - "id": "qwen/qwen3-next-80b-a3b-thinking", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 262144, - "cost": { - "cacheRead": 0, - "cacheWrite": 0, - "input": 0.15, - "output": 1.2 - }, - "id": "qwen/qwen3-next-80b-a3b-thinking", - "input": [ - "text" - ], - "maxTokens": 235929, - "name": "Qwen: Qwen3 Next 80B A3B Thinking", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "off": null - } - }, "qwen/qwen3-vl-235b-a22b-instruct": { "api": { "id": "qwen/qwen3-vl-235b-a22b-instruct", @@ -11745,15 +11695,15 @@ "cost": { "cacheRead": 0, "cacheWrite": 0, - "input": 0.195, - "output": 1.56 + "input": 0.26, + "output": 2.6 }, "id": "qwen/qwen3.5-27b", "input": [ "text", "image" ], - "maxTokens": 65536, + "maxTokens": 81920, "name": "Qwen: Qwen3.5-27B", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -11779,7 +11729,7 @@ "cost": { "cacheRead": 0.05, "cacheWrite": 0, - "input": 0.15, + "input": 0.14, "output": 1 }, "id": "qwen/qwen3.5-35b-a3b", @@ -11787,7 +11737,7 @@ "text", "image" ], - "maxTokens": 235929, + "maxTokens": 81920, "name": "Qwen: Qwen3.5-35B-A3B", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -11843,19 +11793,19 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 256000, + "contextWindow": 262144, "cost": { "cacheRead": 0, "cacheWrite": 0, - "input": 0.1, - "output": 0.15 + "input": 0.17, + "output": 0.25 }, "id": "qwen/qwen3.5-9b", "input": [ "text", "image" ], - "maxTokens": 32768, + "maxTokens": 235929, "name": "Qwen: Qwen3.5-9B", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -11999,10 +11949,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0.025, + "cacheRead": 0, "cacheWrite": 0, - "input": 0.25, - "output": 2 + "input": 0.45, + "output": 2.7 }, "id": "qwen/qwen3.6-27b", "input": [ @@ -12033,10 +11983,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0.05, + "cacheRead": 0.025, "cacheWrite": 0, - "input": 0.15, - "output": 1 + "input": 0.1, + "output": 0.9 }, "id": "qwen/qwen3.6-35b-a3b", "input": [ @@ -12321,10 +12271,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0.05, + "cacheRead": 0.0375, "cacheWrite": 0, - "input": 0.24, - "output": 2.2 + "input": 0.15, + "output": 1.875 }, "id": "qwen/qwen3.8-27b", "input": [ @@ -12509,6 +12459,42 @@ "providerOptionsKey": "openrouter", "reasoning": true }, + "recursive/flint-v1.0:free": { + "api": { + "id": "recursive/flint-v1.0:free", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 262144, + "cost": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 0, + "output": 0 + }, + "id": "recursive/flint-v1.0:free", + "input": [ + "text" + ], + "maxTokens": 235929, + "name": "Recursive: Flint (free)", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "off": null + } + }, "reflection/beam-1.0-501b-a23b": { "api": { "id": "reflection/beam-1.0-501b-a23b", @@ -12518,16 +12504,16 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.1, + "cacheRead": 0.05, "cacheWrite": 0, - "input": 1, - "output": 3 + "input": 0.25, + "output": 1 }, "id": "reflection/beam-1.0-501b-a23b", "input": [ "text" ], - "maxTokens": 943718, + "maxTokens": 131072, "name": "Reflection: Beam 1.0 501B A23B", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -13247,7 +13233,7 @@ "image", "text" ], - "maxTokens": 4096, + "maxTokens": 65536, "name": "TypeSafe: Jev Router", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -13372,48 +13358,6 @@ "xhigh": "xhigh" } }, - "upstage/solar-pro-3": { - "api": { - "id": "upstage/solar-pro-3", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 131072, - "cost": { - "cacheRead": 0.015, - "cacheWrite": 0, - "input": 0.15, - "output": 0.6 - }, - "id": "upstage/solar-pro-3", - "input": [ - "text" - ], - "maxTokens": 117964, - "name": "Upstage: Solar Pro 3", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "high": "high", - "low": "low", - "max": null, - "medium": "medium", - "minimal": "minimal", - "off": "none", - "xhigh": null - } - }, "upstage/solar-pro4": { "api": { "id": "upstage/solar-pro4", @@ -13423,10 +13367,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 524288, "cost": { - "cacheRead": 0.018, + "cacheRead": 0.03, "cacheWrite": 0, - "input": 0.09, - "output": 0.36 + "input": 0.15, + "output": 0.6 }, "id": "upstage/solar-pro4", "input": [ @@ -13760,19 +13704,19 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 1048576, + "contextWindow": 1000000, "cost": { - "cacheRead": 0.0028, + "cacheRead": 0.08, "cacheWrite": 0, - "input": 0.14, - "output": 0.28 + "input": 0.4, + "output": 2 }, "id": "xiaomi/mimo-v2.5", "input": [ "text", "image" ], - "maxTokens": 131072, + "maxTokens": 65536, "name": "Xiaomi: MiMo-V2.5", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -13794,18 +13738,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 1048576, + "contextWindow": 262144, "cost": { - "cacheRead": 0.0036, + "cacheRead": 0.096, "cacheWrite": 0, - "input": 0.435, - "output": 0.87 + "input": 0.48, + "output": 1.8 }, "id": "xiaomi/mimo-v2.5-pro", "input": [ "text" ], - "maxTokens": 131072, + "maxTokens": 235929, "name": "Xiaomi: MiMo-V2.5-Pro", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -14261,18 +14205,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, + "contextWindow": 202752, "cost": { - "cacheRead": 0.1794, + "cacheRead": 0.26, "cacheWrite": 0, - "input": 0.966, - "output": 3.036 + "input": 1.4, + "output": 4.4 }, "id": "z-ai/glm-5.1", "input": [ "text" ], - "maxTokens": 128000, + "maxTokens": 131072, "name": "Z.ai: GLM 5.1", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -14296,10 +14240,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.14, + "cacheRead": 0.005, "cacheWrite": 0, - "input": 0.62, - "output": 2.42 + "input": 0.006, + "output": 7 }, "id": "z-ai/glm-5.2", "input": [ @@ -14338,9 +14282,9 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.078, + "cacheRead": 0.0689, "cacheWrite": 0, - "input": 0.079, + "input": 0.0699, "output": 1.4 }, "id": "z-ai/glm-5.3", @@ -14380,17 +14324,17 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.006, + "cacheRead": 0.0197, "cacheWrite": 0, - "input": 0.048, - "output": 0.2 + "input": 0.0197, + "output": 0.1238 }, "id": "z-ai/glm-5.3-flash", "input": [ "text", "image" ], - "maxTokens": 943718, + "maxTokens": 131072, "name": "Z.ai: GLM 5.3 Flash", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -14723,10 +14667,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.007, + "cacheRead": 0.0143, "cacheWrite": 0, - "input": 0.058, - "output": 0.6 + "input": 0.0155, + "output": 0.19 }, "id": "~deepseek/deepseek-flash-latest", "input": [ @@ -14766,9 +14710,9 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.23, + "cacheRead": 0.1, "cacheWrite": 0, - "input": 0.225, + "input": 0.1035, "output": 8 }, "id": "~deepseek/deepseek-pro-latest", @@ -14808,10 +14752,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.00336, + "cacheRead": 0.0378, "cacheWrite": 0, - "input": 0.01208, - "output": 0.12 + "input": 0.0388, + "output": 0.3 }, "id": "~deepseek/deepseek-v4-flash-latest", "input": [ @@ -14936,10 +14880,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.228, + "cacheRead": 0.15, "cacheWrite": 0, - "input": 0.238, - "output": 16 + "input": 0.1558, + "output": 10.5478 }, "id": "~moonshotai/kimi-latest", "input": [ @@ -15178,17 +15122,17 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1050000, "cost": { - "cacheRead": 0.2, - "cacheWrite": 2.5, - "input": 2, - "output": 12, + "cacheRead": 0.05, + "cacheWrite": 1.25, + "input": 1, + "output": 5, "tiers": [ { - "cacheRead": 0.4, - "cacheWrite": 5, - "input": 4, + "cacheRead": 0.1, + "cacheWrite": 2.5, + "input": 2, "inputTokensAbove": 272000, - "output": 18 + "output": 7.5 } ] }, @@ -15316,17 +15260,17 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.006, + "cacheRead": 0.0197, "cacheWrite": 0, - "input": 0.048, - "output": 0.2 + "input": 0.0197, + "output": 0.1238 }, "id": "~z-ai/glm-flash-latest", "input": [ "text", "image" ], - "maxTokens": 943718, + "maxTokens": 131072, "name": "Z.ai: GLM Flash Latest", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -15359,9 +15303,9 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.078, + "cacheRead": 0.0689, "cacheWrite": 0, - "input": 0.079, + "input": 0.0699, "output": 1.4 }, "id": "~z-ai/glm-latest", diff --git a/packages/aikit/src/model/model.ts b/packages/aikit/src/model/model.ts index 879a06cd..5cc4bae7 100644 --- a/packages/aikit/src/model/model.ts +++ b/packages/aikit/src/model/model.ts @@ -160,7 +160,10 @@ export const CompatibilitySchema = Type.Object({ /** The Google protocol accepts thinking levels rather than a token budget. */ supportsThinkingLevel: Type.Optional(Type.Boolean()), supportsToolSearch: Type.Optional(Type.Boolean()), - /** Requires `supportsMidConvoSystemMessages`. */ + /** + * Tool changes go in place: Responses `additional_tools` items, Anthropic tool references. + * Requires `supportsMidConvoSystemMessages`. + */ supportsAdditionalTools: Type.Optional(Type.Boolean()), /** The model accepts developer messages after the conversation has started. */ supportsMidConvoSystemMessages: Type.Optional(Type.Boolean()), diff --git a/packages/aikit/test/__artifacts__/modelgen.json b/packages/aikit/test/__artifacts__/modelgen.json index 5bbf7420..b27e677e 100644 --- a/packages/aikit/test/__artifacts__/modelgen.json +++ b/packages/aikit/test/__artifacts__/modelgen.json @@ -8,7 +8,9 @@ }, "baseUrl": "https://api.anthropic.com/v1", "compat": { - "forceAdaptiveThinking": true + "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true }, "contextWindow": 1000000, "cost": { @@ -50,7 +52,9 @@ }, "baseUrl": "https://api.anthropic.com/v1", "compat": { - "forceAdaptiveThinking": true + "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true }, "contextWindow": 1000000, "cost": { @@ -161,6 +165,8 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -367,6 +373,8 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -409,6 +417,8 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -451,6 +461,8 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -647,6 +659,8 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -1585,6 +1599,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.25, @@ -1641,6 +1658,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 400000, "cost": { "cacheRead": 0.075, @@ -1735,6 +1755,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 1050000, "cost": { "cacheRead": 0, @@ -1791,6 +1814,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.5, @@ -1903,6 +1929,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.02, @@ -1959,6 +1988,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.4, @@ -2015,6 +2047,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.2, @@ -2071,6 +2106,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 1, @@ -2128,6 +2166,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.01, @@ -2184,6 +2225,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.2, @@ -2240,6 +2284,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.1, diff --git a/packages/aikit/test/transcript.test.ts b/packages/aikit/test/transcript.test.ts index 13216232..2a9939f2 100644 --- a/packages/aikit/test/transcript.test.ts +++ b/packages/aikit/test/transcript.test.ts @@ -1,11 +1,12 @@ -/** Prompt and tool changes carried by transcript system messages, from the context down to the Responses wire. */ +/** Prompt and tool changes carried by transcript system messages, from the context down to the Responses and Anthropic wires. */ import Type from "typebox"; import { describe, expect, it } from "vite-plus/test"; import * as Message from "../src/message/message.ts"; import * as Transcript from "../src/message/transcript.ts"; import type * as Model from "../src/model/model.ts"; import { stream } from "../src/stream.ts"; -import { makeUserMessage } from "./utils/fixtures.ts"; +import { fakeAnthropic, textReply } from "./utils/anthropic.ts"; +import { makeAssistantMessage, makeCompletedToolCall, makeModel, makeUserMessage } from "./utils/fixtures.ts"; import { createOpenAICodexMockFetch, OPENAI_CODEX_TEST_API_KEY, @@ -363,3 +364,122 @@ describe("Transcript", () => { ).toEqual([changed, lateTool]); }); }); + +describe("transcript prompt changes on the Anthropic wire", () => { + type AnthropicBody = { + system?: Array<{ text: string }>; + messages: Array<{ role: string; content: Array<{ type: string; text?: string }> }>; + }; + const claude = (inPlace: boolean) => + makeModel({ npm: "@ai-sdk/anthropic", compat: { supportsMidConvoSystemMessages: inPlace } }); + const wire = async (model: Model.Info) => { + const server = fakeAnthropic(textReply); + await stream( + model, + { + systemPrompt: "base prompt", + messages: [ + makeUserMessage("first"), + system({ sections: { codeword: "alpha" } }), + makeAssistantMessage(model, { stopReason: "toolUse", parts: [makeCompletedToolCall("call-1")] }), + system({ sections: { codeword: "bravo" } }), + makeUserMessage("second"), + ], + }, + { apiKey: "test-key", factoryOptions: { fetch: server.fetch } }, + ).result(); + const body = server.body() as AnthropicBody; + return { + system: body.system?.map((part) => part.text), + messages: body.messages.map((message) => [ + message.role, + message.content.map((part) => part.text ?? part.type).join(" | "), + ]), + }; + }; + + it("joins a declaration before the first reply to the leading prompt and holds a later one past the tool result and prompt", async () => { + expect(await wire(claude(true))).toEqual({ + system: ["base prompt", 'Updated system prompt section "codeword":\n\nalpha'], + messages: [ + ["user", "first"], + ["assistant", "tool_use"], + ["user", "tool_result | second"], + ["system", 'Updated system prompt section "codeword":\n\nbravo'], + ], + }); + }); + + it("folds every change into the leading prompt on a model without mid-conversation system messages", async () => { + expect(await wire(claude(false))).toEqual({ + system: ["base prompt\n\nbravo"], + messages: [ + ["user", "first"], + ["assistant", "tool_use"], + ["user", "tool_result | second"], + ], + }); + }); +}); + +describe("transcript tool changes on the Anthropic wire", () => { + type ToolsBody = { + tools?: Array<{ name: string; defer_loading?: boolean }>; + messages: Array<{ role: string; content: Array<{ type: string; text?: string; tool?: { name: string } }> }>; + }; + const claude = makeModel({ + npm: "@ai-sdk/anthropic", + compat: { supportsMidConvoSystemMessages: true, supportsAdditionalTools: true }, + }); + const capture = async (messages: Message.Message[]) => { + const server = fakeAnthropic(textReply); + await stream( + claude, + { + systemPrompt: "base prompt", + tools: [baseTool, dropTool], + deferredTools: [lateTool, tool("spare_tool")], + messages, + }, + { apiKey: "test-key", factoryOptions: { fetch: server.fetch } }, + ).result(); + const body = server.body() as ToolsBody; + return { + tools: body.tools?.map((one) => `${one.name}${one.defer_loading ? " (deferred)" : ""}`), + messages: body.messages.map((message) => [ + message.role, + message.content + .map((part) => (part.tool ? `${part.type} ${part.tool.name}` : (part.text ?? part.type))) + .join(" | "), + ]), + }; + }; + const first = [makeUserMessage("first"), system({ toolsAdded: [tool("early_tool")] })]; + const toolTurn = makeAssistantMessage(claude, { stopReason: "toolUse", parts: [makeCompletedToolCall("call-1")] }); + const later = [ + toolTurn, + system({ toolsAdded: [lateTool], toolsRemoved: [{ name: "drop_tool" }] }), + makeUserMessage("second"), + ]; + + it("declares the catalog deferred up front and references a later change", async () => { + const before = await capture(first); + const after = await capture([...first, ...later]); + + // Every tool is declared from the first request on, so a later change keeps the list as it was. + expect(after.tools).toEqual(before.tools); + expect(after.tools).toEqual([ + "base_tool", + "drop_tool", + "early_tool", + "late_tool (deferred)", + "spare_tool (deferred)", + ]); + expect(after.messages).toEqual([ + ["user", "first"], + ["assistant", "tool_use"], + ["user", "tool_result | second"], + ["system", "tool_removal drop_tool | tool_addition late_tool"], + ]); + }); +}); diff --git a/packages/codework/src/server/contract.ts b/packages/codework/src/server/contract.ts index 30fbea21..d2a2295e 100644 --- a/packages/codework/src/server/contract.ts +++ b/packages/codework/src/server/contract.ts @@ -89,6 +89,8 @@ export const ThreadInfo = Schema.Struct({ model: optional(Schema.Struct({ provider: Schema.String, id: Schema.String })), /** The thinking level chosen for this thread; absent when settings decide. */ thinkingLevel: optional(Schema.Enum(Model.ThinkingLevelEnum)), + /** The prompt foundation chosen for this thread; absent when prompt plugins render it. */ + foundation: optional(Schema.String), /** Where it runs instead of the session's space and directory. */ placement: optional(Schema.Struct({ spaceId: Schema.String, directory: Session.AbsolutePath })), /** Its unfinished run was cut off by restarts too often to resume on its own; the next prompt runs it. */ @@ -146,10 +148,11 @@ export const LogSynced = Schema.Struct({ }); export type LogSynced = typeof LogSynced.Type; -/** A thread's model and thinking choices: a given field replaces the stored one, `null` clears it. */ +/** A thread's model, thinking and foundation choices: a given field replaces the stored one, `null` clears it. */ export const RuntimeConfig = Schema.Struct({ model: optional(Schema.NullOr(Schema.Struct({ provider: Schema.String, id: Schema.String }))), thinkingLevel: optional(Schema.NullOr(Schema.Enum(Model.ThinkingLevelEnum))), + foundation: optional(Schema.NullOr(Schema.NonEmptyString)), }); export type RuntimeConfig = typeof RuntimeConfig.Type; diff --git a/packages/codework/src/server/handlers.ts b/packages/codework/src/server/handlers.ts index 85c47814..0a6e17c2 100644 --- a/packages/codework/src/server/handlers.ts +++ b/packages/codework/src/server/handlers.ts @@ -63,6 +63,7 @@ const toThreadInfo = (info: Session.ThreadInfo): ThreadInfo => ({ updatedAt: info.updatedAt, ...(info.model === undefined ? {} : { model: info.model }), ...(info.thinkingLevel === undefined ? {} : { thinkingLevel: info.thinkingLevel }), + ...(info.foundation === undefined ? {} : { foundation: info.foundation }), ...(info.placement === undefined ? {} : { placement: info.placement }), ...(info.parked ? { parked: true } : {}), usage: info.usage, @@ -74,6 +75,7 @@ const toThreadInfo = (info: Session.ThreadInfo): ThreadInfo => ({ const runtimeInput = (runtime: RuntimeConfig | undefined): Session.RuntimeInput => ({ ...(runtime?.model == null ? {} : { model: runtime.model }), ...(runtime?.thinkingLevel == null ? {} : { thinkingLevel: runtime.thinkingLevel }), + ...(runtime?.foundation == null ? {} : { foundation: runtime.foundation }), }); const threadInfo = (threadId: ThreadSchema.ID) => Session.threadInfo(threadId).pipe(Effect.map(toThreadInfo)); @@ -239,6 +241,7 @@ export const layer = Contract.Api.toLayer( const change = { ...(runtime.model === undefined ? {} : { model: runtime.model }), ...(runtime.thinkingLevel === undefined ? {} : { thinkingLevel: runtime.thinkingLevel }), + ...(runtime.foundation === undefined ? {} : { foundation: runtime.foundation }), }; if (Object.keys(change).length > 0) { yield* events.publish(EventList.ThreadConfigured, { diff --git a/packages/harness/src/context/codec.ts b/packages/harness/src/context/codec.ts index 48ca0a52..c9251fcc 100644 --- a/packages/harness/src/context/codec.ts +++ b/packages/harness/src/context/codec.ts @@ -199,11 +199,12 @@ export const decodeSyntheticMessage = Effect.fn("Context.decodeSyntheticMessage" ); }); -/** The transcript system message of a tool change; empty sides are left out. */ +/** The transcript system message of a section and tool change; empty parts are left out. */ export const systemMessage = ( messageId: string, created: number, changes: { + readonly sections?: Readonly> | undefined; readonly toolsAdded: ReadonlyArray; readonly toolsRemoved: ReadonlyArray; }, @@ -211,6 +212,7 @@ export const systemMessage = ( messageId, role: "system" as const, time: { created }, + ...(changes.sections === undefined ? {} : { sections: { ...changes.sections } }), ...(changes.toolsAdded.length === 0 ? {} : { toolsAdded: [...changes.toolsAdded] }), ...(changes.toolsRemoved.length === 0 ? {} : { toolsRemoved: changes.toolsRemoved.map(({ name }) => ({ name })) }), }); diff --git a/packages/harness/src/document/schema.ts b/packages/harness/src/document/schema.ts index 90725581..1e6abbec 100644 --- a/packages/harness/src/document/schema.ts +++ b/packages/harness/src/document/schema.ts @@ -48,6 +48,8 @@ export const Agent = Schema.Struct({ modelTools: optional(ModelTools), /** Physical boundary override; absent = the session's. */ placement: optional(Placement), + /** The prompt foundation for this thread, in place of the one prompt plugins render. */ + foundation: optional(Schema.NonEmptyString), }); export type Agent = typeof Agent.Type; @@ -57,6 +59,7 @@ export const AgentChange = Schema.Struct({ thinkingLevel: optional(Schema.NullOr(ThinkingLevel)), modelTools: optional(Schema.NullOr(ModelTools)), placement: optional(Schema.NullOr(Placement)), + foundation: optional(Schema.NullOr(Schema.NonEmptyString)), }); export type AgentChange = typeof AgentChange.Type; diff --git a/packages/harness/src/effect/session.ts b/packages/harness/src/effect/session.ts index 971de97a..bdd8c7e1 100644 --- a/packages/harness/src/effect/session.ts +++ b/packages/harness/src/effect/session.ts @@ -16,7 +16,7 @@ import { PromptSchema } from "../session/prompt/schema.ts"; import * as SessionRuntime from "../session/runtime.ts"; import type { ThreadRow } from "../db/schema.sql.ts"; import { Document } from "../document/document.ts"; -import type { DocumentSchema } from "../document/schema.ts"; +import { DocumentSchema } from "../document/schema.ts"; import { SessionSchema } from "../session/schema.ts"; import { Session as SessionStore } from "../session/session.ts"; import { ThreadSchema } from "../thread/schema.ts"; @@ -43,6 +43,8 @@ export interface SystemPromptConfig { export interface RuntimeInput { readonly model?: ModelConfig; readonly thinkingLevel?: Model.ThinkingLevel; + /** The thread's own prompt foundation, in place of the rendered one; `null` goes back to it. */ + readonly foundation?: string | null; readonly tools?: ToolsConfig; readonly systemPrompt?: SystemPromptConfig; } @@ -122,6 +124,8 @@ export interface ThreadInfo { readonly model?: { readonly provider: string; readonly id: string }; /** The thinking level chosen for this thread over settings; absent when settings decide. */ readonly thinkingLevel?: Model.ThinkingLevel; + /** The prompt foundation chosen for this thread; absent when prompt plugins render it. */ + readonly foundation?: string; /** Where it runs instead of the session's space and directory. */ readonly placement?: DocumentSchema.Placement; /** @@ -158,7 +162,7 @@ export interface ThreadHandle { readonly resume: () => ReturnType; /** Stops active work and waits for its cleanup; idle is a no-op returning false. */ readonly interrupt: () => ReturnType; - /** Records model and thinking choices in the thread's `agent` document. */ + /** Records model, thinking and foundation choices in the thread's `agent` document. */ readonly configure: (input: RuntimeInput) => Effect.Effect; /** A new thread cut at a committed entry of this one's line; `text` refills the editor for `before`. */ readonly fork: ( @@ -196,9 +200,10 @@ const runtimeBindings = (input: RuntimeInput): SessionRuntime.Bindings => ({ }); /** The `agent` fields this input names. */ -const chosen = (input: RuntimeInput): DocumentSchema.Agent => ({ +const chosen = (input: RuntimeInput): DocumentSchema.AgentChange => ({ ...(input.model === undefined ? {} : { model: { provider: input.model.provider, id: input.model.id } }), ...(input.thinkingLevel === undefined ? {} : { thinkingLevel: input.thinkingLevel }), + ...(input.foundation === undefined ? {} : { foundation: input.foundation }), }); /** Records a choice as one `agent` revision; nothing when the input names none. */ @@ -290,6 +295,7 @@ export const threadInfo = Effect.fn("Session.threadInfo")(function* (id: ThreadS updatedAt: DateTime.toEpochMillis(row.updatedAt), ...(agent.model === undefined ? {} : { model: agent.model }), ...(agent.thinkingLevel === undefined ? {} : { thinkingLevel: agent.thinkingLevel }), + ...(agent.foundation === undefined ? {} : { foundation: agent.foundation }), ...(agent.placement === undefined ? {} : { placement: agent.placement }), parked, usage: { @@ -405,7 +411,7 @@ export const create = Effect.fn("Session.create")(function* (input: CreateInput directory: location.directory, ...(input.title === undefined ? {} : { title: input.title }), ...(input.hostDir === undefined ? {} : { hostDir: declaredHostDir(input.hostDir) }), - agent: chosen(input), + agent: DocumentSchema.applyAgentChange({}, chosen(input)), }); yield* runtime.set(id, runtimeBindings(input)); return yield* makeHandle(id).pipe(Effect.orDie); diff --git a/packages/harness/src/event/list.ts b/packages/harness/src/event/list.ts index 1be3a6cc..cd893099 100644 --- a/packages/harness/src/event/list.ts +++ b/packages/harness/src/event/list.ts @@ -113,15 +113,16 @@ export const ThreadConfigured = EventSchema.define({ export type ThreadConfigured = typeof ThreadConfigured.Type; /** - * The tools offered to a thread's next request differ from those its transcript declares, such as - * after `toolsearch` loaded some. Appends a `system` entry carrying the change. + * The prompt sections or tools for a thread's next request differ from those its transcript + * declares: the first request, a changed section, or `toolsearch` loading a tool. Appends a + * `system` entry carrying the change. */ -export const ToolsChanged = EventSchema.define({ - type: "session.thread.tools.changed", +export const SystemChanged = EventSchema.define({ + type: "session.thread.system.changed", ...durableOptions, - schema: { ...baseOptions, messageId: SessionMessageSchema.ID, ...SessionSchema.ToolChanges.fields }, + schema: { ...baseOptions, messageId: SessionMessageSchema.ID, ...SessionSchema.SystemChange.fields }, }); -export type ToolsChanged = typeof ToolsChanged.Type; +export type SystemChanged = typeof SystemChanged.Type; const LLMFields = { ...baseOptions, @@ -397,7 +398,7 @@ export const Definitions = EventSchema.inventory( SessionCreated, ThreadForked, ThreadConfigured, - ToolsChanged, + SystemChanged, TurnStarted, TurnEnded, TurnAborted, @@ -434,7 +435,7 @@ export const DurableDefinitions = EventSchema.inventory( SessionCreated, ThreadForked, ThreadConfigured, - ToolsChanged, + SystemChanged, TurnEnded, LLMStarted, LLMEnded, diff --git a/packages/harness/src/plugin/prompt/render.ts b/packages/harness/src/plugin/prompt/render.ts index dd4dea20..65e7af75 100644 --- a/packages/harness/src/plugin/prompt/render.ts +++ b/packages/harness/src/plugin/prompt/render.ts @@ -35,7 +35,17 @@ const toolIndex = (tools: ReadonlyArray): string => { return listed.map((tool) => `- ${tool.name}: ${tool.promptSnippet}`).join("\n"); }; -export const render = (input: Input): string => { +/** + * The prompt as the transcript carries it: the untagged foundation, then named sections, each + * holding its own tags. A section's text is compared whole between requests, so one that did not + * change is never sent again. + */ +export interface Rendered { + readonly foundation: string; + readonly sections: Readonly>; +} + +export const render = (input: Input): Rendered => { const written = new Map(input.prompt.sections); const guidelines = input.tools.flatMap((tool) => tool.promptGuidelines ?? []); @@ -44,17 +54,20 @@ export const render = (input: Input): string => { entries: [...guidelines, ...(written.get(Section.Rules.name)?.entries ?? [])], }); - const parts: string[] = [input.prompt.foundation ?? fallback, tag("tools", toolIndex(input.tools))]; + const sections: Record = { tools: tag("tools", toolIndex(input.tools)) }; const push = (name: string) => { const section = written.get(name); const rendered = section === undefined ? undefined : body(section); - if (rendered !== undefined) parts.push(tag(name, rendered)); + if (rendered !== undefined) sections[name] = tag(name, rendered); written.delete(name); }; for (const section of Section.builtins) push(section.name); - parts.push(tag("cwd", input.directory)); + sections.cwd = tag("cwd", input.directory); for (const name of Array.from(written.keys())) push(name); - return parts.join("\n\n"); + // A blank foundation would leave the Responses transport without instructions, and it would + // promote the first system message in its stead. + const foundation = input.prompt.foundation?.trim() ? input.prompt.foundation : fallback; + return { foundation, sections }; }; diff --git a/packages/harness/src/plugin/registry.ts b/packages/harness/src/plugin/registry.ts index ae9b898c..0fceeb02 100644 --- a/packages/harness/src/plugin/registry.ts +++ b/packages/harness/src/plugin/registry.ts @@ -34,7 +34,7 @@ export const make = (servers: ReadonlyArray) => { return { tools: resolved, // Only declared tools: loading a deferred one must not change the prompt. - systemPrompt: render({ prompt: prompt.snapshot(), tools: resolved.defs.filter(declared), directory }), + prompt: render({ prompt: prompt.snapshot(), tools: resolved.defs.filter(declared), directory }), }; }, }; diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index 5f3fe98f..29e89f0e 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -275,11 +275,12 @@ export const layer = (options: Options = {}) => }); /** - * The request keeps the declared tools as `tools`; what the thread offers beyond them (or - * withholds) is declared by a `system` entry, published when the transcript's tool state - * differs from the offered one. Returns its message, for the request at hand. + * The request keeps the foundation as `systemPrompt` and the declared tools as `tools`. The + * prompt sections, and what the thread offers beyond those tools (or withholds), are declared + * by a `system` entry, published when the transcript's state differs from the snapshot's: the + * first request declares every section. Returns its message, for the request at hand. */ - const declareToolChanges = Effect.fn("Loop.declareToolChanges")(function* ( + const declareSystemChanges = Effect.fn("Loop.declareSystemChanges")(function* ( snapshot: State.Snapshot, messages: ReadonlyArray, ) { @@ -287,12 +288,14 @@ export const layer = (options: Options = {}) => { messages: [...messages], tools: [...snapshot.tools.wire] }, "none", ).requestTools; - const changes = Transcript.getToolStateChanges(previous, snapshot.tools.offered); - if (changes.toolsAdded.length === 0 && changes.toolsRemoved.length === 0) return []; + const tools = Transcript.getToolStateChanges(previous, snapshot.tools.offered); + const sections = Transcript.diffSections(Transcript.getCurrentSections(messages), snapshot.prompt.sections); + if (sections === undefined && tools.toolsAdded.length === 0 && tools.toolsRemoved.length === 0) return []; + const changes = { ...(sections === undefined ? {} : { sections }), ...tools }; const messageId = SessionMessageSchema.ID.create(); const timestamp = yield* DateTime.now; const hash = Hash.sha256(yield* encodeJson(snapshot.tools.offered.map(Transcript.toToolDeclaration))); - yield* events.publish(EventList.ToolsChanged, { + yield* events.publish(EventList.SystemChanged, { timestamp, sessionId: snapshot.sessionId, threadId: snapshot.threadId, @@ -326,7 +329,7 @@ export const layer = (options: Options = {}) => if (leaf !== undefined && (leaf.stopReason === "stop" || leaf.stopReason === "length")) { return undefined; } - const messages = [...assembled.messages, ...(yield* declareToolChanges(snapshot, assembled.messages))]; + const messages = [...assembled.messages, ...(yield* declareSystemChanges(snapshot, assembled.messages))]; // The latest attempt's publisher; a retried turn has one per provider request. let publisher: LLMEventPublisher.Publisher | undefined; @@ -351,9 +354,10 @@ export const layer = (options: Options = {}) => const prepared = yield* hooks.prepareRequest( { context: { - systemPrompt: snapshot.systemPrompt, + systemPrompt: snapshot.prompt.foundation, messages: [...messages], tools: [...snapshot.tools.wire], + deferredTools: [...snapshot.tools.deferred], }, provider: snapshot.provider, model: snapshot.model, diff --git a/packages/harness/src/session/projector.ts b/packages/harness/src/session/projector.ts index bbdb9f5a..94e9e760 100644 --- a/packages/harness/src/session/projector.ts +++ b/packages/harness/src/session/projector.ts @@ -198,16 +198,21 @@ export const layer = Layer.effectDiscard( }), ); - yield* events.project(EventList.ToolsChanged, (event) => + yield* events.project(EventList.SystemChanged, (event) => Effect.gen(function* () { - const { toolsAdded, toolsRemoved, hash } = event.data; + const { sections, toolsAdded, toolsRemoved, hash } = event.data; yield* threads .append({ id: event.data.messageId, threadId: event.data.threadId, seq: yield* seqOf(event), type: "system", - data: yield* encodeSystemData({ toolsAdded, toolsRemoved, hash }).pipe(Effect.orDie), + data: yield* encodeSystemData({ + ...(sections === undefined ? {} : { sections }), + toolsAdded, + toolsRemoved, + hash, + }).pipe(Effect.orDie), }) .pipe(Effect.orDie); }), diff --git a/packages/harness/src/session/schema.ts b/packages/harness/src/session/schema.ts index 5995e9b3..b30955f2 100644 --- a/packages/harness/src/session/schema.ts +++ b/packages/harness/src/session/schema.ts @@ -44,16 +44,20 @@ export const ToolDeclaration = Schema.Struct({ constrainedSampling: optional(Schema.Unknown), }); -/** A change to the tools offered from one point of the transcript on, as a `system` entry stores it. */ -export const ToolChanges = Schema.Struct({ +/** + * A change to the prompt sections and tools offered from one point of the transcript on, as a + * `system` entry stores it. A `null` section is removed. + */ +export const SystemChange = Schema.Struct({ + sections: optional(Schema.Record(Schema.String, Schema.NullOr(Schema.String))), toolsAdded: Schema.Array(ToolDeclaration), toolsRemoved: Schema.Array(Schema.Struct({ name: Schema.String })), /** sha256 of the offered tool declarations after the change. */ hash: Schema.String, }); -export type ToolChanges = typeof ToolChanges.Type; +export type SystemChange = typeof SystemChange.Type; -export const SystemData = Schema.fromJsonString(ToolChanges); +export const SystemData = Schema.fromJsonString(SystemChange); // Harness-owned shape of the billed usage inside a persisted assistant envelope. // This is data-structure integrity only — business validation diff --git a/packages/harness/src/state/state.ts b/packages/harness/src/state/state.ts index 7ad622e1..9281db02 100644 --- a/packages/harness/src/state/state.ts +++ b/packages/harness/src/state/state.ts @@ -37,6 +37,7 @@ import { merge } from "../settings/merge.ts"; import { compose, resolveOptions } from "../settings/resolve.ts"; import type { Block, Info, RetryPolicy } from "../settings/schema.ts"; import { Settings } from "../settings/settings.ts"; +import type { Rendered } from "../plugin/prompt/render.ts"; import type { Resolved } from "../tool/registry.ts"; /** @@ -116,7 +117,8 @@ export interface Snapshot { readonly threadId: ThreadSchema.ID; readonly sandbox: SandboxIO.Identity; readonly location: Location.Info; - readonly systemPrompt: string; + /** The foundation goes out as `systemPrompt`; the sections reach the transcript as system messages. */ + readonly prompt: Rendered; readonly tools: Resolved; readonly provider: string; readonly model: string; @@ -522,6 +524,10 @@ export const layer = ( sandbox, location, ...contributions, + prompt: + agent.foundation === undefined + ? contributions.prompt + : { ...contributions.prompt, foundation: agent.foundation }, provider, model, resolvedModel, diff --git a/packages/harness/src/tool/registry.ts b/packages/harness/src/tool/registry.ts index bb9c1172..e9ca3427 100644 --- a/packages/harness/src/tool/registry.ts +++ b/packages/harness/src/tool/registry.ts @@ -39,6 +39,8 @@ export interface Resolved { readonly wire: ReadonlyArray; /** The offered tools' wire view: the tool state the transcript's system messages bring `wire` to. */ readonly offered: ReadonlyArray; + /** Every deferred tool in the catalog, loaded or not: what a transcript system message may add. */ + readonly deferred: ReadonlyArray; /** * Transform one complete pending tool-call part into a complete terminal part. The **only** * execution entry point — the @@ -86,6 +88,7 @@ export const make = (tools: ReadonlyArray): R const defs = Object.freeze(effective.map((entry) => entry.tool.definition)); const names = Object.freeze(effective.map((entry) => entry.tool.definition.name)); const isDeferred = (entry: ToolRegistration) => entry.tool.definition.exposure === "deferred"; + const catalog = Object.freeze(effective.filter(isDeferred).map((entry) => toAikitTool(entry.tool.definition))); const resolve = (selection?: Selection, live?: Effect.Effect>): Resolved => { const removed = new Set(selection?.remove); @@ -113,6 +116,7 @@ export const make = (tools: ReadonlyArray): R defs: Object.freeze(offered.map((entry) => entry.tool.definition)), wire: Object.freeze(declaredSet.map((entry) => toAikitTool(entry.tool.definition))), offered: Object.freeze([...executor.wire]), + deferred: catalog, handle: executor.handle, }); }; diff --git a/packages/harness/test/__artifacts__/mcp.thread.json b/packages/harness/test/__artifacts__/mcp.thread.json index 73c797f6..79f41a6b 100644 --- a/packages/harness/test/__artifacts__/mcp.thread.json +++ b/packages/harness/test/__artifacts__/mcp.thread.json @@ -489,6 +489,16 @@ }, "changes": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "mcp__e2e__args" ], diff --git a/packages/harness/test/__artifacts__/mcp.thread.loaded.json b/packages/harness/test/__artifacts__/mcp.thread.loaded.json index 1e118ef6..000cc486 100644 --- a/packages/harness/test/__artifacts__/mcp.thread.loaded.json +++ b/packages/harness/test/__artifacts__/mcp.thread.loaded.json @@ -2,6 +2,16 @@ "notAwaited": "connecting", "awaitedChanges": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "mcp__slow__echo", "mcp__slow__big" @@ -11,6 +21,16 @@ ], "whileAway": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "mcp__slow__echo", "mcp__slow__big" @@ -18,6 +38,7 @@ "removed": [] }, { + "sections": [], "added": [], "removed": [ "mcp__slow__echo", @@ -33,6 +54,16 @@ }, "back": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "mcp__slow__echo", "mcp__slow__big" @@ -40,6 +71,7 @@ "removed": [] }, { + "sections": [], "added": [], "removed": [ "mcp__slow__echo", @@ -47,6 +79,7 @@ ] }, { + "sections": [], "added": [ "mcp__slow__echo", "mcp__slow__big" diff --git a/packages/harness/test/__artifacts__/mcp.thread.resumed.json b/packages/harness/test/__artifacts__/mcp.thread.resumed.json index ab578040..6f1e1b20 100644 --- a/packages/harness/test/__artifacts__/mcp.thread.resumed.json +++ b/packages/harness/test/__artifacts__/mcp.thread.resumed.json @@ -14,6 +14,16 @@ "second": { "changes": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "mcp__docs__echo", "mcp__docs__big" diff --git a/packages/harness/test/__artifacts__/mcp.toolsearch.connecting.json b/packages/harness/test/__artifacts__/mcp.toolsearch.connecting.json index 027335de..0ff8ab5b 100644 --- a/packages/harness/test/__artifacts__/mcp.toolsearch.connecting.json +++ b/packages/harness/test/__artifacts__/mcp.toolsearch.connecting.json @@ -9,6 +9,16 @@ ], "secondChanges": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "mcp__slow__echo", "mcp__slow__big" diff --git a/packages/harness/test/__artifacts__/prompt.changes.json b/packages/harness/test/__artifacts__/prompt.changes.json new file mode 100644 index 00000000..a8a65896 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.json @@ -0,0 +1,220 @@ +{ + "requests": [ + { + "roles": [ + "user", + "system" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + } + ], + "mode": "Plan only." + }, + { + "roles": [ + "user", + "system", + "assistant", + "user", + "system" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + }, + { + "mode": "\nBuild allowed.\n" + } + ], + "mode": "Build allowed." + }, + { + "roles": [ + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "system" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + }, + { + "mode": "\nBuild allowed.\n" + }, + { + "mode": null + } + ], + "mode": null + }, + { + "roles": [ + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + }, + { + "mode": "\nBuild allowed.\n" + }, + { + "mode": null + } + ], + "mode": null + }, + { + "roles": [ + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "assistant", + "user" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + }, + { + "mode": "\nBuild allowed.\n" + }, + { + "mode": null + } + ], + "mode": null + }, + { + "roles": [ + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "assistant", + "user", + "assistant", + "user", + "system" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + }, + { + "mode": "\nBuild allowed.\n" + }, + { + "mode": null + }, + { + "mode": "\nPlan again.\n" + } + ], + "mode": "Plan again." + }, + { + "roles": [ + "user", + "system", + "assistant", + "user", + "system" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + }, + { + "mode": null + } + ], + "mode": null + } + ], + "prefixKept": [ + true, + true, + true, + true, + true + ], + "threads": { + "main": [ + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "assistant", + "user", + "assistant", + "user", + "system", + "assistant" + ], + "fork": [ + "user", + "system", + "assistant", + "user", + "system", + "assistant" + ] + } +} \ No newline at end of file diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.anthropic.json b/packages/harness/test/__artifacts__/prompt.changes.live.anthropic.json new file mode 100644 index 00000000..b96adf55 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.live.anthropic.json @@ -0,0 +1,10 @@ +{ + "answers": [ + "alpha", + "bravo" + ], + "promptChange": "in place", + "earlierSectionSent": true, + "change": "after", + "cached": true +} diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.copilot-claude-sonnet-5.5.json b/packages/harness/test/__artifacts__/prompt.changes.live.copilot-claude-sonnet-5.5.json new file mode 100644 index 00000000..b96adf55 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.live.copilot-claude-sonnet-5.5.json @@ -0,0 +1,10 @@ +{ + "answers": [ + "alpha", + "bravo" + ], + "promptChange": "in place", + "earlierSectionSent": true, + "change": "after", + "cached": true +} diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-5.3-codex.json b/packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-5.3-codex.json new file mode 100644 index 00000000..45391da9 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-5.3-codex.json @@ -0,0 +1,8 @@ +{ + "answers": [ + "alpha", + "bravo" + ], + "promptChange": "folded", + "earlierSectionSent": false +} diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-6-luna.json b/packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-6-luna.json new file mode 100644 index 00000000..663e5327 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-6-luna.json @@ -0,0 +1,9 @@ +{ + "answers": [ + "alpha", + "bravo" + ], + "promptChange": "in place", + "earlierSectionSent": true, + "change": "after" +} diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json b/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json new file mode 100644 index 00000000..663e5327 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json @@ -0,0 +1,9 @@ +{ + "answers": [ + "alpha", + "bravo" + ], + "promptChange": "in place", + "earlierSectionSent": true, + "change": "after" +} diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.openai.json b/packages/harness/test/__artifacts__/prompt.changes.live.openai.json new file mode 100644 index 00000000..663e5327 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.live.openai.json @@ -0,0 +1,9 @@ +{ + "answers": [ + "alpha", + "bravo" + ], + "promptChange": "in place", + "earlierSectionSent": true, + "change": "after" +} diff --git a/packages/harness/test/__artifacts__/prompt.foundation.json b/packages/harness/test/__artifacts__/prompt.foundation.json new file mode 100644 index 00000000..886be706 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.foundation.json @@ -0,0 +1,74 @@ +[ + { + "step": "before any choice", + "thread": "main", + "request": { + "foundation": "default", + "declarations": 1 + }, + "info": "default" + }, + { + "step": "after configuring main", + "thread": "main", + "request": { + "foundation": "reviewer", + "declarations": 1 + }, + "info": "reviewer" + }, + { + "step": "fork cut before the choice", + "thread": "early fork", + "request": { + "foundation": "default", + "declarations": 1 + }, + "info": "default" + }, + { + "step": "fork cut after the choice", + "thread": "late fork", + "request": { + "foundation": "reviewer", + "declarations": 1 + }, + "info": "reviewer" + }, + { + "step": "after configuring the fork", + "thread": "late fork", + "request": { + "foundation": "scout", + "declarations": 1 + }, + "info": "scout" + }, + { + "step": "main after the fork's choice", + "thread": "main", + "request": { + "foundation": "reviewer", + "declarations": 1 + }, + "info": "reviewer" + }, + { + "step": "main after a restart", + "thread": "main", + "request": { + "foundation": "reviewer", + "declarations": 1 + }, + "info": "reviewer" + }, + { + "step": "main after clearing", + "thread": "main", + "request": { + "foundation": "default", + "declarations": 1 + }, + "info": "default" + } +] diff --git a/packages/harness/test/__artifacts__/prompt.foundation.live.anthropic.json b/packages/harness/test/__artifacts__/prompt.foundation.live.anthropic.json new file mode 100644 index 00000000..3c0e48dd --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.foundation.live.anthropic.json @@ -0,0 +1,19 @@ +{ + "replies": { + "main": "none", + "reviewer": "REVIEW:", + "tester": "TESTS:", + "mainAfterForks": "none" + }, + "requestFoundations": [ + "default", + "reviewer", + "tester", + "default" + ], + "info": { + "main": "default", + "reviewer": "reviewer", + "tester": "tester" + } +} diff --git a/packages/harness/test/__artifacts__/prompt.foundation.live.copilot-claude-sonnet-5.5.json b/packages/harness/test/__artifacts__/prompt.foundation.live.copilot-claude-sonnet-5.5.json new file mode 100644 index 00000000..3c0e48dd --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.foundation.live.copilot-claude-sonnet-5.5.json @@ -0,0 +1,19 @@ +{ + "replies": { + "main": "none", + "reviewer": "REVIEW:", + "tester": "TESTS:", + "mainAfterForks": "none" + }, + "requestFoundations": [ + "default", + "reviewer", + "tester", + "default" + ], + "info": { + "main": "default", + "reviewer": "reviewer", + "tester": "tester" + } +} diff --git a/packages/harness/test/__artifacts__/prompt.foundation.live.openai-codex.json b/packages/harness/test/__artifacts__/prompt.foundation.live.openai-codex.json new file mode 100644 index 00000000..3c0e48dd --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.foundation.live.openai-codex.json @@ -0,0 +1,19 @@ +{ + "replies": { + "main": "none", + "reviewer": "REVIEW:", + "tester": "TESTS:", + "mainAfterForks": "none" + }, + "requestFoundations": [ + "default", + "reviewer", + "tester", + "default" + ], + "info": { + "main": "default", + "reviewer": "reviewer", + "tester": "tester" + } +} diff --git a/packages/harness/test/__artifacts__/prompt.foundation.live.openai.json b/packages/harness/test/__artifacts__/prompt.foundation.live.openai.json new file mode 100644 index 00000000..3c0e48dd --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.foundation.live.openai.json @@ -0,0 +1,19 @@ +{ + "replies": { + "main": "none", + "reviewer": "REVIEW:", + "tester": "TESTS:", + "mainAfterForks": "none" + }, + "requestFoundations": [ + "default", + "reviewer", + "tester", + "default" + ], + "info": { + "main": "default", + "reviewer": "reviewer", + "tester": "tester" + } +} diff --git a/packages/harness/test/__artifacts__/runner.retry.json b/packages/harness/test/__artifacts__/runner.retry.json index 22367eea..56dbd33f 100644 --- a/packages/harness/test/__artifacts__/runner.retry.json +++ b/packages/harness/test/__artifacts__/runner.retry.json @@ -3,6 +3,9 @@ { "type": "session.created" }, + { + "type": "session.thread.system.changed" + }, { "type": "session.turn.started" }, @@ -68,6 +71,12 @@ "state": "committed", "parts": 1 }, + { + "id": "system", + "type": "system", + "state": "committed", + "parts": 0 + }, { "id": "assistant_1", "type": "assistant", diff --git a/packages/harness/test/__artifacts__/thread.concurrent.json b/packages/harness/test/__artifacts__/thread.concurrent.json index c4e64d5f..54a60090 100644 --- a/packages/harness/test/__artifacts__/thread.concurrent.json +++ b/packages/harness/test/__artifacts__/thread.concurrent.json @@ -2,12 +2,14 @@ "bothActive": true, "main": [ "user committed", + "system committed", "assistant committed", "user committed", "assistant committed" ], "fork": [ "user committed", + "system committed", "assistant committed" ] } diff --git a/packages/harness/test/__artifacts__/thread.fork.json b/packages/harness/test/__artifacts__/thread.fork.json index 54ebb14d..07735f90 100644 --- a/packages/harness/test/__artifacts__/thread.fork.json +++ b/packages/harness/test/__artifacts__/thread.fork.json @@ -10,21 +10,22 @@ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", "3 session.prompt.promoted.1 T1", - "4 session.llm.started.1 T1", - "5 session.llm.ended.1 T1", - "6 session.turn.ended.1 T1", - "7 session.prompt.admitted.1 T1", - "8 session.prompt.promoted.1 T1", - "9 session.llm.started.1 T1", - "10 session.llm.ended.1 T1", - "11 session.turn.ended.1 T1", - "12 session.thread.forked.1 T2", - "13 session.thread.forked.1 T3", - "14 session.thread.forked.1 T4", - "15 session.prompt.admitted.1 T1", - "16 session.prompt.promoted.1 T1", - "17 session.llm.started.1 T1", - "18 session.llm.failed.1 T1" + "4 session.thread.system.changed.1 T1", + "5 session.llm.started.1 T1", + "6 session.llm.ended.1 T1", + "7 session.turn.ended.1 T1", + "8 session.prompt.admitted.1 T1", + "9 session.prompt.promoted.1 T1", + "10 session.llm.started.1 T1", + "11 session.llm.ended.1 T1", + "12 session.turn.ended.1 T1", + "13 session.thread.forked.1 T2", + "14 session.thread.forked.1 T3", + "15 session.thread.forked.1 T4", + "16 session.prompt.admitted.1 T1", + "17 session.prompt.promoted.1 T1", + "18 session.llm.started.1 T1", + "19 session.llm.failed.1 T1" ], "threads": [ { @@ -35,30 +36,31 @@ }, { "thread": "T2", - "origin": "fork T1 at E4", - "createdSeq": 12, + "origin": "fork T1 at E5", + "createdSeq": 13, "title": null }, { "thread": "T3", "origin": "fork T1 at E1", - "createdSeq": 13, + "createdSeq": 14, "title": null }, { "thread": "T4", - "origin": "fork T1 at E2", - "createdSeq": 14, + "origin": "fork T1 at E3", + "createdSeq": 15, "title": null } ], "entries": [ "E1 T1 user seq=3 committed=3 committed", - "E2 T1 assistant seq=4 committed=6 committed", - "E3 T1 user seq=8 committed=8 committed", - "E4 T1 assistant seq=9 committed=11 committed", - "E5 T1 user seq=16 committed=16 committed", - "E6 T1 assistant seq=17 committed=18 aborted" + "E2 T1 system seq=4 committed=4 committed", + "E3 T1 assistant seq=5 committed=7 committed", + "E4 T1 user seq=9 committed=9 committed", + "E5 T1 assistant seq=10 committed=12 committed", + "E6 T1 user seq=17 committed=17 committed", + "E7 T1 assistant seq=18 committed=19 aborted" ], "documents": [ { @@ -72,21 +74,21 @@ "thread": "T2", "kind": "agent", "revisions": [ - "12 {}" + "13 {}" ] }, { "thread": "T3", "kind": "agent", "revisions": [ - "13 {}" + "14 {}" ] }, { "thread": "T4", "kind": "agent", "revisions": [ - "14 {}" + "15 {}" ] } ] diff --git a/packages/harness/test/__artifacts__/thread.recovery.json b/packages/harness/test/__artifacts__/thread.recovery.json index 4faffb68..f5bdc287 100644 --- a/packages/harness/test/__artifacts__/thread.recovery.json +++ b/packages/harness/test/__artifacts__/thread.recovery.json @@ -3,17 +3,19 @@ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", "3 session.prompt.promoted.1 T1", - "4 session.llm.started.1 T1", - "5 session.llm.ended.1 T1", - "6 session.turn.ended.1 T1", - "7 session.thread.forked.1 T2", - "8 session.prompt.admitted.1 T2", - "9 session.prompt.promoted.1 T2", - "10 session.llm.started.1 T2", - "11 session.llm.failed.1 T2", - "12 session.llm.started.1 T2", - "13 session.llm.ended.1 T2", - "14 session.turn.ended.1 T2" + "4 session.thread.system.changed.1 T1", + "5 session.llm.started.1 T1", + "6 session.llm.ended.1 T1", + "7 session.turn.ended.1 T1", + "8 session.thread.forked.1 T2", + "9 session.prompt.admitted.1 T2", + "10 session.prompt.promoted.1 T2", + "11 session.llm.started.1 T2", + "12 session.llm.failed.1 T2", + "13 session.thread.system.changed.1 T2", + "14 session.llm.started.1 T2", + "15 session.llm.ended.1 T2", + "16 session.turn.ended.1 T2" ], "threads": [ { @@ -25,16 +27,18 @@ { "thread": "T2", "origin": "fork T1 at E1", - "createdSeq": 7, + "createdSeq": 8, "title": "cut off" } ], "entries": [ "E1 T1 user seq=3 committed=3 committed", - "E2 T1 assistant seq=4 committed=6 committed", - "E3 T2 user seq=9 committed=9 committed", - "E4 T2 assistant seq=10 committed=11 aborted", - "E5 T2 assistant seq=12 committed=14 committed" + "E2 T1 system seq=4 committed=4 committed", + "E3 T1 assistant seq=5 committed=7 committed", + "E4 T2 user seq=10 committed=10 committed", + "E5 T2 assistant seq=11 committed=12 aborted", + "E6 T2 system seq=13 committed=13 committed", + "E7 T2 assistant seq=14 committed=16 committed" ], "documents": [ { @@ -48,7 +52,7 @@ "thread": "T2", "kind": "agent", "revisions": [ - "7 {}" + "8 {}" ] } ] diff --git a/packages/harness/test/__artifacts__/thread.reset.json b/packages/harness/test/__artifacts__/thread.reset.json index 5b5a328e..f26abe59 100644 --- a/packages/harness/test/__artifacts__/thread.reset.json +++ b/packages/harness/test/__artifacts__/thread.reset.json @@ -2,22 +2,25 @@ "context": { "roles": [ "user", - "user" + "user", + "system" ], - "texts": "The conversation was started over. Carried over from before:\n\n\nkeep the login work\n | after the reset" + "texts": "The conversation was started over. Carried over from before:\n\n\nkeep the login work\n | after the reset | " }, "events": [ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", "3 session.prompt.promoted.1 T1", - "4 session.llm.started.1 T1", - "5 session.llm.ended.1 T1", - "6 session.turn.ended.1 T1", - "8 session.prompt.admitted.1 T1", - "9 session.prompt.promoted.1 T1", - "10 session.llm.started.1 T1", - "11 session.llm.ended.1 T1", - "12 session.turn.ended.1 T1" + "4 session.thread.system.changed.1 T1", + "5 session.llm.started.1 T1", + "6 session.llm.ended.1 T1", + "7 session.turn.ended.1 T1", + "9 session.prompt.admitted.1 T1", + "10 session.prompt.promoted.1 T1", + "11 session.thread.system.changed.1 T1", + "12 session.llm.started.1 T1", + "13 session.llm.ended.1 T1", + "14 session.turn.ended.1 T1" ], "threads": [ { @@ -29,10 +32,12 @@ ], "entries": [ "E1 T1 user seq=3 committed=3 committed", - "E2 T1 assistant seq=4 committed=6 committed", - "E3 T1 reset seq=7 committed=7 committed", - "E4 T1 user seq=9 committed=9 committed", - "E5 T1 assistant seq=10 committed=12 committed" + "E2 T1 system seq=4 committed=4 committed", + "E3 T1 assistant seq=5 committed=7 committed", + "E4 T1 reset seq=8 committed=8 committed", + "E5 T1 user seq=10 committed=10 committed", + "E6 T1 system seq=11 committed=11 committed", + "E7 T1 assistant seq=12 committed=14 committed" ], "documents": [ { diff --git a/packages/harness/test/__artifacts__/thread.walkthrough.json b/packages/harness/test/__artifacts__/thread.walkthrough.json index 6faf4ee8..fc5be405 100644 --- a/packages/harness/test/__artifacts__/thread.walkthrough.json +++ b/packages/harness/test/__artifacts__/thread.walkthrough.json @@ -4,27 +4,28 @@ "2 session.thread.configured.1 T1", "3 session.prompt.admitted.1 T1", "4 session.prompt.promoted.1 T1", - "5 session.llm.started.1 T1", - "6 session.llm.ended.1 T1", - "7 session.thread.configured.1 T1", - "8 session.tool.settled.1 T1", - "9 session.turn.ended.1 T1", - "10 session.llm.started.1 T1", - "11 session.llm.ended.1 T1", - "12 session.turn.ended.1 T1", - "13 session.prompt.admitted.1 T1", - "14 session.prompt.promoted.1 T1", - "15 session.llm.started.1 T1", - "16 session.llm.ended.1 T1", - "17 session.turn.ended.1 T1", - "18 session.thread.forked.1 T2", - "19 session.prompt.admitted.1 T2", - "20 session.prompt.promoted.1 T2", - "21 session.llm.started.1 T2", - "22 session.llm.ended.1 T2", - "23 session.turn.ended.1 T2", - "24 session.thread.configured.1 T2", - "25 session.thread.forked.1 T3" + "5 session.thread.system.changed.1 T1", + "6 session.llm.started.1 T1", + "7 session.llm.ended.1 T1", + "8 session.thread.configured.1 T1", + "9 session.tool.settled.1 T1", + "10 session.turn.ended.1 T1", + "11 session.llm.started.1 T1", + "12 session.llm.ended.1 T1", + "13 session.turn.ended.1 T1", + "14 session.prompt.admitted.1 T1", + "15 session.prompt.promoted.1 T1", + "16 session.llm.started.1 T1", + "17 session.llm.ended.1 T1", + "18 session.turn.ended.1 T1", + "19 session.thread.forked.1 T2", + "20 session.prompt.admitted.1 T2", + "21 session.prompt.promoted.1 T2", + "22 session.llm.started.1 T2", + "23 session.llm.ended.1 T2", + "24 session.turn.ended.1 T2", + "25 session.thread.configured.1 T2", + "26 session.thread.forked.1 T3" ], "threads": [ { @@ -35,25 +36,26 @@ }, { "thread": "T2", - "origin": "fork T1 at E3", - "createdSeq": 18, + "origin": "fork T1 at E4", + "createdSeq": 19, "title": "Use OAuth instead" }, { "thread": "T3", - "origin": "fork T1 at E2", - "createdSeq": 25, + "origin": "fork T1 at E3", + "createdSeq": 26, "title": null } ], "entries": [ "E1 T1 user seq=4 committed=4 committed", - "E2 T1 assistant seq=5 committed=9 committed", - "E3 T1 assistant seq=10 committed=12 committed", - "E4 T1 user seq=14 committed=14 committed", - "E5 T1 assistant seq=15 committed=17 committed", - "E6 T2 user seq=20 committed=20 committed", - "E7 T2 assistant seq=21 committed=23 committed" + "E2 T1 system seq=5 committed=5 committed", + "E3 T1 assistant seq=6 committed=10 committed", + "E4 T1 assistant seq=11 committed=13 committed", + "E5 T1 user seq=15 committed=15 committed", + "E6 T1 assistant seq=16 committed=18 committed", + "E7 T2 user seq=21 committed=21 committed", + "E8 T2 assistant seq=22 committed=24 committed" ], "documents": [ { @@ -62,27 +64,28 @@ "revisions": [ "1 {}", "2 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"}}", - "7 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" + "8 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" ] }, { "thread": "T2", "kind": "agent", "revisions": [ - "18 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}", - "24 {\"model\":{\"provider\":\"anthropic\",\"id\":\"claude-sonnet-4-5\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" + "19 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}", + "25 {\"model\":{\"provider\":\"anthropic\",\"id\":\"claude-sonnet-4-5\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" ] }, { "thread": "T3", "kind": "agent", "revisions": [ - "25 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" + "26 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" ] } ], "t2Context": [ "user", + "system", "assistant", "assistant", "user" diff --git a/packages/harness/test/__artifacts__/tools.read.vision.json b/packages/harness/test/__artifacts__/tools.read.vision.json index 36094334..f62e6c80 100644 --- a/packages/harness/test/__artifacts__/tools.read.vision.json +++ b/packages/harness/test/__artifacts__/tools.read.vision.json @@ -7,6 +7,9 @@ "content": [ "read the image" ] + }, + { + "role": "system" } ] }, @@ -19,6 +22,9 @@ "read the image" ] }, + { + "role": "system" + }, { "role": "assistant", "parts": [ @@ -48,6 +54,9 @@ "read the image" ] }, + { + "role": "system" + }, { "role": "assistant", "parts": [ diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.fork.json b/packages/harness/test/__artifacts__/tools.toolsearch.fork.json index 2aa7e65f..c585146b 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.fork.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.fork.json @@ -10,7 +10,17 @@ "toolsearch", "notes_read" ], - "system": [] + "system": [ + { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + } + ] }, "results": [ { @@ -46,6 +56,16 @@ ], "system": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "weather_forecast" ], @@ -67,13 +87,13 @@ "entries": [ { "id": "E1", - "seq": 24, + "seq": 26, "type": "user", "state": "committed" }, { "id": "E2", - "seq": 25, + "seq": 27, "type": "system", "state": "committed", "data": { @@ -113,13 +133,13 @@ }, { "id": "E3", - "seq": 26, + "seq": 28, "type": "assistant", "state": "committed" }, { "id": "E4", - "seq": 30, + "seq": 32, "type": "assistant", "state": "committed" } diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.limit.json b/packages/harness/test/__artifacts__/tools.toolsearch.limit.json index 47d5e1a9..f2773cd1 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.limit.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.limit.json @@ -57,6 +57,16 @@ ], "system": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "github_create_issue", "jira_search_issues", diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.live.anthropic.json b/packages/harness/test/__artifacts__/tools.toolsearch.live.anthropic.json new file mode 100644 index 00000000..acdce09e --- /dev/null +++ b/packages/harness/test/__artifacts__/tools.toolsearch.live.anthropic.json @@ -0,0 +1,52 @@ +{ + "requests": [ + { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch" + ], + "system": [ + [] + ] + }, + { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch" + ], + "system": [ + [], + [ + "vault_read_token" + ] + ] + }, + { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch" + ], + "system": [ + [], + [ + "vault_read_token" + ] + ] + } + ], + "calls": [ + "toolsearch:completed", + "vault_read_token:completed" + ], + "found": true, + "cachedAfterLoad": true +} diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.live.openai-codex.json b/packages/harness/test/__artifacts__/tools.toolsearch.live.openai-codex.json index ed9cba22..0c1ba101 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.live.openai-codex.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.live.openai-codex.json @@ -8,7 +8,9 @@ "edit", "toolsearch" ], - "system": [] + "system": [ + [] + ] }, { "tools": [ @@ -19,6 +21,7 @@ "toolsearch" ], "system": [ + [], [ "vault_read_token" ] @@ -33,6 +36,7 @@ "toolsearch" ], "system": [ + [], [ "vault_read_token" ] diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.load.json b/packages/harness/test/__artifacts__/tools.toolsearch.load.json index f622611c..330d91e8 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.load.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.load.json @@ -9,7 +9,17 @@ "toolsearch", "notes_read" ], - "system": [] + "system": [ + { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + } + ] }, { "tools": [ @@ -22,6 +32,16 @@ ], "system": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "weather_forecast" ], @@ -40,6 +60,16 @@ ], "system": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "weather_forecast" ], @@ -88,50 +118,54 @@ }, { "seq": 4, - "type": "session.llm.started.1" + "type": "session.thread.system.changed.1" }, { "seq": 5, - "type": "session.llm.ended.1" + "type": "session.llm.started.1" }, { "seq": 6, - "type": "session.tool.settled.1" + "type": "session.llm.ended.1" }, { "seq": 7, - "type": "session.turn.ended.1" + "type": "session.tool.settled.1" }, { "seq": 8, - "type": "session.thread.tools.changed.1" + "type": "session.turn.ended.1" }, { "seq": 9, - "type": "session.llm.started.1" + "type": "session.thread.system.changed.1" }, { "seq": 10, - "type": "session.llm.ended.1" + "type": "session.llm.started.1" }, { "seq": 11, - "type": "session.tool.settled.1" + "type": "session.llm.ended.1" }, { "seq": 12, - "type": "session.turn.ended.1" + "type": "session.tool.settled.1" }, { "seq": 13, - "type": "session.llm.started.1" + "type": "session.turn.ended.1" }, { "seq": 14, - "type": "session.llm.ended.1" + "type": "session.llm.started.1" }, { "seq": 15, + "type": "session.llm.ended.1" + }, + { + "seq": 16, "type": "session.turn.ended.1" } ], @@ -145,12 +179,28 @@ { "id": "E2", "seq": 4, + "type": "system", + "state": "committed", + "data": { + "sections": [ + "tools", + "rules", + "cwd" + ], + "toolsAdded": [], + "toolsRemoved": [], + "hash": "2aedbc0fffd70a2b41a7ce78f786f9809d727be73bf31b672ac2d603d3817fbb" + } + }, + { + "id": "E3", + "seq": 5, "type": "assistant", "state": "committed" }, { - "id": "E3", - "seq": 8, + "id": "E4", + "seq": 9, "type": "system", "state": "committed", "data": { @@ -189,14 +239,14 @@ } }, { - "id": "E4", - "seq": 9, + "id": "E5", + "seq": 10, "type": "assistant", "state": "committed" }, { - "id": "E5", - "seq": 13, + "id": "E6", + "seq": 14, "type": "assistant", "state": "committed" } @@ -207,7 +257,7 @@ "value": {} }, { - "seq": 6, + "seq": 7, "value": { "modelTools": { "add": [ diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json b/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json index 9dd4139e..c8151453 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json @@ -10,6 +10,11 @@ ], "system": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], "added": [], "removed": [ "notes_read" @@ -64,6 +69,11 @@ "type": "system", "state": "committed", "data": { + "sections": [ + "tools", + "rules", + "cwd" + ], "toolsAdded": [], "toolsRemoved": [ { diff --git a/packages/harness/test/fixtures/instruction.spec.ts b/packages/harness/test/fixtures/instruction.spec.ts index 6ff5cb66..b1fbefce 100644 --- a/packages/harness/test/fixtures/instruction.spec.ts +++ b/packages/harness/test/fixtures/instruction.spec.ts @@ -1,3 +1,4 @@ +import { Transcript } from "@codeworksh/aikit"; import { Effect, type Layer, type Scope } from "effect"; import fs from "node:fs/promises"; import os from "node:os"; @@ -77,7 +78,7 @@ const run = ( hostCwd: path.join(host.root, "cwd"), sandboxes: drivers, llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), diff --git a/packages/harness/test/fixtures/llm.ts b/packages/harness/test/fixtures/llm.ts index 3bf5fe87..5a127476 100644 --- a/packages/harness/test/fixtures/llm.ts +++ b/packages/harness/test/fixtures/llm.ts @@ -59,3 +59,10 @@ export const toolTurn = (...calls: ReadonlyArray): return stream; }); }; + +// Read from the checkout's AGENTS.md and installed skills, so they differ per machine. +const machineSections = new Set(["project_context", "skills"]); + +/** The section names a system message declares, without those that differ per machine. */ +export const sectionNames = (sections: Readonly> | undefined): Array => + Object.keys(sections ?? {}).filter((name) => !machineSections.has(name)); diff --git a/packages/harness/test/fixtures/mcp/turns.ts b/packages/harness/test/fixtures/mcp/turns.ts index 3617ff66..2c5abe06 100644 --- a/packages/harness/test/fixtures/mcp/turns.ts +++ b/packages/harness/test/fixtures/mcp/turns.ts @@ -10,7 +10,7 @@ import { Event } from "../../../src/event/event.ts"; import { EventList } from "../../../src/event/list.ts"; import { toolSearchPlugin } from "../../../src/plugin/builtin/tool/toolsearch.ts"; import type { LLM } from "../../../src/runner/llm.ts"; -import { assistant } from "../llm.ts"; +import { assistant, sectionNames } from "../llm.ts"; const fixtures = fileURLToPath(new URL("./", import.meta.url)); @@ -111,12 +111,13 @@ export const results = (thread: Session.ThreadHandle) => return out; }); -/** Tool changes the transcript declared, as names. */ +/** Section and tool changes the transcript declared, as names. */ export const changes = (messages: ReadonlyArray) => messages.flatMap((message) => message.role === "system" ? [ { + sections: sectionNames(message.sections), added: (message.toolsAdded ?? []).map((tool) => tool.name), removed: (message.toolsRemoved ?? []).map((tool) => tool.name), }, diff --git a/packages/harness/test/fixtures/skill.spec.ts b/packages/harness/test/fixtures/skill.spec.ts index ad2c0b0e..0db0f181 100644 --- a/packages/harness/test/fixtures/skill.spec.ts +++ b/packages/harness/test/fixtures/skill.spec.ts @@ -1,3 +1,4 @@ +import { Transcript } from "@codeworksh/aikit"; import { Effect, type Layer, type Scope } from "effect"; import fs from "node:fs/promises"; import os from "node:os"; @@ -87,7 +88,7 @@ const run = ( hostCwd: path.join(host.root, "cwd"), sandboxes: drivers, llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), diff --git a/packages/harness/test/mcp.thread.test.ts b/packages/harness/test/mcp.thread.test.ts index 536fec7f..8c6cee15 100644 --- a/packages/harness/test/mcp.thread.test.ts +++ b/packages/harness/test/mcp.thread.test.ts @@ -435,7 +435,11 @@ it("mcp: a thread resumed in a new process awaits the server of a tool it loaded expect(first.calls.first).toBe("hello: first"); expect(first.loaded?.add).toContain("mcp__docs__echo"); - expect(second.changes).toEqual([{ added: ["mcp__docs__echo", "mcp__docs__big"], removed: [] }]); + // The prompt and the loaded tools stay declared by the first run's system entries; nothing is declared again. + expect(second.changes).toEqual([ + { sections: ["tools", "rules", "cwd"], added: [], removed: [] }, + { sections: [], added: ["mcp__docs__echo", "mcp__docs__big"], removed: [] }, + ]); expect(second.calls.again).toBe("hello: again"); const { sessionId: _sessionId, ...firstRun } = first; await expect(`${JSON.stringify({ first: firstRun, second }, null, "\t")}\n`).toMatchFileSnapshot( diff --git a/packages/harness/test/plugin.examples.test.ts b/packages/harness/test/plugin.examples.test.ts index 99c7bf78..359ffaa8 100644 --- a/packages/harness/test/plugin.examples.test.ts +++ b/packages/harness/test/plugin.examples.test.ts @@ -1,5 +1,5 @@ import "./utils/env.ts"; -import type { Message } from "@codeworksh/aikit"; +import { type Message, Transcript } from "@codeworksh/aikit"; import { Effect } from "effect"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -65,7 +65,7 @@ describe("extras examples", () => { const { contexts } = await run(root, selection); const context = contexts[0]; expect(context?.tools?.map((tool) => tool.name)).toEqual(["bash", "list_processes", "project_fact"]); - const prompt = context?.systemPrompt ?? ""; + const prompt = context === undefined ? "" : Transcript.getCurrentSystemPrompt(context); // The prompt plugin was named after the tool plugins, so its index carries them. expect(prompt).toContain("- list_processes: List the processes running in the session's environment."); expect(prompt).toContain("- project_fact: Look up a project convention: deploy."); @@ -89,9 +89,11 @@ describe("extras examples", () => { selection, toolTurn(pendingCall("project_fact", { key: "deploy" }, "call_fact")), ); - expect(JSON.parse(path[1]?.parts[0]?.data ?? "{}")).toMatchObject({ - status: "completed", - result: { content: [{ type: "text", text: "vercel" }] }, - }); + expect(JSON.parse(path.find((item) => item.entry.type === "assistant")?.parts[0]?.data ?? "{}")).toMatchObject( + { + status: "completed", + result: { content: [{ type: "text", text: "vercel" }] }, + }, + ); })); }); diff --git a/packages/harness/test/plugin.external.test.ts b/packages/harness/test/plugin.external.test.ts index 76d21576..bb13ceec 100644 --- a/packages/harness/test/plugin.external.test.ts +++ b/packages/harness/test/plugin.external.test.ts @@ -1,5 +1,5 @@ import "./utils/env.ts"; -import type { Message } from "@codeworksh/aikit"; +import { type Message, Transcript } from "@codeworksh/aikit"; import { Cause, Effect, Exit, Fiber, Schema, Stream } from "effect"; import { join } from "node:path"; import { writeFile } from "node:fs/promises"; @@ -57,7 +57,7 @@ const exchange = (input: { database: ":memory:", llm: (request, signal) => { contexts.push(request.context); - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, ...(input.plugins === undefined ? {} : { plugins: input.plugins }), @@ -125,8 +125,8 @@ describe("third-party plugins", () => { expect(contexts[0]?.tools?.map((tool) => tool.name)).toEqual(["acme_echo"]); expect(prompts[0]).toContain("- acme_echo: Echo a value back"); - expect(path.map((item) => item.entry.type)).toEqual(["user", "assistant", "assistant"]); - const settled = JSON.parse(path[1]?.parts[0]?.data ?? "{}"); + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant", "assistant"]); + const settled = JSON.parse(path.find((item) => item.entry.type === "assistant")?.parts[0]?.data ?? "{}"); expect(settled).toMatchObject({ status: "completed", result: { @@ -153,7 +153,9 @@ describe("third-party plugins", () => { pendingCall("acme_secret", { value: "allow" }, "call_allowed"), ), }); - const parts = (path[1]?.parts ?? []).map((part) => JSON.parse(part.data)); + const parts = (path.find((item) => item.entry.type === "assistant")?.parts ?? []).map((part) => + JSON.parse(part.data), + ); const blocked = parts.find((part) => part.callID === "call_blocked"); const allowed = parts.find((part) => part.callID === "call_allowed"); expect(blocked).toMatchObject({ @@ -174,7 +176,7 @@ describe("third-party plugins", () => { llm: toolTurn(pendingCall("bash", { command: "echo hi" }, "call_bash")), }); expect(prompts[0]).toContain("- bash: Run a command through the acme shell"); - const settled = JSON.parse(path[1]?.parts[0]?.data ?? "{}"); + const settled = JSON.parse(path.find((item) => item.entry.type === "assistant")?.parts[0]?.data ?? "{}"); expect(settled).toMatchObject({ status: "completed", result: { content: [{ type: "text", text: "acme-override:echo hi" }] }, @@ -292,7 +294,7 @@ describe("third-party plugins", () => { }); expect(contexts[0]?.tools?.[0]).toMatchObject({ name: "acme_echo", description: "Echo, relabelled by acme" }); expect(prompts[0]).toContain("- acme_echo: Echo a value back"); - const settled = JSON.parse(path[1]?.parts[0]?.data ?? "{}"); + const settled = JSON.parse(path.find((item) => item.entry.type === "assistant")?.parts[0]?.data ?? "{}"); expect(settled.result.content).toEqual([ { type: "text", text: "hi" }, { type: "text", text: "(acme-checked)" }, @@ -340,7 +342,7 @@ describe("third-party plugins", () => { }); expect(contexts[0]?.tools).toEqual([]); expect(prompts[0]).toContain("\n(none)\n"); - const settled = JSON.parse(path[1]?.parts[0]?.data ?? "{}"); + const settled = JSON.parse(path.find((item) => item.entry.type === "assistant")?.parts[0]?.data ?? "{}"); expect(settled).toMatchObject({ status: "error", result: { isError: true } }); expect(settled.result.content[0].text).toContain("Unknown Tool: bash"); })); diff --git a/packages/harness/test/plugin.git.live.test.ts b/packages/harness/test/plugin.git.live.test.ts index 664510e3..7fd81b56 100644 --- a/packages/harness/test/plugin.git.live.test.ts +++ b/packages/harness/test/plugin.git.live.test.ts @@ -1,5 +1,5 @@ import "./utils/env.ts"; -import type { Message } from "@codeworksh/aikit"; +import { type Message, Transcript } from "@codeworksh/aikit"; import { Effect } from "effect"; import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -125,8 +125,8 @@ live("plugins installed from git, as a third party ships them", () => { ]); // A prompt plugin runs after every tool plugin, so it indexed the tool above even // though its settings entry is written after it. - expect(contexts[0]?.systemPrompt).toContain(""); - expect(contexts[0]?.systemPrompt).toContain( + expect(Transcript.getCurrentSystemPrompt(contexts[0]!)).toContain(""); + expect(Transcript.getCurrentSystemPrompt(contexts[0]!)).toContain( "Tools available: bash, read, write, edit, spacex_launch_manifest.", ); }), @@ -173,7 +173,7 @@ live("plugins installed from git, as a third party ships them", () => { const { contexts } = await exchange(workspace); // The 0.4.0 section, which 0.1.0 does not render. - expect(contexts[0]?.systemPrompt).toContain(""); + expect(Transcript.getCurrentSystemPrompt(contexts[0]!)).toContain(""); }), ); diff --git a/packages/harness/test/plugin.host.test.ts b/packages/harness/test/plugin.host.test.ts index 93ae96fb..5b293433 100644 --- a/packages/harness/test/plugin.host.test.ts +++ b/packages/harness/test/plugin.host.test.ts @@ -1,5 +1,5 @@ import "./utils/env.ts"; -import { createAssistantMessageEventStream, type Model } from "@codeworksh/aikit"; +import { createAssistantMessageEventStream, type Model, Transcript } from "@codeworksh/aikit"; import { Deferred, Effect, Fiber, Schema } from "effect"; import { describe, expect, it } from "vite-plus/test"; import { join } from "node:path"; @@ -42,9 +42,14 @@ describe("plugin domains and exchange host", () => { buckets.registry.prompt.foundation.set("Base."); const snapshot = buckets.freeze("/work"); // The patched guideline reaches the harness-rendered ; no snippet, so no index line. - expect(snapshot.systemPrompt).toBe( - ["Base.", "\n(none)\n", "\n- one\n", "\n/work\n"].join("\n\n"), - ); + expect(snapshot.prompt).toEqual({ + foundation: "Base.", + sections: { + tools: "\n(none)\n", + rules: "\n- one\n", + cwd: "\n/work\n", + }, + }); expect(snapshot.tools.defs[0]?.description).toBe("patched"); expect(snapshot.tools.wire[0]?.description).toBe("patched"); expect((await Effect.runPromise(snapshot.tools.handle(pendingCall("echo")))).status).toBe("completed"); @@ -93,7 +98,7 @@ describe("plugin domains and exchange host", () => { database: ":memory:", llm: (input, signal) => { models.push(input.resolvedModel); - observed.push(input.context.systemPrompt ?? ""); + observed.push(Transcript.getCurrentSystemPrompt(input.context)); return open(input, signal); }, plugins: [ @@ -326,7 +331,7 @@ describe("plugin pipelines in the kernel loop", () => { ]); if (mode === "sequential") expect(trace.indexOf("a:done")).toBeLessThan(trace.indexOf("b:before")); const path = yield* session.history(); - expect(path).toHaveLength(3); + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant", "assistant"]); }).pipe( Effect.provide( Harness.layer({ diff --git a/packages/harness/test/plugin.instruction.test.ts b/packages/harness/test/plugin.instruction.test.ts index 096e5b42..a310d49e 100644 --- a/packages/harness/test/plugin.instruction.test.ts +++ b/packages/harness/test/plugin.instruction.test.ts @@ -1,3 +1,4 @@ +import { Transcript } from "@codeworksh/aikit"; import "./utils/env.ts"; import { Effect, Logger } from "effect"; import { execFile } from "node:child_process"; @@ -46,7 +47,7 @@ const prompt = async (input: { root: string; directory: string; hostCwd: string; hostCwd: input.hostCwd, database: ":memory:", llm: (request, signal) => { - observed.push(request.context.systemPrompt ?? ""); + observed.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), diff --git a/packages/harness/test/plugin.passes.test.ts b/packages/harness/test/plugin.passes.test.ts index a32ba1e8..9a4ef0a8 100644 --- a/packages/harness/test/plugin.passes.test.ts +++ b/packages/harness/test/plugin.passes.test.ts @@ -1,3 +1,4 @@ +import { Transcript } from "@codeworksh/aikit"; import "./utils/env.ts"; import { Effect, Option } from "effect"; import { createHash } from "node:crypto"; @@ -108,7 +109,7 @@ describe("a running session", () => { hostCwd: project, database: ":memory:", llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), @@ -280,7 +281,7 @@ describe("a running session and the store", () => { hostCwd: project, database: ":memory:", llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), @@ -349,7 +350,7 @@ describe("boot", () => { hostCwd: project, database: ":memory:", llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), @@ -407,7 +408,7 @@ describe("boot", () => { hostCwd: bin, database: ":memory:", llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), @@ -464,7 +465,7 @@ describe("reload", () => { hostCwd: input.hostCwd ?? input.project, database: ":memory:", llm: (request, signal) => { - input.prompts.push(request.context.systemPrompt ?? ""); + input.prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), @@ -717,7 +718,7 @@ describe("linking a session to a host directory", () => { hostCwd: elsewhere, database: ":memory:", llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), diff --git a/packages/harness/test/plugin.prompt.test.ts b/packages/harness/test/plugin.prompt.test.ts index 887c68c7..75d98a5f 100644 --- a/packages/harness/test/plugin.prompt.test.ts +++ b/packages/harness/test/plugin.prompt.test.ts @@ -1,3 +1,4 @@ +import { Transcript } from "@codeworksh/aikit"; import "./utils/env.ts"; import { Effect, Schema } from "effect"; import { describe, expect, it } from "vite-plus/test"; @@ -38,7 +39,7 @@ describe("codework.prompt.default", () => { hostCwd: root, database: ":memory:", llm: (input, signal) => { - observed.push(input.context.systemPrompt ?? ""); + observed.push(Transcript.getCurrentSystemPrompt(input.context)); return open(input, signal); }, ...harness, diff --git a/packages/harness/test/plugin.skill.test.ts b/packages/harness/test/plugin.skill.test.ts index 5400d35b..62afbae4 100644 --- a/packages/harness/test/plugin.skill.test.ts +++ b/packages/harness/test/plugin.skill.test.ts @@ -1,3 +1,4 @@ +import { Transcript } from "@codeworksh/aikit"; import "./utils/env.ts"; import { Effect, Logger, Schema } from "effect"; import { execFile } from "node:child_process"; @@ -59,7 +60,7 @@ const prompt = async (input: { database: ":memory:", ...(input.plugins === undefined ? {} : { plugins: input.plugins }), llm: (request, signal) => { - observed.push(request.context.systemPrompt ?? ""); + observed.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), @@ -386,7 +387,7 @@ describe("codework.prompt.skill", () => { database: ":memory:", llm: (request, signal) => { requests.push({ - systemPrompt: request.context.systemPrompt ?? "", + systemPrompt: Transcript.getCurrentSystemPrompt(request.context), messages: request.context.messages, }); return open(request, signal); diff --git a/packages/harness/test/prompt.changes.live.test.ts b/packages/harness/test/prompt.changes.live.test.ts new file mode 100644 index 00000000..73b76be4 --- /dev/null +++ b/packages/harness/test/prompt.changes.live.test.ts @@ -0,0 +1,151 @@ +import { Effect } from "effect"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vite-plus/test"; +import { ContextCodec } from "../src/context/codec.ts"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import { define, Section } from "../src/plugin/index.ts"; +import { SessionRuntime } from "../src/session/runtime.ts"; +import type { State } from "../src/state/state.ts"; +import { withSettings } from "./fixtures/settings.ts"; +import "./utils/live.ts"; + +/** + * A prompt section changed between two prompts reaches a real model. These models take system + * messages in place, so the second request still carries the first codeword and adds the change + * after it, right after the prompt it came with. Where caching is explicit (Anthropic), the second request + * reads the first one's sections from the cache; OpenAI's is best effort, so it is not checked there. + */ + +const Codeword = Section.define("codeword"); +const Reference = Section.define("reference"); +/** A fixed section past every provider's minimum cacheable prefix. */ +const reference = Array.from({ length: 300 }, (_, i) => `Reference line ${i}: the archive holds item ${i * 7}.`).join( + "\n", +); +process.env.CODEWORK_CACHE_RETENTION ??= "short"; +const CASES = [ + { name: "openai-codex", provider: "openai-codex", id: "gpt-6-luna", env: "OPENAI_CODEX_API_KEY", change: "after" }, + { name: "openai", provider: "openai", id: "gpt-6-luna", env: "OPENAI_API_KEY", change: "after" }, + { + name: "anthropic", + provider: "anthropic", + id: "claude-sonnet-5-5", + env: "ANTHROPIC_API_KEY", + change: "after", + cache: true, + }, + { + name: "copilot-claude-sonnet-5.5", + provider: "github-copilot", + id: "claude-sonnet-5.5", + env: "COPILOT_GITHUB_TOKEN", + change: "after", + cache: true, + }, + { + name: "copilot-gpt-6-luna", + provider: "github-copilot", + id: "gpt-6-luna", + env: "COPILOT_GITHUB_TOKEN", + change: "after", + }, + // Without mid-conversation system messages: the change is folded into the leading prompt. + { name: "copilot-gpt-5.3-codex", provider: "github-copilot", id: "gpt-5.3-codex", env: "COPILOT_GITHUB_TOKEN" }, +] as const; + +describe.each(CASES)("prompt sections against live $name", (c) => { + const { name, provider, id, env } = c; + const change = "change" in c ? c.change : undefined; + const cache = "cache" in c && c.cache; + it.skipIf(!process.env[env]?.trim())( + "answers from a section changed between prompts, sent in place", + { timeout: 300_000 }, + () => + withSettings(async ({ root, custom }) => { + await writeFile(join(custom, "settings.jsonc"), JSON.stringify({ model: { provider, id } })); + const state = { word: "alpha" }; + const codeword = define({ + id: "acme.prompt.codeword", + kind: "prompt", + setup: (ctx) => { + ctx.plugin.prompt.sections.append(Reference, reference); + ctx.plugin.prompt.sections.append( + Codeword, + `The codeword is ${state.word}. Asked for the codeword, reply with it and nothing else.`, + ); + }, + }); + const bodies: string[] = []; + const onPayload: NonNullable = async (payload) => { + bodies.push(JSON.stringify(payload)); + return payload; + }; + + const { replies, cacheRead } = await Effect.gen(function* () { + const handle = yield* Session.create({ directory: root }); + const runtime = yield* SessionRuntime.Service; + yield* runtime.update(handle.id, { onPayload }); + yield* handle.main.run("What is the codeword?"); + state.word = "bravo"; + yield* handle.main.run("What is the codeword now?"); + const messages = yield* Effect.forEach(yield* handle.main.history(), ContextCodec.decodeMessage); + const assistants = messages.flatMap((message) => (message.role === "assistant" ? [message] : [])); + return { + replies: assistants.map((message) => + message.parts + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join("") + .trim() + .toLowerCase(), + ), + cacheRead: assistants.at(-1)?.usage.cacheRead ?? 0, + }; + }).pipe( + Effect.provide( + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: ":memory:", + userConfigDir: custom, + plugins: [codeword], + }), + ), + Effect.scoped, + Effect.timeout("240 seconds"), + Effect.runPromise, + ); + + expect(bodies).toHaveLength(2); + expect(replies).toHaveLength(2); + expect(replies[0]).toContain("alpha"); + expect(replies[1]).toContain("bravo"); + const second = bodies[1]!; + const bravo = second.indexOf("The codeword is bravo."); + const artifact = { + answers: replies.map((reply) => ["alpha", "bravo"].find((word) => reply.includes(word))), + promptChange: second.includes('Updated system prompt section \\"codeword\\"') ? "in place" : "folded", + earlierSectionSent: second.includes("The codeword is alpha."), + ...(change === undefined + ? {} + : { + change: + bravo < 0 + ? "missing" + : bravo < second.indexOf("What is the codeword now?") + ? "before" + : "after", + }), + // The change kept the prefix: the second request reads the first one's sections from the cache. + ...(cache ? { cached: cacheRead > 0 } : {}), + }; + expect(artifact.promptChange).toBe(change === undefined ? "folded" : "in place"); + if (change !== undefined) expect(artifact.change).toBe(change); + if (cache) expect(artifact.cached).toBe(true); + await expect(`${JSON.stringify(artifact, null, "\t")}\n`).toMatchFileSnapshot( + `./__artifacts__/prompt.changes.live.${name}.json`, + ); + }), + ); +}); diff --git a/packages/harness/test/prompt.changes.test.ts b/packages/harness/test/prompt.changes.test.ts new file mode 100644 index 00000000..3e6ec274 --- /dev/null +++ b/packages/harness/test/prompt.changes.test.ts @@ -0,0 +1,118 @@ +import { type Message, Transcript } from "@codeworksh/aikit"; +import "./utils/env.ts"; +import { Effect } from "effect"; +import { describe, expect, it } from "vite-plus/test"; +import { join } from "node:path"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import { defaultPromptPlugin } from "../src/plugin/builtin/prompt/default.ts"; +import { define, Section } from "../src/plugin/index.ts"; +import { immediateOpen } from "./fixtures/llm.ts"; +import { withSettings } from "./fixtures/settings.ts"; + +const Mode = Section.define("mode"); + +/** A section's text only where the test sets it; the rest depend on the machine. */ +const shown = (sections: Message.SystemMessage["sections"]) => + Object.fromEntries( + Object.entries(sections ?? {}).map(([name, value]) => [name, name === Mode.name || value === null ? value : "…"]), + ); + +/** What each request carried: its roles, and the section patch of every system message in it. */ +const request = (context: Message.Context) => ({ + roles: context.messages.map((message) => message.role), + declared: context.messages.flatMap((message) => (message.role === "system" ? [shown(message.sections)] : [])), + mode: /\n([\s\S]*?)\n<\/mode>/.exec(Transcript.getCurrentSystemPrompt(context))?.[1] ?? null, +}); + +/** The developer messages a model with in-place system messages gets: the head, then each update. */ +const inPlace = (context: Message.Context) => { + const prompt = Transcript.resolveTranscriptPrompt(context, true); + const updates = context.messages + .filter((message): message is Message.SystemMessage => message.role === "system") + .slice(prompt.foldedSystemMessages) + .map(Transcript.renderSystemPromptUpdate); + return [prompt.systemPrompt, ...updates]; +}; + +const isPrefix = (head: ReadonlyArray, whole: ReadonlyArray) => + head.every((value, index) => whole[index] === value); + +describe("prompt sections across prompts", () => { + it("declares every section once, then only what changed, through resume and fork", () => + withSettings(async ({ root }) => { + const contexts: Message.Context[] = []; + // One open for both runtimes: its reply ids count on across them. + const open = immediateOpen(contexts); + const state: { mode: string | undefined } = { mode: "Plan only." }; + const runtime = () => + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: join(root, "sessions.db"), + llm: open, + plugins: [ + defaultPromptPlugin, + define({ + id: "acme.prompt.mode", + kind: "prompt", + setup: (ctx) => { + if (state.mode !== undefined) ctx.plugin.prompt.sections.append(Mode, state.mode); + }, + }), + ], + }); + const types = (history: ReadonlyArray<{ readonly entry: { readonly type: string } }>) => + history.map(({ entry }) => entry.type); + + const sessionId = await Effect.runPromise( + Effect.gen(function* () { + const { id, main } = yield* Session.create({ directory: root }); + yield* main.run("one"); + state.mode = "Build allowed."; + yield* main.run("two"); + state.mode = undefined; + yield* main.run("three"); + yield* main.run("four"); + return id; + }).pipe(Effect.provide(runtime()), Effect.scoped), + ); + + const threads = await Effect.runPromise( + Effect.gen(function* () { + const main = yield* Session.attach({ sessionId }); + // Resumed in a new process with nothing changed: the transcript already declares it all. + yield* main.run("five"); + // A removed section added again is declared again. + state.mode = "Plan again."; + yield* main.run("six"); + state.mode = undefined; + const history = yield* main.history(); + // Cut after the first reply, whose transcript still has the plan-only section. + const firstReply = history.find(({ entry }) => entry.type === "assistant")!; + const { thread: fork } = yield* main.fork({ entryId: firstReply.entry.id }); + yield* fork.run("seven"); + return { main: types(yield* main.history()), fork: types(yield* fork.history()) }; + }).pipe(Effect.provide(runtime()), Effect.scoped), + ); + + const requests = contexts.map(request); + // Each request extends the one before it, so a model taking changes in place keeps its prefix. + const prefixKept = contexts + .slice(1, 6) + .map((context, index) => isPrefix(inPlace(contexts[index]!), inPlace(context))); + expect(prefixKept).toEqual([true, true, true, true, true]); + expect(requests.map(({ mode }) => mode)).toEqual([ + "Plan only.", + "Build allowed.", + null, + null, + null, + "Plan again.", + null, + ]); + await expect(JSON.stringify({ requests, prefixKept, threads }, null, "\t")).toMatchFileSnapshot( + "./__artifacts__/prompt.changes.json", + ); + })); +}); diff --git a/packages/harness/test/prompt.foundation.live.test.ts b/packages/harness/test/prompt.foundation.live.test.ts new file mode 100644 index 00000000..afd52f24 --- /dev/null +++ b/packages/harness/test/prompt.foundation.live.test.ts @@ -0,0 +1,137 @@ +import { Effect } from "effect"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vite-plus/test"; +import { ContextCodec } from "../src/context/codec.ts"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import { SessionRuntime } from "../src/session/runtime.ts"; +import type { State } from "../src/state/state.ts"; +import { withSettings } from "./fixtures/settings.ts"; +import "./utils/live.ts"; + +/** + * One session on a buggy function, three threads: main explains it with the default foundation, + * a reviewer fork and a tester fork each lead with their own. Each reply opens with its thread's + * marker, every request carries its own thread's foundation, and the thread info reports it. + */ + +const REVIEWER = + "You are a strict code reviewer. Begin every reply with the line `REVIEW:`, then list the concrete defects you find."; +const TESTER = + "You write unit tests. Begin every reply with the line `TESTS:`, then give the test code and nothing else."; +const CODE = "function average(xs) { return xs.reduce((a, b) => a + b) / xs.length; }"; + +const CASES = [ + { name: "anthropic", provider: "anthropic", id: "claude-sonnet-5-5", env: "ANTHROPIC_API_KEY" }, + { name: "openai", provider: "openai", id: "gpt-6-luna", env: "OPENAI_API_KEY" }, + { name: "openai-codex", provider: "openai-codex", id: "gpt-6-luna", env: "OPENAI_CODEX_API_KEY" }, + { + name: "copilot-claude-sonnet-5.5", + provider: "github-copilot", + id: "claude-sonnet-5.5", + env: "COPILOT_GITHUB_TOKEN", + }, +] as const; + +describe.each(CASES)("thread foundations against live $name", ({ name, provider, id, env }) => { + it.skipIf(!process.env[env]?.trim())("leads each forked thread with its own foundation", { timeout: 300_000 }, () => + withSettings(async ({ root, custom }) => { + await writeFile(join(custom, "settings.jsonc"), JSON.stringify({ model: { provider, id } })); + const bodies: string[] = []; + const onPayload: NonNullable = async (payload) => { + bodies.push(JSON.stringify(payload)); + return payload; + }; + /** The text of a thread's last reply. */ + const lastReply = (thread: Session.ThreadHandle) => + Effect.gen(function* () { + const messages = yield* Effect.forEach(yield* thread.history(), ContextCodec.decodeMessage); + const reply = messages.findLast((message) => message.role === "assistant")!; + return ContextCodec.parts(reply) + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join("") + .trim(); + }); + + const result = await Effect.gen(function* () { + const handle = yield* Session.create({ directory: root }); + const runtime = yield* SessionRuntime.Service; + yield* runtime.update(handle.id, { onPayload }); + const main = handle.main; + yield* main.run(`Here is my function:\n\n${CODE}\n\nIn one sentence, what does it do?`); + const explained = (yield* main.history()).findLast(({ entry }) => entry.type === "assistant")!; + const explanation = yield* lastReply(main); + + const { thread: reviewer } = yield* main.fork({ entryId: explained.entry.id }); + yield* reviewer.configure({ foundation: REVIEWER }); + yield* reviewer.run("Review the function."); + const review = yield* lastReply(reviewer); + + const { thread: tester } = yield* main.fork({ entryId: explained.entry.id }); + yield* tester.configure({ foundation: TESTER }); + yield* tester.run("Write one test for the input that breaks it."); + const tests = yield* lastReply(tester); + + yield* main.run("Thanks. Is there anything to watch out for when calling it?"); + const followUp = yield* lastReply(main); + + const info = { + main: (yield* main.info).foundation, + reviewer: (yield* reviewer.info).foundation, + tester: (yield* tester.info).foundation, + }; + return { explanation, review, tests, followUp, info }; + }).pipe( + Effect.provide( + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: ":memory:", + userConfigDir: custom, + plugins: [], + }), + ), + Effect.scoped, + Effect.timeout("240 seconds"), + Effect.runPromise, + ); + + const opensWith = (text: string) => + text.startsWith("REVIEW:") ? "REVIEW:" : text.startsWith("TESTS:") ? "TESTS:" : "none"; + // Requests in order: main, reviewer, tester, main. + const foundations = bodies.map((body) => + body.includes("strict code reviewer") + ? "reviewer" + : body.includes("You write unit tests") + ? "tester" + : "default", + ); + const artifact = { + replies: { + main: opensWith(result.explanation), + reviewer: opensWith(result.review), + tester: opensWith(result.tests), + mainAfterForks: opensWith(result.followUp), + }, + requestFoundations: foundations, + info: { + main: result.info.main === undefined ? "default" : "own", + reviewer: result.info.reviewer === REVIEWER ? "reviewer" : "other", + tester: result.info.tester === TESTER ? "tester" : "other", + }, + }; + expect(artifact.replies).toEqual({ + main: "none", + reviewer: "REVIEW:", + tester: "TESTS:", + mainAfterForks: "none", + }); + expect(foundations).toEqual(["default", "reviewer", "tester", "default"]); + expect(artifact.info).toEqual({ main: "default", reviewer: "reviewer", tester: "tester" }); + await expect(`${JSON.stringify(artifact, null, "\t")}\n`).toMatchFileSnapshot( + `./__artifacts__/prompt.foundation.live.${name}.json`, + ); + }), + ); +}); diff --git a/packages/harness/test/prompt.foundation.test.ts b/packages/harness/test/prompt.foundation.test.ts new file mode 100644 index 00000000..4b0839f6 --- /dev/null +++ b/packages/harness/test/prompt.foundation.test.ts @@ -0,0 +1,110 @@ +import type { Message } from "@codeworksh/aikit"; +import "./utils/env.ts"; +import { Effect } from "effect"; +import { join } from "node:path"; +import { describe, expect, it } from "vite-plus/test"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import { defaultPromptPlugin } from "../src/plugin/builtin/prompt/default.ts"; +import { immediateOpen } from "./fixtures/llm.ts"; +import { withSettings } from "./fixtures/settings.ts"; + +const REVIEWER = "You are a code reviewer."; +const SCOUT = "You are a scout."; + +/** A foundation by the persona it names; plugins render the default. */ +const persona = (foundation: string | undefined) => + foundation === REVIEWER ? "reviewer" : foundation === SCOUT ? "scout" : "default"; + +/** The foundation a request led with, and how many section declarations it carried. */ +const request = (context: Message.Context) => ({ + foundation: persona(context.systemPrompt), + declarations: context.messages.filter((message) => message.role === "system").length, +}); + +describe("a thread's own prompt foundation", () => { + it("leads only that thread's requests, follows forks as of the cut, survives a restart and clears", () => + withSettings(async ({ root }) => { + const contexts: Message.Context[] = []; + // One open for both runtimes: its reply ids count on across them. + const open = immediateOpen(contexts); + const runtime = () => + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: join(root, "sessions.db"), + llm: open, + plugins: [defaultPromptPlugin], + }); + const steps: Array<{ + step: string; + thread: string; + request: ReturnType; + info: ReturnType; + }> = []; + /** What the thread's last request led with, and what its info reports. */ + const ran = (step: string, thread: string, handle: Session.ThreadHandle) => + Effect.map(handle.info, (info) => + steps.push({ step, thread, request: request(contexts.at(-1)!), info: persona(info.foundation) }), + ); + + const sessionId = await Effect.runPromise( + Effect.gen(function* () { + const { id, main } = yield* Session.create({ directory: root }); + yield* main.run("one"); + yield* ran("before any choice", "main", main); + const firstReply = (yield* main.history()).find(({ entry }) => entry.type === "assistant")!; + + yield* main.configure({ foundation: REVIEWER }); + yield* main.run("two"); + yield* ran("after configuring main", "main", main); + + // Cut before the choice: the fork has the foundation of that point. + const { thread: early } = yield* main.fork({ entryId: firstReply.entry.id }); + yield* early.run("three"); + yield* ran("fork cut before the choice", "early fork", early); + + // Cut after it: the fork starts with main's choice, then makes its own. + const latest = (yield* main.history()).findLast(({ entry }) => entry.type === "assistant")!; + const { thread: late } = yield* main.fork({ entryId: latest.entry.id }); + yield* late.run("four"); + yield* ran("fork cut after the choice", "late fork", late); + yield* late.configure({ foundation: SCOUT }); + yield* late.run("five"); + yield* ran("after configuring the fork", "late fork", late); + yield* main.run("six"); + yield* ran("main after the fork's choice", "main", main); + return id; + }).pipe(Effect.provide(runtime()), Effect.scoped), + ); + + await Effect.runPromise( + Effect.gen(function* () { + const main = yield* Session.attach({ sessionId }); + yield* main.run("seven"); + yield* ran("main after a restart", "main", main); + yield* main.configure({ foundation: null }); + yield* main.run("eight"); + yield* ran("main after clearing", "main", main); + }).pipe(Effect.provide(runtime()), Effect.scoped), + ); + + expect(steps.map(({ request }) => request.foundation)).toEqual([ + "default", + "reviewer", + "default", + "reviewer", + "scout", + "reviewer", + "reviewer", + "default", + ]); + // Each thread's info reports the foundation its request led with. + expect(steps.every(({ request, info }) => request.foundation === info)).toBe(true); + // A foundation choice declares no section again: each thread's declarations stay at one. + expect(steps.every(({ request }) => request.declarations === 1)).toBe(true); + await expect(`${JSON.stringify(steps, null, "\t")}\n`).toMatchFileSnapshot( + "./__artifacts__/prompt.foundation.json", + ); + })); +}); diff --git a/packages/harness/test/runner.loop.test.ts b/packages/harness/test/runner.loop.test.ts index bf4f8ae1..a5c98be1 100644 --- a/packages/harness/test/runner.loop.test.ts +++ b/packages/harness/test/runner.loop.test.ts @@ -4,7 +4,7 @@ import { Global } from "../src/global.ts"; import { builtins as plugins } from "../src/plugin/builtin.ts"; import type { Info } from "../src/settings/schema.ts"; import { Settings } from "../src/settings/settings.ts"; -import { createAssistantMessageEventStream, Message } from "@codeworksh/aikit"; +import { createAssistantMessageEventStream, Message, Transcript } from "@codeworksh/aikit"; import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Schema } from "effect"; import { SqlClient, SqlSchema } from "effect/sql"; import { describe, expect } from "vite-plus/test"; @@ -173,6 +173,7 @@ describe("runner loop — aikit input/output", () => { expect(bySeq.map((row) => row.id)).toEqual(["msg_steer", "msg_follow"]); expect((yield* threads.line(threadId)).map((item) => item.entry.type)).toEqual([ "user", + "system", "assistant", "user", "assistant", @@ -181,8 +182,8 @@ describe("runner loop — aikit input/output", () => { // Every request is rebuilt from durable history. The second request sees // the first terminal assistant and the newly promoted follow-up. expect(contexts.map((context) => context.messages.map((message) => message.role))).toEqual([ - ["user"], - ["user", "assistant", "user"], + ["user", "system"], + ["user", "system", "assistant", "user"], ]); }), ); @@ -199,7 +200,11 @@ describe("runner loop — aikit input/output", () => { yield* execution.resume(threadId); expect(yield* delivered(threadId)).toEqual([{ id: "msg_once", promotedSeq: 3 }]); - expect((yield* threads.line(threadId)).map((item) => item.entry.type)).toEqual(["user", "assistant"]); + expect((yield* threads.line(threadId)).map((item) => item.entry.type)).toEqual([ + "user", + "system", + "assistant", + ]); }), ); @@ -222,6 +227,7 @@ describe("runner loop — aikit input/output", () => { "user", "user", "user", + "system", "assistant", "user", "assistant", @@ -343,14 +349,15 @@ describe("runner loop — tool continuation and lifecycle gate", () => { expect(contexts).toHaveLength(2); expect(snapshots).toBe(2); - const second = contexts[1]!.messages[1]!; + const second = contexts[1]!.messages.find((message) => message.role === "assistant"); const firstTool = - second.role === "assistant" ? second.parts.find((part) => part.type === "toolCall") : undefined; + second?.role === "assistant" ? second.parts.find((part) => part.type === "toolCall") : undefined; expect(firstTool).toMatchObject({ callID: "call_echo", status: "completed" }); const path = yield* threads.line(threadId); expect(path.map((item) => [item.entry.type, item.entry.state])).toEqual([ ["user", "committed"], + ["system", "committed"], ["assistant", "committed"], ["assistant", "committed"], ]); @@ -359,8 +366,8 @@ describe("runner loop — tool continuation and lifecycle gate", () => { `) as ReadonlyArray<{ readonly type: string; readonly seq: number }>; const firstStarted = eventRows.find((row) => row.type === "session.llm.started.1")!; const firstEnded = eventRows.find((row) => row.type === "session.llm.ended.1")!; - expect(path[1]!.entry.seq).toBe(firstStarted.seq); - expect(path[1]!.entry.seq).toBeLessThan(firstEnded.seq); + expect(path[2]!.entry.seq).toBe(firstStarted.seq); + expect(path[2]!.entry.seq).toBeLessThan(firstEnded.seq); expect(path.filter((item) => item.entry.id === "assistant_1")).toHaveLength(1); }), ); @@ -505,9 +512,10 @@ describe("runner loop — tool interruption", () => { expect(interrupted.data.reason).toBe("user"); const path = yield* threads.line(threadId); - expect(path[1]!.entry.state).toBe("committed"); - expect(path[1]!.parts.map((part) => Option.getOrThrow(part.status))).toEqual(["aborted", "aborted"]); - for (const part of path[1]!.parts) { + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); + expect(path[2]!.entry.state).toBe("committed"); + expect(path[2]!.parts.map((part) => Option.getOrThrow(part.status))).toEqual(["aborted", "aborted"]); + for (const part of path[2]!.parts) { expect(JSON.parse(part.data)).toMatchObject({ status: "aborted", result: { content: [{ type: "text", text: "Tool Execution Interrupted" }], isError: true }, @@ -570,10 +578,10 @@ describe("runner loop — provider interruption", () => { expect(Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause)).toBe(true); const path = yield* threads.line(threadId); - expect(path.map((item) => item.entry.type)).toEqual(["user", "assistant"]); - expect(path[1]!.entry.state).toBe("aborted"); - expect(JSON.parse(path[1]!.entry.data)).toMatchObject({ stopReason: "aborted" }); - expect(path[1]!.parts).toEqual([]); + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); + expect(path[2]!.entry.state).toBe("aborted"); + expect(JSON.parse(path[2]!.entry.data)).toMatchObject({ stopReason: "aborted" }); + expect(path[2]!.parts).toEqual([]); expect(Array.from(yield* execution.active)).toEqual([]); }), { timeout: 10_000 }, @@ -627,10 +635,10 @@ describe("runner loop — provider failure", () => { } const path = yield* threads.line(threadId); - expect(path.map((item) => item.entry.type)).toEqual(["user", "assistant"]); - expect(path[1]!.entry.state).toBe("aborted"); - expect(path[1]!.parts).toEqual([]); - expect(JSON.parse(path[1]!.entry.data)).toMatchObject({ + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); + expect(path[2]!.entry.state).toBe("aborted"); + expect(path[2]!.parts).toEqual([]); + expect(JSON.parse(path[2]!.entry.data)).toMatchObject({ stopReason: "error", errorMessage: "provider failed", failure: { _tag: "Authentication", reason: "missing", retryable: false }, @@ -721,6 +729,8 @@ describe("runner loop — retry", () => { Effect.sync(() => { if (!event.type.startsWith("session.")) return; if (event.type.startsWith("session.prompt.") || event.type.startsWith("session.execution.")) return; + // A system entry's id is random. + if (event.type === EventList.SystemChanged.type) return void seen.push({ type: event.type }); const data = event.data as Record; const picked = Object.fromEntries( ["messageId", "attempt", "maxRetries", "delayMs", "success", "reason", "cause", "message"] @@ -736,7 +746,7 @@ describe("runner loop — retry", () => { const pathOf = Effect.fnUntraced(function* (threadId: ThreadSchema.ID) { const threads = yield* Thread.Service; return (yield* threads.line(threadId)).map((item) => ({ - id: item.entry.id, + id: item.entry.type === "system" ? "system" : item.entry.id, type: item.entry.type, state: item.entry.state, parts: item.parts.length, @@ -765,7 +775,13 @@ describe("runner loop — retry", () => { yield* execution.resume(threadId); const path = yield* pathOf(threadId); - expect(path.map((entry) => entry.state)).toEqual(["committed", "aborted", "aborted", "committed"]); + expect(path.map((entry) => entry.state)).toEqual([ + "committed", + "committed", + "aborted", + "aborted", + "committed", + ]); // Every attempt is sent the original context: the failed drafts and their partial text never leak in. expect(contexts).toHaveLength(3); expect(contexts[1]).toEqual(contexts[0]); @@ -804,7 +820,11 @@ describe("runner loop — retry", () => { const exit = yield* Fiber.await(waiting); expect(Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause)).toBe(true); - expect((yield* pathOf(threadId)).map((entry) => entry.state)).toEqual(["committed", "aborted"]); + expect((yield* pathOf(threadId)).map((entry) => entry.state)).toEqual([ + "committed", + "committed", + "aborted", + ]); expect(seen.slice(-2)).toEqual([ { type: EventList.RetryFinished.type, attempt: 1, success: false, message: "interrupted" }, { type: EventList.TurnAborted.type, cause: { _tag: "interrupted" } }, @@ -835,7 +855,11 @@ describe("runner loop — retry", () => { const exit = yield* execution.resume(threadId).pipe(Effect.exit); expect(Exit.isFailure(exit)).toBe(true); expect(seen.some((event) => String(event.type).startsWith("session.retry."))).toBe(false); - expect((yield* pathOf(threadId)).map((entry) => entry.state)).toEqual(["committed", "aborted"]); + expect((yield* pathOf(threadId)).map((entry) => entry.state)).toEqual([ + "committed", + "committed", + "aborted", + ]); }), ); } @@ -1254,6 +1278,7 @@ describe("runner loop — resume after restart", () => { expect(path.map((entry) => [entry.entry.type, entry.entry.state])).toEqual([ ["user", "committed"], ["assistant", "aborted"], + ["system", "committed"], ["assistant", "committed"], ]); expect(yield* stateOf(threadId)).toEqual({ execution: null, restarts: 0 }); @@ -1535,9 +1560,10 @@ describe("runner loop — lifecycle hooks", () => { const path = yield* threads.line(threadId); expect(path.map((item) => [item.entry.type, item.entry.state])).toEqual([ ["user", "committed"], + ["system", "committed"], ["assistant", "committed"], ]); - const tool = path[1]!.parts.find((part) => part.type === "toolCall"); + const tool = path[2]!.parts.find((part) => part.type === "toolCall"); expect(tool).toBeDefined(); const rows = yield* delivered(threadId); expect(rows.find((row) => row.id === "msg_queued")?.promotedSeq).toBeNull(); @@ -1545,8 +1571,14 @@ describe("runner loop — lifecycle hooks", () => { // The next run answers the tool results first, then picks the queued follow-up up. yield* execution.resume(threadId); expect(contexts).toHaveLength(3); - expect(contexts[1]!.messages).toHaveLength(2); - expect(contexts[2]!.messages).toHaveLength(4); + expect(contexts[1]!.messages.map((message) => message.role)).toEqual(["user", "system", "assistant"]); + expect(contexts[2]!.messages.map((message) => message.role)).toEqual([ + "user", + "system", + "assistant", + "assistant", + "user", + ]); const next = yield* delivered(threadId); expect(next.find((row) => row.id === "msg_queued")?.promotedSeq).not.toBeNull(); }), @@ -1661,7 +1693,8 @@ describe("runner loop — lifecycle hooks", () => { runtime({ open: recording(contexts), hooks: hooksOf({ - prepareNextTurn: (_turn, next) => during.run.pipe(Effect.as({ ...next, systemPrompt: "replaced" })), + prepareNextTurn: (_turn, next) => + during.run.pipe(Effect.as({ ...next, prompt: { ...next.prompt, foundation: "replaced" } })), }), plugins: [...plugins, { id: "test.tool.echo", kind: "tool", setup: (ctx) => ctx.plugin.tools.add(echo) }], }), @@ -1831,9 +1864,11 @@ describe("runner loop — tool exposure", () => { expect(contexts).toHaveLength(2); const section = (name: string) => - new RegExp(`<${name}>\\n([\\s\\S]*?)\\n`).exec(contexts[0]!.systemPrompt ?? "")?.[1]; - const reply = contexts[1]!.messages[1]!; - const call = reply.role === "assistant" ? reply.parts.find((part) => part.type === "toolCall") : undefined; + new RegExp(`<${name}>\\n([\\s\\S]*?)\\n`).exec( + Transcript.getCurrentSystemPrompt(contexts[0]!), + )?.[1]; + const reply = contexts[1]!.messages.find((message) => message.role === "assistant"); + const call = reply?.role === "assistant" ? reply.parts.find((part) => part.type === "toolCall") : undefined; const result = { requestTools: contexts.map((context) => (context.tools ?? []).map((tool) => tool.name)), tools: section("tools"), diff --git a/packages/harness/test/sdk.test.ts b/packages/harness/test/sdk.test.ts index 5d6b44f2..3edb0faa 100644 --- a/packages/harness/test/sdk.test.ts +++ b/packages/harness/test/sdk.test.ts @@ -86,8 +86,10 @@ describe("Harness Effect SDK", () => { yield* Effect.gen(function* () { const session = yield* Session.attach({ sessionId }); yield* session.run("second"); + // The restarted runtime finds the prompt the first run declared, so declares nothing again. expect((yield* session.history()).map(({ entry }) => entry.type)).toEqual([ "user", + "system", "assistant", "user", "assistant", diff --git a/packages/harness/test/settings.loop.test.ts b/packages/harness/test/settings.loop.test.ts index c4b35dbc..e30fac4b 100644 --- a/packages/harness/test/settings.loop.test.ts +++ b/packages/harness/test/settings.loop.test.ts @@ -1,5 +1,5 @@ import "./utils/env.ts"; -import { createAssistantMessageEventStream } from "@codeworksh/aikit"; +import { createAssistantMessageEventStream, Transcript } from "@codeworksh/aikit"; import { Deferred, Effect, Fiber, Schema } from "effect"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -100,7 +100,9 @@ describe("settings at turn boundaries", () => { // The "high" edit landed during the tool batch, so the continuation turn reads it. expect(inputs.map((input) => input.thinkingLevel)).toEqual(["low", "high", "high"]); expect(inputs.map((input) => input.options?.headers?.revision)).toEqual(["low", "high", "high"]); - expect(inputs[1]?.context.systemPrompt).toBe(inputs[0]?.context.systemPrompt); + expect(Transcript.getCurrentSystemPrompt(inputs[1]!.context)).toBe( + Transcript.getCurrentSystemPrompt(inputs[0]!.context), + ); yield* Effect.promise(() => file(custom, "off")); yield* handle.run("idle steer"); expect(inputs.at(-1)?.thinkingLevel).toBe("off"); diff --git a/packages/harness/test/thread.test.ts b/packages/harness/test/thread.test.ts index c4e85ca2..60b2383d 100644 --- a/packages/harness/test/thread.test.ts +++ b/packages/harness/test/thread.test.ts @@ -306,8 +306,15 @@ describe("threads", () => { yield* run(sessionId, main, "Now add tests"); const line = yield* threads.line(main); - expect(line.map(({ entry }) => entry.type)).toEqual(["user", "assistant", "assistant", "user", "assistant"]); - const [, e2, e3] = line; + expect(line.map(({ entry }) => entry.type)).toEqual([ + "user", + "system", + "assistant", + "assistant", + "user", + "assistant", + ]); + const [, , e2, e3] = line; const forked = yield* threads.fork({ threadId: main, entryId: e3!.entry.id }); const t2 = ThreadSchema.ID.make(forked.thread.id); @@ -352,7 +359,9 @@ describe("threads", () => { const { sessionId, main } = yield* createSession; yield* run(sessionId, main, "first"); yield* run(sessionId, main, "second"); - const [e1, e2, e3, e4] = yield* threads.line(main); + const line = yield* threads.line(main); + expect(line.map(({ entry }) => entry.type)).toEqual(["user", "system", "assistant", "user", "assistant"]); + const [e1, , e2, e3, e4] = line; // An inherited entry re-points the origin to the thread that wrote it. const t2 = ThreadSchema.ID.make((yield* threads.fork({ threadId: main, entryId: e4!.entry.id })).thread.id); @@ -445,11 +454,13 @@ describe("threads", () => { const result = { bothActive: true, main: yield* own(main), fork: yield* own(t2) }; expect(result.main).toEqual([ "user committed", + "system committed", "assistant committed", "user committed", "assistant committed", ]); - expect(result.fork).toEqual(["user committed", "assistant committed"]); + // The fork cuts before main's system entry, so its first request declares its own. + expect(result.fork).toEqual(["user committed", "system committed", "assistant committed"]); // Seqs interleave between the two runs, so the artifact holds per-thread facts only. yield* artifact("thread.concurrent.json", result); }), @@ -475,7 +486,8 @@ describe("threads", () => { }); yield* run(sessionId, main, "after the reset"); const last = contexts.at(-1)!; - expect(last.roles).toEqual(["user", "user"]); + // The reset drops the earlier system entry too, so the request declares the prompt again. + expect(last.roles).toEqual(["user", "user", "system"]); expect(last.texts.includes("before the reset")).toBe(false); expect(last.texts.includes("keep the login work")).toBe(true); yield* artifact("thread.reset.json", { @@ -537,14 +549,14 @@ describe("threads", () => { const own = (yield* threads.line(t2)) .filter(({ entry }) => entry.threadId === t2) .map(({ entry }) => `${entry.type} ${entry.state}`); - expect(own).toEqual(["user committed", "assistant aborted", "assistant committed"]); + expect(own).toEqual(["user committed", "assistant aborted", "system committed", "assistant committed"]); const thread = Option.getOrThrow(yield* threads.get(t2)); expect(Option.getOrUndefined(thread.execution)).toBeUndefined(); expect(thread.restarts).toBe(0); // Only the thread that was cut off ran again, from its committed entries: the inherited // prompt and its own, without the aborted draft. expect(contexts.filter((one) => one.threadId === main)).toHaveLength(1); - expect(contexts.at(-1)!.roles).toEqual(["user", "user"]); + expect(contexts.at(-1)!.roles).toEqual(["user", "user", "system"]); yield* artifact("thread.recovery.json", yield* snapshot(sessionId)); }), ); diff --git a/packages/harness/test/tools.toolsearch.live.test.ts b/packages/harness/test/tools.toolsearch.live.test.ts index 4ff4bacf..7a1f1a39 100644 --- a/packages/harness/test/tools.toolsearch.live.test.ts +++ b/packages/harness/test/tools.toolsearch.live.test.ts @@ -17,11 +17,16 @@ import { available, LIVE } from "./utils/live.ts"; /** * One run against a real model: the token is held by a deferred tool the model is not offered, * so a right reply means it searched, the load reached its next request, and it called the tool. + * On Anthropic the deferred catalog is declared up front and the load names the tool by reference, + * so the request after the load reads the one before it from the cache. */ const TOKEN = "vq7-lumen-4821"; -const codex = LIVE.find((one) => one.name === "openai-codex")!; -const MODEL = "gpt-6-luna"; +process.env.CODEWORK_CACHE_RETENTION ??= "short"; +const CASES = [ + { live: LIVE.find((one) => one.name === "openai-codex")!, model: "gpt-6-luna", cache: false }, + { live: LIVE.find((one) => one.name === "anthropic")!, model: "claude-sonnet-5-5", cache: true }, +]; /** A request as sent: its declared tools, and the tools its system messages add. */ type Sent = { readonly tools: ReadonlyArray; readonly system: ReadonlyArray> }; @@ -32,12 +37,12 @@ const sentOf = (context: Message.Context): Sent => ({ ), }); -describe("toolsearch against live openai-codex", () => { - it.skipIf(!available(codex))("finds a deferred tool, then calls it", { timeout: 300_000 }, () => +describe.each(CASES)("toolsearch against live $live.name", ({ live, model, cache }) => { + it.skipIf(!available(live))("finds a deferred tool, then calls it", { timeout: 300_000 }, () => withSettings(async ({ root, custom }) => { await writeFile( join(custom, "settings.jsonc"), - JSON.stringify({ model: { provider: codex.provider, id: MODEL } }), + JSON.stringify({ model: { provider: live.provider, id: model } }), ); const vault = define({ id: "acme.tool.vault", @@ -65,7 +70,7 @@ describe("toolsearch against live openai-codex", () => { return LLM.open(input, signal); }; - const { calls, reply } = await Effect.gen(function* () { + const { calls, reply, cacheReads } = await Effect.gen(function* () { const handle = yield* Session.create({ directory: root }); yield* handle.main.run( "Find the tool that reads the deployment access token from the secrets vault, call it, " + @@ -82,6 +87,10 @@ describe("toolsearch against live openai-codex", () => { reply: (last === undefined ? [] : ContextCodec.parts(last)) .flatMap((part) => (part.type === "text" ? [part.text] : [])) .join(""), + // One assistant message per request, in order. + cacheReads: messages.flatMap((message) => + message.role === "assistant" ? [message.usage.cacheRead] : [], + ), }; }).pipe( Effect.provide( @@ -108,10 +117,12 @@ describe("toolsearch against live openai-codex", () => { expect(loadedAt).toBeGreaterThan(0); expect(sent.every((one) => !one.tools.includes("vault_read_token"))).toBe(true); expect(found).toBe(true); + const cachedAfterLoad = (cacheReads[loadedAt] ?? 0) > 0; + if (cache) expect(cachedAfterLoad).toBe(true); - await expect(`${JSON.stringify({ requests: sent, calls, found }, null, "\t")}\n`).toMatchFileSnapshot( - "./__artifacts__/tools.toolsearch.live.openai-codex.json", - ); + await expect( + `${JSON.stringify({ requests: sent, calls, found, ...(cache ? { cachedAfterLoad } : {}) }, null, "\t")}\n`, + ).toMatchFileSnapshot(`./__artifacts__/tools.toolsearch.live.${live.name}.json`); }), ); }); diff --git a/packages/harness/test/tools.toolsearch.test.ts b/packages/harness/test/tools.toolsearch.test.ts index b698a32a..1ed32fcb 100644 --- a/packages/harness/test/tools.toolsearch.test.ts +++ b/packages/harness/test/tools.toolsearch.test.ts @@ -31,7 +31,7 @@ import { State } from "../src/state/state.ts"; import { ThreadSchema } from "../src/thread/schema.ts"; import { Thread } from "../src/thread/thread.ts"; import * as Tool from "../src/tool/tool.ts"; -import { assistant } from "./fixtures/llm.ts"; +import { assistant, sectionNames } from "./fixtures/llm.ts"; import { pooled } from "./fixtures/pool.ts"; import { seedSpace } from "./fixtures/space.ts"; import { testEffect } from "./utils/effect.ts"; @@ -233,13 +233,17 @@ const run = (sessionId: SessionSchema.ID, threadId: ThreadSchema.ID, prompt: str yield* control.run({ sessionId, threadId, prompt: PromptSchema.Prompt.fromText(prompt), delivery: "followUp" }); }); -/** A request as the model saw it: its declared tools, and the tool changes its system messages carry. */ +/** + * A request as the model saw it: its declared tools, and the section names and tool changes its + * system messages carry. + */ const sent = (context: Message.Context) => ({ tools: (context.tools ?? []).map((tool) => tool.name), system: context.messages.flatMap((message) => message.role === "system" ? [ { + sections: sectionNames(message.sections), added: (message.toolsAdded ?? []).map((tool) => tool.name), removed: (message.toolsRemoved ?? []).map((tool) => tool.name), }, @@ -274,6 +278,12 @@ const results = Effect.fnUntraced(function* (threadId: ThreadSchema.ID) { ); }); +/** A system entry's data, its sections by name only. */ +const systemData = (data: string) => { + const { sections, ...rest } = JSON.parse(data) as { readonly sections?: Record }; + return { ...(sections === undefined ? {} : { sections: sectionNames(sections) }), ...rest }; +}; + /** The thread's events, entries and `agent` revisions, by seq; ids aliased in order seen. */ const ledger = Effect.fnUntraced(function* (sessionId: SessionSchema.ID, threadId: ThreadSchema.ID) { const sql = yield* SqlClient.SqlClient; @@ -304,13 +314,16 @@ const ledger = Effect.fnUntraced(function* (sessionId: SessionSchema.ID, threadI seq: entry.seq, type: entry.type, state: entry.state, - ...(entry.type === "system" ? { data: JSON.parse(entry.data) as unknown } : {}), + ...(entry.type === "system" ? { data: systemData(entry.data) } : {}), })), agent: revisions.map((revision) => ({ seq: revision.seq, value: JSON.parse(revision.value) as unknown })), }; }); type Ledger = Effect.Success>; +/** The first request's system message: every prompt section, no tool change. */ +const prompt = { sections: ["tools", "rules", "cwd"], added: [], removed: [] }; + const systemEntries = (book: Ledger) => book.entries.filter((entry) => entry.type === "system"); const settledSeqs = (book: Ledger) => book.events.filter((event) => event.type.startsWith(`${EventList.ToolSettled.type}.`)).map((event) => event.seq); @@ -352,15 +365,17 @@ describe("toolsearch", () => { expect(first!.tools).toContain(TOOL_SEARCH_NAME); expect(first!.tools).not.toContain("weather_forecast"); expect(second!.tools).toEqual(first!.tools); - expect(first!.system).toEqual([]); - expect(second!.system).toEqual([{ added: ["weather_forecast"], removed: [] }]); - // The third request replays the one system entry; nothing new is declared. + // The first request declares the prompt sections; loading a tool leaves them as they are. + expect(first!.system).toEqual([prompt]); + expect(second!.system).toEqual([prompt, { sections: [], added: ["weather_forecast"], removed: [] }]); + // The third request replays both system entries; nothing new is declared. expect(third!.tools).toEqual(first!.tools); expect(third!.system).toEqual(second!.system); - expect(systemEntries(book)).toHaveLength(1); - expect(systemEntries(book)[0]!.data).toMatchObject({ - toolsAdded: [{ name: "weather_forecast" }], + expect(systemEntries(book)).toHaveLength(2); + expect(systemEntries(book)[1]!.data).toEqual({ + toolsAdded: [expect.objectContaining({ name: "weather_forecast" })], toolsRemoved: [], + hash: expect.any(String), }); // The loaded tool runs. expect(outcomes[1]).toMatchObject({ @@ -397,14 +412,14 @@ describe("toolsearch", () => { ]); for (const failed of outcomes.slice(0, 3)) expect(failed.details).toMatchObject({ _tag: "ToolSearchFailed" }); expect(outcomes[3]!.details).toEqual({ loaded: [] }); - // Nothing loaded: no `agent` change and no system entry. + // Nothing loaded: no `agent` change and no system entry past the first request's prompt. expect(book.agent.some((revision) => JSON.stringify(revision.value).includes("modelTools"))).toBe(false); - expect(systemEntries(book)).toEqual([]); + expect(systemEntries(book)).toHaveLength(1); expect( requestsOf(main) .map(sent) .map((one) => one.system), - ).toEqual([[], []]); + ).toEqual([[prompt], [prompt]]); yield* artifact("tools.toolsearch.refused.json", { results: outcomes, agent: book.agent }); }), ); @@ -429,8 +444,7 @@ describe("toolsearch", () => { expect(add.slice(0, 2)).toEqual(loadedOf(limited!.details)); expect(new Set(add)).toEqual(new Set([...loadedOf(limited!.details), ...loadedOf(all!.details)])); const [, next] = requestsOf(main).map(sent); - expect(next!.system).toHaveLength(1); - expect(next!.system[0]!.added).toEqual(add); + expect(next!.system).toEqual([prompt, { sections: [], added: add, removed: [] }]); yield* artifact("tools.toolsearch.limit.json", { results: [limited, all], @@ -449,11 +463,11 @@ describe("toolsearch", () => { const { sessionId, main } = yield* createSession; yield* run(sessionId, main, "Load the forecast tool", [call(search("weather forecast", 1)), reply]); const line = yield* threads.line(main); - expect(line.map(({ entry }) => entry.type)).toEqual(["user", "assistant", "system", "assistant"]); - const [prompt, searched] = line; + expect(line.map(({ entry }) => entry.type)).toEqual(["user", "system", "assistant", "system", "assistant"]); + const [asked, , searched] = line; const before = ThreadSchema.ID.make( - (yield* threads.fork({ threadId: main, entryId: prompt!.entry.id })).thread.id, + (yield* threads.fork({ threadId: main, entryId: asked!.entry.id })).thread.id, ); const after = ThreadSchema.ID.make( (yield* threads.fork({ threadId: main, entryId: searched!.entry.id })).thread.id, @@ -471,14 +485,15 @@ describe("toolsearch", () => { const [afterFirst] = requestsOf(after).map(sent); const beforeResults = yield* results(before); const afterResults = yield* results(after); - expect(beforeFirst!.system).toEqual([]); + // Cut before main's first system entry, the fork declares the prompt itself. + expect(beforeFirst!.system).toEqual([prompt]); expect(beforeResults[0]).toMatchObject({ status: "error", text: 'Tool "weather_forecast" is not available in this thread.', details: { _tag: "ToolUnavailable", name: "weather_forecast" }, }); - // The history up to the cut has no system entry; the fork declares the tool itself. - expect(afterFirst!.system).toEqual([{ added: ["weather_forecast"], removed: [] }]); + // The history up to the cut declares the prompt but not the tool; the fork declares the tool itself. + expect(afterFirst!.system).toEqual([prompt, { sections: [], added: ["weather_forecast"], removed: [] }]); expect(afterResults[0]).toMatchObject({ status: "completed", text: "weather_forecast ran" }); const afterBook = yield* ledger(sessionId, after); expect(systemEntries(afterBook)).toHaveLength(1); @@ -536,7 +551,10 @@ describe("toolsearch", () => { }); // The removal is declared to the model as a system entry. expect(systemEntries(book)).toHaveLength(1); - expect(systemEntries(book)[0]!.data).toMatchObject({ toolsRemoved: [{ name: "notes_read" }] }); + expect(systemEntries(book)[0]!.data).toMatchObject({ + sections: prompt.sections, + toolsRemoved: [{ name: "notes_read" }], + }); }), ); }); diff --git a/packages/harness/test/utils/live.ts b/packages/harness/test/utils/live.ts index 06787483..4b6c7b31 100644 --- a/packages/harness/test/utils/live.ts +++ b/packages/harness/test/utils/live.ts @@ -5,22 +5,26 @@ import { join } from "node:path"; import "./env.ts"; /** - * The Codex login's current access token, for `OPENAI_CODEX_API_KEY`. The login file itself is - * never used by a test: a refresh would rotate the refresh token and sign the developer out. An - * expired token skips the Codex cases rather than refreshing. + * A login's current access token, for its key variable. The login file itself is never used by a + * test: a refresh would rotate the refresh token and sign the developer out. An expired token + * skips the provider's cases rather than refreshing; `expires: 0` never expires (GitHub Copilot). */ -const codexToken = (): string | undefined => { +const loginToken = (provider: string): string | undefined => { try { - const login = JSON.parse(readFileSync(join(homedir(), ".codework", "aikit", "auth.json"), "utf8"))[ - "openai-codex" - ]; - return typeof login?.access === "string" && login.expires > Date.now() + 10 * 60_000 ? login.access : undefined; + const login = JSON.parse(readFileSync(join(homedir(), ".codework", "aikit", "auth.json"), "utf8"))[provider]; + const fresh = login?.expires === 0 || login?.expires > Date.now() + 10 * 60_000; + return typeof login?.access === "string" && fresh ? login.access : undefined; } catch { return undefined; } }; -const codex = process.env.OPENAI_CODEX_API_KEY ?? codexToken(); -if (codex !== undefined) process.env.OPENAI_CODEX_API_KEY = codex; +for (const [env, provider] of [ + ["OPENAI_CODEX_API_KEY", "openai-codex"], + ["COPILOT_GITHUB_TOKEN", "github-copilot"], +] as const) { + const token = process.env[env] ?? loginToken(provider); + if (token !== undefined) process.env[env] = token; +} /** Every live case runs against each of these; `env` names the key that enables it. */ export const LIVE = [