From 5b1dd2e8ee7aa8e3564845179350fb8c6b3fe2ce Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Fri, 30 Oct 2026 22:50:36 +0530 Subject: [PATCH 01/11] feat(harness): declare prompt sections as transcript system messages The foundation goes out as the system prompt; every tagged section is declared by a system entry. The first request declares them all, a later one only the sections that changed, with any tool change, so resume and fork replay what the model saw and in-place models keep their cache. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 + packages/aikit/src/message/transcript.ts | 27 ++- packages/harness/src/context/codec.ts | 4 +- packages/harness/src/event/list.ts | 17 +- packages/harness/src/plugin/prompt/render.ts | 37 ++- packages/harness/src/plugin/registry.ts | 2 +- packages/harness/src/runner/loop.ts | 22 +- packages/harness/src/session/projector.ts | 11 +- packages/harness/src/session/schema.ts | 12 +- packages/harness/src/state/state.ts | 4 +- .../test/__artifacts__/mcp.thread.json | 10 + .../test/__artifacts__/mcp.thread.loaded.json | 33 +++ .../__artifacts__/mcp.thread.resumed.json | 10 + .../mcp.toolsearch.connecting.json | 10 + .../test/__artifacts__/prompt.changes.json | 220 ++++++++++++++++++ .../prompt.changes.live.openai-codex.json | 8 + .../test/__artifacts__/runner.retry.json | 9 + .../test/__artifacts__/thread.concurrent.json | 2 + .../test/__artifacts__/thread.fork.json | 58 ++--- .../test/__artifacts__/thread.recovery.json | 38 +-- .../test/__artifacts__/thread.reset.json | 33 +-- .../__artifacts__/thread.walkthrough.json | 73 +++--- .../test/__artifacts__/tools.read.vision.json | 9 + .../__artifacts__/tools.toolsearch.fork.json | 30 ++- .../__artifacts__/tools.toolsearch.limit.json | 10 + .../tools.toolsearch.live.openai-codex.json | 6 +- .../__artifacts__/tools.toolsearch.load.json | 88 +++++-- .../tools.toolsearch.unavailable.json | 10 + .../harness/test/fixtures/instruction.spec.ts | 3 +- packages/harness/test/fixtures/llm.ts | 7 + packages/harness/test/fixtures/mcp/turns.ts | 5 +- packages/harness/test/fixtures/skill.spec.ts | 3 +- packages/harness/test/mcp.thread.test.ts | 6 +- packages/harness/test/plugin.examples.test.ts | 14 +- packages/harness/test/plugin.external.test.ts | 18 +- packages/harness/test/plugin.git.live.test.ts | 8 +- packages/harness/test/plugin.host.test.ts | 17 +- .../harness/test/plugin.instruction.test.ts | 3 +- packages/harness/test/plugin.passes.test.ts | 13 +- packages/harness/test/plugin.prompt.test.ts | 3 +- packages/harness/test/plugin.skill.test.ts | 5 +- .../harness/test/prompt.changes.live.test.ts | 95 ++++++++ packages/harness/test/prompt.changes.test.ts | 118 ++++++++++ packages/harness/test/runner.loop.test.ts | 95 +++++--- packages/harness/test/sdk.test.ts | 2 + packages/harness/test/settings.loop.test.ts | 6 +- packages/harness/test/thread.test.ts | 26 ++- .../harness/test/tools.toolsearch.test.ts | 60 +++-- 48 files changed, 1042 insertions(+), 261 deletions(-) create mode 100644 packages/harness/test/__artifacts__/prompt.changes.json create mode 100644 packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json create mode 100644 packages/harness/test/prompt.changes.live.test.ts create mode 100644 packages/harness/test/prompt.changes.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d892f4d..d718ea23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ This file is the canonical source for unreleased changes and published release n - MCP OAuth: servers sign in with discovery, client registration and PKCE, tokens stored in `~/.codework/mcpauth.json` by server name and URL and refreshed under a lock shared across processes. A server waiting for sign-in reconnects on the next message once its sign-in changes. `auth: { provider }` sends a model provider's credential instead. - The ephemeral `mcp.server.status` event reports each MCP server state change (`connecting`, `connected`, `disconnected`, `failed`, `authenticate`, `disabled`) to every thread using the server. A tool the MCP SDK rejects is skipped instead of failing its server, and listed as `skipped` on the `connected` status. - `toolsearch` turns itself on when a thread has deferred tools or an enabled MCP server with `deferred` exposure; a settings entry still turns it off or on. A search waits up to 10 seconds for such servers still connecting. +- Prompt sections reach the model as transcript system messages. A request sends the foundation as its system prompt; the first request of a thread appends a `system` entry declaring every tagged section (``, ``, ``, plugin sections), and a later request appends one only for the sections that changed, a removed section as `null`, together with any tool change. Resume and fork replay these entries, so the model sees what it saw before, and a model that takes system messages in place (OpenAI Codex, GitHub Copilot Responses) keeps its cached prefix when a section changes. #### Breaking Changes @@ -50,6 +51,7 @@ This file is the canonical source for unreleased changes and published release n - Migrations add `session_input.cancelled_seq`, and `session.execution` and `session.restarts`. - Stored context messages are user and assistant messages only; a stored system message fails to decode with "system messages are not stored as entries". - Tests use `gpt-6-luna` in place of the deprecated `gpt-5.5`. +- `session.thread.tools.changed` (`ToolsChanged`) is now `session.thread.system.changed` (`SystemChanged`), and its `system` entry data has an optional `sections` patch. `State.Snapshot.systemPrompt` is now `prompt: { foundation, sections }`. ### @codeworksh/cli @@ -91,6 +93,7 @@ This file is the canonical source for unreleased changes and published release n - `Transcript` helpers: `getCurrentTools`, `getDeclaredTools`, `getToolStateChanges`, `declarationsEqual`, `toToolDeclaration` and `resolveTranscriptTools`. - Transcript system messages also change the system prompt: `content` adds instructions from that point on, and `sections` replace named sections, `null` removing one. `Context.systemPrompt` is the initial prompt. OpenAI Codex and GitHub Copilot Responses models with `supportsMidConvoSystemMessages` keep the leading prompt and get each change in place as a developer message (`Updated system prompt section "": …`), so the cached prompt prefix holds; every other model gets the current prompt as its system prompt. Helpers: `getCurrentSystemPrompt`, `renderSystemPrompt`, `renderSystemPromptUpdate` and `resolveTranscriptPrompt`. - `supportsMidConvoSystemMessages` compat flag, set by `modelgen` for the OpenAI Codex and GitHub Copilot Responses models that accept developer messages mid-conversation. In-place tool declarations (`supportsAdditionalTools`) require it. +- `Transcript.getCurrentSections`: the sections a transcript declares after replaying its system messages. - `filterGitHubCopilotModels` keeps the catalog models a Copilot login can reach. - GitHub Copilot GPT models turn reasoning off with `reasoning.effort: "none"` where OpenAI accepts it, such as gpt-5.3-codex, gpt-5.6-luna and gpt-6-luna. diff --git a/packages/aikit/src/message/transcript.ts b/packages/aikit/src/message/transcript.ts index 0902408d..84b71d58 100644 --- a/packages/aikit/src/message/transcript.ts +++ b/packages/aikit/src/message/transcript.ts @@ -45,24 +45,29 @@ export function renderSystemPromptUpdate(message: PromptChange): string { } /** - * The prompt after every transcript change in order: content accumulates, sections are - * replaced by name, and a removed section added again goes last. + * The sections after every transcript change in order: replaced by name, removed by `null`, and + * a removed section added again goes last. */ -export function getCurrentSystemPrompt(context: Pick): string { - const content: string[] = []; +export function getCurrentSections(messages: readonly Message[]): Record { const sections = new Map(); - const changes: PromptChange[] = [ - { content: context.systemPrompt ?? "" }, - ...context.messages.filter(isSystemMessage), - ]; - for (const message of changes) { - if (message.content) content.push(message.content); + for (const message of messages) { + if (!isSystemMessage(message)) continue; for (const [name, value] of Object.entries(message.sections ?? {})) { if (value === null) sections.delete(name); else sections.set(name, value); } } - return renderSystemPrompt({ content: content.join("\n\n"), sections: Object.fromEntries(sections) }); + return Object.fromEntries(sections); +} + +/** The prompt after every transcript change in order: content accumulates, sections as {@link getCurrentSections}. */ +export function getCurrentSystemPrompt(context: Pick): string { + const content: string[] = [context.systemPrompt ?? ""]; + for (const message of context.messages) if (isSystemMessage(message)) content.push(message.content ?? ""); + return renderSystemPrompt({ + content: content.filter((part) => part.length > 0).join("\n\n"), + sections: getCurrentSections(context.messages), + }); } export interface TranscriptPrompt { diff --git a/packages/harness/src/context/codec.ts b/packages/harness/src/context/codec.ts index 48ca0a52..c9251fcc 100644 --- a/packages/harness/src/context/codec.ts +++ b/packages/harness/src/context/codec.ts @@ -199,11 +199,12 @@ export const decodeSyntheticMessage = Effect.fn("Context.decodeSyntheticMessage" ); }); -/** The transcript system message of a tool change; empty sides are left out. */ +/** The transcript system message of a section and tool change; empty parts are left out. */ export const systemMessage = ( messageId: string, created: number, changes: { + readonly sections?: Readonly> | undefined; readonly toolsAdded: ReadonlyArray; readonly toolsRemoved: ReadonlyArray; }, @@ -211,6 +212,7 @@ export const systemMessage = ( messageId, role: "system" as const, time: { created }, + ...(changes.sections === undefined ? {} : { sections: { ...changes.sections } }), ...(changes.toolsAdded.length === 0 ? {} : { toolsAdded: [...changes.toolsAdded] }), ...(changes.toolsRemoved.length === 0 ? {} : { toolsRemoved: changes.toolsRemoved.map(({ name }) => ({ name })) }), }); diff --git a/packages/harness/src/event/list.ts b/packages/harness/src/event/list.ts index 1be3a6cc..cd893099 100644 --- a/packages/harness/src/event/list.ts +++ b/packages/harness/src/event/list.ts @@ -113,15 +113,16 @@ export const ThreadConfigured = EventSchema.define({ export type ThreadConfigured = typeof ThreadConfigured.Type; /** - * The tools offered to a thread's next request differ from those its transcript declares, such as - * after `toolsearch` loaded some. Appends a `system` entry carrying the change. + * The prompt sections or tools for a thread's next request differ from those its transcript + * declares: the first request, a changed section, or `toolsearch` loading a tool. Appends a + * `system` entry carrying the change. */ -export const ToolsChanged = EventSchema.define({ - type: "session.thread.tools.changed", +export const SystemChanged = EventSchema.define({ + type: "session.thread.system.changed", ...durableOptions, - schema: { ...baseOptions, messageId: SessionMessageSchema.ID, ...SessionSchema.ToolChanges.fields }, + schema: { ...baseOptions, messageId: SessionMessageSchema.ID, ...SessionSchema.SystemChange.fields }, }); -export type ToolsChanged = typeof ToolsChanged.Type; +export type SystemChanged = typeof SystemChanged.Type; const LLMFields = { ...baseOptions, @@ -397,7 +398,7 @@ export const Definitions = EventSchema.inventory( SessionCreated, ThreadForked, ThreadConfigured, - ToolsChanged, + SystemChanged, TurnStarted, TurnEnded, TurnAborted, @@ -434,7 +435,7 @@ export const DurableDefinitions = EventSchema.inventory( SessionCreated, ThreadForked, ThreadConfigured, - ToolsChanged, + SystemChanged, TurnEnded, LLMStarted, LLMEnded, diff --git a/packages/harness/src/plugin/prompt/render.ts b/packages/harness/src/plugin/prompt/render.ts index dd4dea20..38957446 100644 --- a/packages/harness/src/plugin/prompt/render.ts +++ b/packages/harness/src/plugin/prompt/render.ts @@ -35,7 +35,17 @@ const toolIndex = (tools: ReadonlyArray): string => { return listed.map((tool) => `- ${tool.name}: ${tool.promptSnippet}`).join("\n"); }; -export const render = (input: Input): string => { +/** + * The prompt as the transcript carries it: the untagged foundation, then named sections, each + * holding its own tags. A section's text is compared whole between requests, so one that did not + * change is never sent again. + */ +export interface Rendered { + readonly foundation: string; + readonly sections: Readonly>; +} + +export const render = (input: Input): Rendered => { const written = new Map(input.prompt.sections); const guidelines = input.tools.flatMap((tool) => tool.promptGuidelines ?? []); @@ -44,17 +54,34 @@ export const render = (input: Input): string => { entries: [...guidelines, ...(written.get(Section.Rules.name)?.entries ?? [])], }); - const parts: string[] = [input.prompt.foundation ?? fallback, tag("tools", toolIndex(input.tools))]; + const sections: Record = { tools: tag("tools", toolIndex(input.tools)) }; const push = (name: string) => { const section = written.get(name); const rendered = section === undefined ? undefined : body(section); - if (rendered !== undefined) parts.push(tag(name, rendered)); + if (rendered !== undefined) sections[name] = tag(name, rendered); written.delete(name); }; for (const section of Section.builtins) push(section.name); - parts.push(tag("cwd", input.directory)); + sections.cwd = tag("cwd", input.directory); for (const name of Array.from(written.keys())) push(name); - return parts.join("\n\n"); + // A blank foundation would leave models that take system messages in place with no leading + // prompt, and their transport would promote a later section update in its stead. + const foundation = input.prompt.foundation?.trim() ? input.prompt.foundation : fallback; + return { foundation, sections }; +}; + +/** + * The patch that brings the sections the transcript declares to `current`, `null` removing one; + * `undefined` when they match. + */ +export const diff = ( + previous: Readonly>, + current: Readonly>, +): Record | undefined => { + const patch: Record = {}; + for (const [name, text] of Object.entries(current)) if (previous[name] !== text) patch[name] = text; + for (const name of Object.keys(previous)) if (!Object.hasOwn(current, name)) patch[name] = null; + return Object.keys(patch).length === 0 ? undefined : patch; }; diff --git a/packages/harness/src/plugin/registry.ts b/packages/harness/src/plugin/registry.ts index ae9b898c..0fceeb02 100644 --- a/packages/harness/src/plugin/registry.ts +++ b/packages/harness/src/plugin/registry.ts @@ -34,7 +34,7 @@ export const make = (servers: ReadonlyArray) => { return { tools: resolved, // Only declared tools: loading a deferred one must not change the prompt. - systemPrompt: render({ prompt: prompt.snapshot(), tools: resolved.defs.filter(declared), directory }), + prompt: render({ prompt: prompt.snapshot(), tools: resolved.defs.filter(declared), directory }), }; }, }; diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index 5f3fe98f..f26d2ac2 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -8,6 +8,7 @@ import { type Message, Transcript } from "@codeworksh/aikit"; import { Cause, DateTime, Duration, Effect, Exit, Layer, Option, Schema } from "effect"; import { ContextCodec } from "../context/codec.ts"; +import { diff } from "../plugin/prompt/render.ts"; import { Context } from "../context/context.ts"; import { Event } from "../event/event.ts"; import { EventList } from "../event/list.ts"; @@ -275,11 +276,12 @@ export const layer = (options: Options = {}) => }); /** - * The request keeps the declared tools as `tools`; what the thread offers beyond them (or - * withholds) is declared by a `system` entry, published when the transcript's tool state - * differs from the offered one. Returns its message, for the request at hand. + * The request keeps the foundation as `systemPrompt` and the declared tools as `tools`. The + * prompt sections, and what the thread offers beyond those tools (or withholds), are declared + * by a `system` entry, published when the transcript's state differs from the snapshot's: the + * first request declares every section. Returns its message, for the request at hand. */ - const declareToolChanges = Effect.fn("Loop.declareToolChanges")(function* ( + const declareSystemChanges = Effect.fn("Loop.declareSystemChanges")(function* ( snapshot: State.Snapshot, messages: ReadonlyArray, ) { @@ -287,12 +289,14 @@ export const layer = (options: Options = {}) => { messages: [...messages], tools: [...snapshot.tools.wire] }, "none", ).requestTools; - const changes = Transcript.getToolStateChanges(previous, snapshot.tools.offered); - if (changes.toolsAdded.length === 0 && changes.toolsRemoved.length === 0) return []; + const tools = Transcript.getToolStateChanges(previous, snapshot.tools.offered); + const sections = diff(Transcript.getCurrentSections(messages), snapshot.prompt.sections); + if (sections === undefined && tools.toolsAdded.length === 0 && tools.toolsRemoved.length === 0) return []; + const changes = { ...(sections === undefined ? {} : { sections }), ...tools }; const messageId = SessionMessageSchema.ID.create(); const timestamp = yield* DateTime.now; const hash = Hash.sha256(yield* encodeJson(snapshot.tools.offered.map(Transcript.toToolDeclaration))); - yield* events.publish(EventList.ToolsChanged, { + yield* events.publish(EventList.SystemChanged, { timestamp, sessionId: snapshot.sessionId, threadId: snapshot.threadId, @@ -326,7 +330,7 @@ export const layer = (options: Options = {}) => if (leaf !== undefined && (leaf.stopReason === "stop" || leaf.stopReason === "length")) { return undefined; } - const messages = [...assembled.messages, ...(yield* declareToolChanges(snapshot, assembled.messages))]; + const messages = [...assembled.messages, ...(yield* declareSystemChanges(snapshot, assembled.messages))]; // The latest attempt's publisher; a retried turn has one per provider request. let publisher: LLMEventPublisher.Publisher | undefined; @@ -351,7 +355,7 @@ export const layer = (options: Options = {}) => const prepared = yield* hooks.prepareRequest( { context: { - systemPrompt: snapshot.systemPrompt, + systemPrompt: snapshot.prompt.foundation, messages: [...messages], tools: [...snapshot.tools.wire], }, diff --git a/packages/harness/src/session/projector.ts b/packages/harness/src/session/projector.ts index bbdb9f5a..94e9e760 100644 --- a/packages/harness/src/session/projector.ts +++ b/packages/harness/src/session/projector.ts @@ -198,16 +198,21 @@ export const layer = Layer.effectDiscard( }), ); - yield* events.project(EventList.ToolsChanged, (event) => + yield* events.project(EventList.SystemChanged, (event) => Effect.gen(function* () { - const { toolsAdded, toolsRemoved, hash } = event.data; + const { sections, toolsAdded, toolsRemoved, hash } = event.data; yield* threads .append({ id: event.data.messageId, threadId: event.data.threadId, seq: yield* seqOf(event), type: "system", - data: yield* encodeSystemData({ toolsAdded, toolsRemoved, hash }).pipe(Effect.orDie), + data: yield* encodeSystemData({ + ...(sections === undefined ? {} : { sections }), + toolsAdded, + toolsRemoved, + hash, + }).pipe(Effect.orDie), }) .pipe(Effect.orDie); }), diff --git a/packages/harness/src/session/schema.ts b/packages/harness/src/session/schema.ts index 5995e9b3..b30955f2 100644 --- a/packages/harness/src/session/schema.ts +++ b/packages/harness/src/session/schema.ts @@ -44,16 +44,20 @@ export const ToolDeclaration = Schema.Struct({ constrainedSampling: optional(Schema.Unknown), }); -/** A change to the tools offered from one point of the transcript on, as a `system` entry stores it. */ -export const ToolChanges = Schema.Struct({ +/** + * A change to the prompt sections and tools offered from one point of the transcript on, as a + * `system` entry stores it. A `null` section is removed. + */ +export const SystemChange = Schema.Struct({ + sections: optional(Schema.Record(Schema.String, Schema.NullOr(Schema.String))), toolsAdded: Schema.Array(ToolDeclaration), toolsRemoved: Schema.Array(Schema.Struct({ name: Schema.String })), /** sha256 of the offered tool declarations after the change. */ hash: Schema.String, }); -export type ToolChanges = typeof ToolChanges.Type; +export type SystemChange = typeof SystemChange.Type; -export const SystemData = Schema.fromJsonString(ToolChanges); +export const SystemData = Schema.fromJsonString(SystemChange); // Harness-owned shape of the billed usage inside a persisted assistant envelope. // This is data-structure integrity only — business validation diff --git a/packages/harness/src/state/state.ts b/packages/harness/src/state/state.ts index 7ad622e1..07890514 100644 --- a/packages/harness/src/state/state.ts +++ b/packages/harness/src/state/state.ts @@ -37,6 +37,7 @@ import { merge } from "../settings/merge.ts"; import { compose, resolveOptions } from "../settings/resolve.ts"; import type { Block, Info, RetryPolicy } from "../settings/schema.ts"; import { Settings } from "../settings/settings.ts"; +import type { Rendered } from "../plugin/prompt/render.ts"; import type { Resolved } from "../tool/registry.ts"; /** @@ -116,7 +117,8 @@ export interface Snapshot { readonly threadId: ThreadSchema.ID; readonly sandbox: SandboxIO.Identity; readonly location: Location.Info; - readonly systemPrompt: string; + /** The foundation goes out as `systemPrompt`; the sections reach the transcript as system messages. */ + readonly prompt: Rendered; readonly tools: Resolved; readonly provider: string; readonly model: string; diff --git a/packages/harness/test/__artifacts__/mcp.thread.json b/packages/harness/test/__artifacts__/mcp.thread.json index 73c797f6..79f41a6b 100644 --- a/packages/harness/test/__artifacts__/mcp.thread.json +++ b/packages/harness/test/__artifacts__/mcp.thread.json @@ -489,6 +489,16 @@ }, "changes": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "mcp__e2e__args" ], diff --git a/packages/harness/test/__artifacts__/mcp.thread.loaded.json b/packages/harness/test/__artifacts__/mcp.thread.loaded.json index 1e118ef6..000cc486 100644 --- a/packages/harness/test/__artifacts__/mcp.thread.loaded.json +++ b/packages/harness/test/__artifacts__/mcp.thread.loaded.json @@ -2,6 +2,16 @@ "notAwaited": "connecting", "awaitedChanges": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "mcp__slow__echo", "mcp__slow__big" @@ -11,6 +21,16 @@ ], "whileAway": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "mcp__slow__echo", "mcp__slow__big" @@ -18,6 +38,7 @@ "removed": [] }, { + "sections": [], "added": [], "removed": [ "mcp__slow__echo", @@ -33,6 +54,16 @@ }, "back": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "mcp__slow__echo", "mcp__slow__big" @@ -40,6 +71,7 @@ "removed": [] }, { + "sections": [], "added": [], "removed": [ "mcp__slow__echo", @@ -47,6 +79,7 @@ ] }, { + "sections": [], "added": [ "mcp__slow__echo", "mcp__slow__big" diff --git a/packages/harness/test/__artifacts__/mcp.thread.resumed.json b/packages/harness/test/__artifacts__/mcp.thread.resumed.json index ab578040..6f1e1b20 100644 --- a/packages/harness/test/__artifacts__/mcp.thread.resumed.json +++ b/packages/harness/test/__artifacts__/mcp.thread.resumed.json @@ -14,6 +14,16 @@ "second": { "changes": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "mcp__docs__echo", "mcp__docs__big" diff --git a/packages/harness/test/__artifacts__/mcp.toolsearch.connecting.json b/packages/harness/test/__artifacts__/mcp.toolsearch.connecting.json index 027335de..0ff8ab5b 100644 --- a/packages/harness/test/__artifacts__/mcp.toolsearch.connecting.json +++ b/packages/harness/test/__artifacts__/mcp.toolsearch.connecting.json @@ -9,6 +9,16 @@ ], "secondChanges": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "mcp__slow__echo", "mcp__slow__big" diff --git a/packages/harness/test/__artifacts__/prompt.changes.json b/packages/harness/test/__artifacts__/prompt.changes.json new file mode 100644 index 00000000..a8a65896 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.json @@ -0,0 +1,220 @@ +{ + "requests": [ + { + "roles": [ + "user", + "system" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + } + ], + "mode": "Plan only." + }, + { + "roles": [ + "user", + "system", + "assistant", + "user", + "system" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + }, + { + "mode": "\nBuild allowed.\n" + } + ], + "mode": "Build allowed." + }, + { + "roles": [ + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "system" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + }, + { + "mode": "\nBuild allowed.\n" + }, + { + "mode": null + } + ], + "mode": null + }, + { + "roles": [ + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + }, + { + "mode": "\nBuild allowed.\n" + }, + { + "mode": null + } + ], + "mode": null + }, + { + "roles": [ + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "assistant", + "user" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + }, + { + "mode": "\nBuild allowed.\n" + }, + { + "mode": null + } + ], + "mode": null + }, + { + "roles": [ + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "assistant", + "user", + "assistant", + "user", + "system" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + }, + { + "mode": "\nBuild allowed.\n" + }, + { + "mode": null + }, + { + "mode": "\nPlan again.\n" + } + ], + "mode": "Plan again." + }, + { + "roles": [ + "user", + "system", + "assistant", + "user", + "system" + ], + "declared": [ + { + "tools": "…", + "rules": "…", + "cwd": "…", + "mode": "\nPlan only.\n" + }, + { + "mode": null + } + ], + "mode": null + } + ], + "prefixKept": [ + true, + true, + true, + true, + true + ], + "threads": { + "main": [ + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "system", + "assistant", + "user", + "assistant", + "user", + "assistant", + "user", + "system", + "assistant" + ], + "fork": [ + "user", + "system", + "assistant", + "user", + "system", + "assistant" + ] + } +} \ No newline at end of file diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json b/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json new file mode 100644 index 00000000..9fb6e344 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json @@ -0,0 +1,8 @@ +{ + "answers": [ + "alpha", + "bravo" + ], + "promptChange": "in place", + "earlierSectionSent": true +} diff --git a/packages/harness/test/__artifacts__/runner.retry.json b/packages/harness/test/__artifacts__/runner.retry.json index 22367eea..56dbd33f 100644 --- a/packages/harness/test/__artifacts__/runner.retry.json +++ b/packages/harness/test/__artifacts__/runner.retry.json @@ -3,6 +3,9 @@ { "type": "session.created" }, + { + "type": "session.thread.system.changed" + }, { "type": "session.turn.started" }, @@ -68,6 +71,12 @@ "state": "committed", "parts": 1 }, + { + "id": "system", + "type": "system", + "state": "committed", + "parts": 0 + }, { "id": "assistant_1", "type": "assistant", diff --git a/packages/harness/test/__artifacts__/thread.concurrent.json b/packages/harness/test/__artifacts__/thread.concurrent.json index c4e64d5f..54a60090 100644 --- a/packages/harness/test/__artifacts__/thread.concurrent.json +++ b/packages/harness/test/__artifacts__/thread.concurrent.json @@ -2,12 +2,14 @@ "bothActive": true, "main": [ "user committed", + "system committed", "assistant committed", "user committed", "assistant committed" ], "fork": [ "user committed", + "system committed", "assistant committed" ] } diff --git a/packages/harness/test/__artifacts__/thread.fork.json b/packages/harness/test/__artifacts__/thread.fork.json index 54ebb14d..07735f90 100644 --- a/packages/harness/test/__artifacts__/thread.fork.json +++ b/packages/harness/test/__artifacts__/thread.fork.json @@ -10,21 +10,22 @@ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", "3 session.prompt.promoted.1 T1", - "4 session.llm.started.1 T1", - "5 session.llm.ended.1 T1", - "6 session.turn.ended.1 T1", - "7 session.prompt.admitted.1 T1", - "8 session.prompt.promoted.1 T1", - "9 session.llm.started.1 T1", - "10 session.llm.ended.1 T1", - "11 session.turn.ended.1 T1", - "12 session.thread.forked.1 T2", - "13 session.thread.forked.1 T3", - "14 session.thread.forked.1 T4", - "15 session.prompt.admitted.1 T1", - "16 session.prompt.promoted.1 T1", - "17 session.llm.started.1 T1", - "18 session.llm.failed.1 T1" + "4 session.thread.system.changed.1 T1", + "5 session.llm.started.1 T1", + "6 session.llm.ended.1 T1", + "7 session.turn.ended.1 T1", + "8 session.prompt.admitted.1 T1", + "9 session.prompt.promoted.1 T1", + "10 session.llm.started.1 T1", + "11 session.llm.ended.1 T1", + "12 session.turn.ended.1 T1", + "13 session.thread.forked.1 T2", + "14 session.thread.forked.1 T3", + "15 session.thread.forked.1 T4", + "16 session.prompt.admitted.1 T1", + "17 session.prompt.promoted.1 T1", + "18 session.llm.started.1 T1", + "19 session.llm.failed.1 T1" ], "threads": [ { @@ -35,30 +36,31 @@ }, { "thread": "T2", - "origin": "fork T1 at E4", - "createdSeq": 12, + "origin": "fork T1 at E5", + "createdSeq": 13, "title": null }, { "thread": "T3", "origin": "fork T1 at E1", - "createdSeq": 13, + "createdSeq": 14, "title": null }, { "thread": "T4", - "origin": "fork T1 at E2", - "createdSeq": 14, + "origin": "fork T1 at E3", + "createdSeq": 15, "title": null } ], "entries": [ "E1 T1 user seq=3 committed=3 committed", - "E2 T1 assistant seq=4 committed=6 committed", - "E3 T1 user seq=8 committed=8 committed", - "E4 T1 assistant seq=9 committed=11 committed", - "E5 T1 user seq=16 committed=16 committed", - "E6 T1 assistant seq=17 committed=18 aborted" + "E2 T1 system seq=4 committed=4 committed", + "E3 T1 assistant seq=5 committed=7 committed", + "E4 T1 user seq=9 committed=9 committed", + "E5 T1 assistant seq=10 committed=12 committed", + "E6 T1 user seq=17 committed=17 committed", + "E7 T1 assistant seq=18 committed=19 aborted" ], "documents": [ { @@ -72,21 +74,21 @@ "thread": "T2", "kind": "agent", "revisions": [ - "12 {}" + "13 {}" ] }, { "thread": "T3", "kind": "agent", "revisions": [ - "13 {}" + "14 {}" ] }, { "thread": "T4", "kind": "agent", "revisions": [ - "14 {}" + "15 {}" ] } ] diff --git a/packages/harness/test/__artifacts__/thread.recovery.json b/packages/harness/test/__artifacts__/thread.recovery.json index 4faffb68..f5bdc287 100644 --- a/packages/harness/test/__artifacts__/thread.recovery.json +++ b/packages/harness/test/__artifacts__/thread.recovery.json @@ -3,17 +3,19 @@ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", "3 session.prompt.promoted.1 T1", - "4 session.llm.started.1 T1", - "5 session.llm.ended.1 T1", - "6 session.turn.ended.1 T1", - "7 session.thread.forked.1 T2", - "8 session.prompt.admitted.1 T2", - "9 session.prompt.promoted.1 T2", - "10 session.llm.started.1 T2", - "11 session.llm.failed.1 T2", - "12 session.llm.started.1 T2", - "13 session.llm.ended.1 T2", - "14 session.turn.ended.1 T2" + "4 session.thread.system.changed.1 T1", + "5 session.llm.started.1 T1", + "6 session.llm.ended.1 T1", + "7 session.turn.ended.1 T1", + "8 session.thread.forked.1 T2", + "9 session.prompt.admitted.1 T2", + "10 session.prompt.promoted.1 T2", + "11 session.llm.started.1 T2", + "12 session.llm.failed.1 T2", + "13 session.thread.system.changed.1 T2", + "14 session.llm.started.1 T2", + "15 session.llm.ended.1 T2", + "16 session.turn.ended.1 T2" ], "threads": [ { @@ -25,16 +27,18 @@ { "thread": "T2", "origin": "fork T1 at E1", - "createdSeq": 7, + "createdSeq": 8, "title": "cut off" } ], "entries": [ "E1 T1 user seq=3 committed=3 committed", - "E2 T1 assistant seq=4 committed=6 committed", - "E3 T2 user seq=9 committed=9 committed", - "E4 T2 assistant seq=10 committed=11 aborted", - "E5 T2 assistant seq=12 committed=14 committed" + "E2 T1 system seq=4 committed=4 committed", + "E3 T1 assistant seq=5 committed=7 committed", + "E4 T2 user seq=10 committed=10 committed", + "E5 T2 assistant seq=11 committed=12 aborted", + "E6 T2 system seq=13 committed=13 committed", + "E7 T2 assistant seq=14 committed=16 committed" ], "documents": [ { @@ -48,7 +52,7 @@ "thread": "T2", "kind": "agent", "revisions": [ - "7 {}" + "8 {}" ] } ] diff --git a/packages/harness/test/__artifacts__/thread.reset.json b/packages/harness/test/__artifacts__/thread.reset.json index 5b5a328e..f26abe59 100644 --- a/packages/harness/test/__artifacts__/thread.reset.json +++ b/packages/harness/test/__artifacts__/thread.reset.json @@ -2,22 +2,25 @@ "context": { "roles": [ "user", - "user" + "user", + "system" ], - "texts": "The conversation was started over. Carried over from before:\n\n\nkeep the login work\n | after the reset" + "texts": "The conversation was started over. Carried over from before:\n\n\nkeep the login work\n | after the reset | " }, "events": [ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", "3 session.prompt.promoted.1 T1", - "4 session.llm.started.1 T1", - "5 session.llm.ended.1 T1", - "6 session.turn.ended.1 T1", - "8 session.prompt.admitted.1 T1", - "9 session.prompt.promoted.1 T1", - "10 session.llm.started.1 T1", - "11 session.llm.ended.1 T1", - "12 session.turn.ended.1 T1" + "4 session.thread.system.changed.1 T1", + "5 session.llm.started.1 T1", + "6 session.llm.ended.1 T1", + "7 session.turn.ended.1 T1", + "9 session.prompt.admitted.1 T1", + "10 session.prompt.promoted.1 T1", + "11 session.thread.system.changed.1 T1", + "12 session.llm.started.1 T1", + "13 session.llm.ended.1 T1", + "14 session.turn.ended.1 T1" ], "threads": [ { @@ -29,10 +32,12 @@ ], "entries": [ "E1 T1 user seq=3 committed=3 committed", - "E2 T1 assistant seq=4 committed=6 committed", - "E3 T1 reset seq=7 committed=7 committed", - "E4 T1 user seq=9 committed=9 committed", - "E5 T1 assistant seq=10 committed=12 committed" + "E2 T1 system seq=4 committed=4 committed", + "E3 T1 assistant seq=5 committed=7 committed", + "E4 T1 reset seq=8 committed=8 committed", + "E5 T1 user seq=10 committed=10 committed", + "E6 T1 system seq=11 committed=11 committed", + "E7 T1 assistant seq=12 committed=14 committed" ], "documents": [ { diff --git a/packages/harness/test/__artifacts__/thread.walkthrough.json b/packages/harness/test/__artifacts__/thread.walkthrough.json index 6faf4ee8..fc5be405 100644 --- a/packages/harness/test/__artifacts__/thread.walkthrough.json +++ b/packages/harness/test/__artifacts__/thread.walkthrough.json @@ -4,27 +4,28 @@ "2 session.thread.configured.1 T1", "3 session.prompt.admitted.1 T1", "4 session.prompt.promoted.1 T1", - "5 session.llm.started.1 T1", - "6 session.llm.ended.1 T1", - "7 session.thread.configured.1 T1", - "8 session.tool.settled.1 T1", - "9 session.turn.ended.1 T1", - "10 session.llm.started.1 T1", - "11 session.llm.ended.1 T1", - "12 session.turn.ended.1 T1", - "13 session.prompt.admitted.1 T1", - "14 session.prompt.promoted.1 T1", - "15 session.llm.started.1 T1", - "16 session.llm.ended.1 T1", - "17 session.turn.ended.1 T1", - "18 session.thread.forked.1 T2", - "19 session.prompt.admitted.1 T2", - "20 session.prompt.promoted.1 T2", - "21 session.llm.started.1 T2", - "22 session.llm.ended.1 T2", - "23 session.turn.ended.1 T2", - "24 session.thread.configured.1 T2", - "25 session.thread.forked.1 T3" + "5 session.thread.system.changed.1 T1", + "6 session.llm.started.1 T1", + "7 session.llm.ended.1 T1", + "8 session.thread.configured.1 T1", + "9 session.tool.settled.1 T1", + "10 session.turn.ended.1 T1", + "11 session.llm.started.1 T1", + "12 session.llm.ended.1 T1", + "13 session.turn.ended.1 T1", + "14 session.prompt.admitted.1 T1", + "15 session.prompt.promoted.1 T1", + "16 session.llm.started.1 T1", + "17 session.llm.ended.1 T1", + "18 session.turn.ended.1 T1", + "19 session.thread.forked.1 T2", + "20 session.prompt.admitted.1 T2", + "21 session.prompt.promoted.1 T2", + "22 session.llm.started.1 T2", + "23 session.llm.ended.1 T2", + "24 session.turn.ended.1 T2", + "25 session.thread.configured.1 T2", + "26 session.thread.forked.1 T3" ], "threads": [ { @@ -35,25 +36,26 @@ }, { "thread": "T2", - "origin": "fork T1 at E3", - "createdSeq": 18, + "origin": "fork T1 at E4", + "createdSeq": 19, "title": "Use OAuth instead" }, { "thread": "T3", - "origin": "fork T1 at E2", - "createdSeq": 25, + "origin": "fork T1 at E3", + "createdSeq": 26, "title": null } ], "entries": [ "E1 T1 user seq=4 committed=4 committed", - "E2 T1 assistant seq=5 committed=9 committed", - "E3 T1 assistant seq=10 committed=12 committed", - "E4 T1 user seq=14 committed=14 committed", - "E5 T1 assistant seq=15 committed=17 committed", - "E6 T2 user seq=20 committed=20 committed", - "E7 T2 assistant seq=21 committed=23 committed" + "E2 T1 system seq=5 committed=5 committed", + "E3 T1 assistant seq=6 committed=10 committed", + "E4 T1 assistant seq=11 committed=13 committed", + "E5 T1 user seq=15 committed=15 committed", + "E6 T1 assistant seq=16 committed=18 committed", + "E7 T2 user seq=21 committed=21 committed", + "E8 T2 assistant seq=22 committed=24 committed" ], "documents": [ { @@ -62,27 +64,28 @@ "revisions": [ "1 {}", "2 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"}}", - "7 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" + "8 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" ] }, { "thread": "T2", "kind": "agent", "revisions": [ - "18 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}", - "24 {\"model\":{\"provider\":\"anthropic\",\"id\":\"claude-sonnet-4-5\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" + "19 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}", + "25 {\"model\":{\"provider\":\"anthropic\",\"id\":\"claude-sonnet-4-5\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" ] }, { "thread": "T3", "kind": "agent", "revisions": [ - "25 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" + "26 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" ] } ], "t2Context": [ "user", + "system", "assistant", "assistant", "user" diff --git a/packages/harness/test/__artifacts__/tools.read.vision.json b/packages/harness/test/__artifacts__/tools.read.vision.json index 36094334..f62e6c80 100644 --- a/packages/harness/test/__artifacts__/tools.read.vision.json +++ b/packages/harness/test/__artifacts__/tools.read.vision.json @@ -7,6 +7,9 @@ "content": [ "read the image" ] + }, + { + "role": "system" } ] }, @@ -19,6 +22,9 @@ "read the image" ] }, + { + "role": "system" + }, { "role": "assistant", "parts": [ @@ -48,6 +54,9 @@ "read the image" ] }, + { + "role": "system" + }, { "role": "assistant", "parts": [ diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.fork.json b/packages/harness/test/__artifacts__/tools.toolsearch.fork.json index 2aa7e65f..c585146b 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.fork.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.fork.json @@ -10,7 +10,17 @@ "toolsearch", "notes_read" ], - "system": [] + "system": [ + { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + } + ] }, "results": [ { @@ -46,6 +56,16 @@ ], "system": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "weather_forecast" ], @@ -67,13 +87,13 @@ "entries": [ { "id": "E1", - "seq": 24, + "seq": 26, "type": "user", "state": "committed" }, { "id": "E2", - "seq": 25, + "seq": 27, "type": "system", "state": "committed", "data": { @@ -113,13 +133,13 @@ }, { "id": "E3", - "seq": 26, + "seq": 28, "type": "assistant", "state": "committed" }, { "id": "E4", - "seq": 30, + "seq": 32, "type": "assistant", "state": "committed" } diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.limit.json b/packages/harness/test/__artifacts__/tools.toolsearch.limit.json index 47d5e1a9..f2773cd1 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.limit.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.limit.json @@ -57,6 +57,16 @@ ], "system": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "github_create_issue", "jira_search_issues", diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.live.openai-codex.json b/packages/harness/test/__artifacts__/tools.toolsearch.live.openai-codex.json index ed9cba22..0c1ba101 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.live.openai-codex.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.live.openai-codex.json @@ -8,7 +8,9 @@ "edit", "toolsearch" ], - "system": [] + "system": [ + [] + ] }, { "tools": [ @@ -19,6 +21,7 @@ "toolsearch" ], "system": [ + [], [ "vault_read_token" ] @@ -33,6 +36,7 @@ "toolsearch" ], "system": [ + [], [ "vault_read_token" ] diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.load.json b/packages/harness/test/__artifacts__/tools.toolsearch.load.json index f622611c..330d91e8 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.load.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.load.json @@ -9,7 +9,17 @@ "toolsearch", "notes_read" ], - "system": [] + "system": [ + { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + } + ] }, { "tools": [ @@ -22,6 +32,16 @@ ], "system": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "weather_forecast" ], @@ -40,6 +60,16 @@ ], "system": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], + "added": [], + "removed": [] + }, + { + "sections": [], "added": [ "weather_forecast" ], @@ -88,50 +118,54 @@ }, { "seq": 4, - "type": "session.llm.started.1" + "type": "session.thread.system.changed.1" }, { "seq": 5, - "type": "session.llm.ended.1" + "type": "session.llm.started.1" }, { "seq": 6, - "type": "session.tool.settled.1" + "type": "session.llm.ended.1" }, { "seq": 7, - "type": "session.turn.ended.1" + "type": "session.tool.settled.1" }, { "seq": 8, - "type": "session.thread.tools.changed.1" + "type": "session.turn.ended.1" }, { "seq": 9, - "type": "session.llm.started.1" + "type": "session.thread.system.changed.1" }, { "seq": 10, - "type": "session.llm.ended.1" + "type": "session.llm.started.1" }, { "seq": 11, - "type": "session.tool.settled.1" + "type": "session.llm.ended.1" }, { "seq": 12, - "type": "session.turn.ended.1" + "type": "session.tool.settled.1" }, { "seq": 13, - "type": "session.llm.started.1" + "type": "session.turn.ended.1" }, { "seq": 14, - "type": "session.llm.ended.1" + "type": "session.llm.started.1" }, { "seq": 15, + "type": "session.llm.ended.1" + }, + { + "seq": 16, "type": "session.turn.ended.1" } ], @@ -145,12 +179,28 @@ { "id": "E2", "seq": 4, + "type": "system", + "state": "committed", + "data": { + "sections": [ + "tools", + "rules", + "cwd" + ], + "toolsAdded": [], + "toolsRemoved": [], + "hash": "2aedbc0fffd70a2b41a7ce78f786f9809d727be73bf31b672ac2d603d3817fbb" + } + }, + { + "id": "E3", + "seq": 5, "type": "assistant", "state": "committed" }, { - "id": "E3", - "seq": 8, + "id": "E4", + "seq": 9, "type": "system", "state": "committed", "data": { @@ -189,14 +239,14 @@ } }, { - "id": "E4", - "seq": 9, + "id": "E5", + "seq": 10, "type": "assistant", "state": "committed" }, { - "id": "E5", - "seq": 13, + "id": "E6", + "seq": 14, "type": "assistant", "state": "committed" } @@ -207,7 +257,7 @@ "value": {} }, { - "seq": 6, + "seq": 7, "value": { "modelTools": { "add": [ diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json b/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json index 9dd4139e..c8151453 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json @@ -10,6 +10,11 @@ ], "system": [ { + "sections": [ + "tools", + "rules", + "cwd" + ], "added": [], "removed": [ "notes_read" @@ -64,6 +69,11 @@ "type": "system", "state": "committed", "data": { + "sections": [ + "tools", + "rules", + "cwd" + ], "toolsAdded": [], "toolsRemoved": [ { diff --git a/packages/harness/test/fixtures/instruction.spec.ts b/packages/harness/test/fixtures/instruction.spec.ts index 6ff5cb66..b1fbefce 100644 --- a/packages/harness/test/fixtures/instruction.spec.ts +++ b/packages/harness/test/fixtures/instruction.spec.ts @@ -1,3 +1,4 @@ +import { Transcript } from "@codeworksh/aikit"; import { Effect, type Layer, type Scope } from "effect"; import fs from "node:fs/promises"; import os from "node:os"; @@ -77,7 +78,7 @@ const run = ( hostCwd: path.join(host.root, "cwd"), sandboxes: drivers, llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), diff --git a/packages/harness/test/fixtures/llm.ts b/packages/harness/test/fixtures/llm.ts index 3bf5fe87..5a127476 100644 --- a/packages/harness/test/fixtures/llm.ts +++ b/packages/harness/test/fixtures/llm.ts @@ -59,3 +59,10 @@ export const toolTurn = (...calls: ReadonlyArray): return stream; }); }; + +// Read from the checkout's AGENTS.md and installed skills, so they differ per machine. +const machineSections = new Set(["project_context", "skills"]); + +/** The section names a system message declares, without those that differ per machine. */ +export const sectionNames = (sections: Readonly> | undefined): Array => + Object.keys(sections ?? {}).filter((name) => !machineSections.has(name)); diff --git a/packages/harness/test/fixtures/mcp/turns.ts b/packages/harness/test/fixtures/mcp/turns.ts index 3617ff66..2c5abe06 100644 --- a/packages/harness/test/fixtures/mcp/turns.ts +++ b/packages/harness/test/fixtures/mcp/turns.ts @@ -10,7 +10,7 @@ import { Event } from "../../../src/event/event.ts"; import { EventList } from "../../../src/event/list.ts"; import { toolSearchPlugin } from "../../../src/plugin/builtin/tool/toolsearch.ts"; import type { LLM } from "../../../src/runner/llm.ts"; -import { assistant } from "../llm.ts"; +import { assistant, sectionNames } from "../llm.ts"; const fixtures = fileURLToPath(new URL("./", import.meta.url)); @@ -111,12 +111,13 @@ export const results = (thread: Session.ThreadHandle) => return out; }); -/** Tool changes the transcript declared, as names. */ +/** Section and tool changes the transcript declared, as names. */ export const changes = (messages: ReadonlyArray) => messages.flatMap((message) => message.role === "system" ? [ { + sections: sectionNames(message.sections), added: (message.toolsAdded ?? []).map((tool) => tool.name), removed: (message.toolsRemoved ?? []).map((tool) => tool.name), }, diff --git a/packages/harness/test/fixtures/skill.spec.ts b/packages/harness/test/fixtures/skill.spec.ts index ad2c0b0e..0db0f181 100644 --- a/packages/harness/test/fixtures/skill.spec.ts +++ b/packages/harness/test/fixtures/skill.spec.ts @@ -1,3 +1,4 @@ +import { Transcript } from "@codeworksh/aikit"; import { Effect, type Layer, type Scope } from "effect"; import fs from "node:fs/promises"; import os from "node:os"; @@ -87,7 +88,7 @@ const run = ( hostCwd: path.join(host.root, "cwd"), sandboxes: drivers, llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), diff --git a/packages/harness/test/mcp.thread.test.ts b/packages/harness/test/mcp.thread.test.ts index 536fec7f..8c6cee15 100644 --- a/packages/harness/test/mcp.thread.test.ts +++ b/packages/harness/test/mcp.thread.test.ts @@ -435,7 +435,11 @@ it("mcp: a thread resumed in a new process awaits the server of a tool it loaded expect(first.calls.first).toBe("hello: first"); expect(first.loaded?.add).toContain("mcp__docs__echo"); - expect(second.changes).toEqual([{ added: ["mcp__docs__echo", "mcp__docs__big"], removed: [] }]); + // The prompt and the loaded tools stay declared by the first run's system entries; nothing is declared again. + expect(second.changes).toEqual([ + { sections: ["tools", "rules", "cwd"], added: [], removed: [] }, + { sections: [], added: ["mcp__docs__echo", "mcp__docs__big"], removed: [] }, + ]); expect(second.calls.again).toBe("hello: again"); const { sessionId: _sessionId, ...firstRun } = first; await expect(`${JSON.stringify({ first: firstRun, second }, null, "\t")}\n`).toMatchFileSnapshot( diff --git a/packages/harness/test/plugin.examples.test.ts b/packages/harness/test/plugin.examples.test.ts index 99c7bf78..359ffaa8 100644 --- a/packages/harness/test/plugin.examples.test.ts +++ b/packages/harness/test/plugin.examples.test.ts @@ -1,5 +1,5 @@ import "./utils/env.ts"; -import type { Message } from "@codeworksh/aikit"; +import { type Message, Transcript } from "@codeworksh/aikit"; import { Effect } from "effect"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -65,7 +65,7 @@ describe("extras examples", () => { const { contexts } = await run(root, selection); const context = contexts[0]; expect(context?.tools?.map((tool) => tool.name)).toEqual(["bash", "list_processes", "project_fact"]); - const prompt = context?.systemPrompt ?? ""; + const prompt = context === undefined ? "" : Transcript.getCurrentSystemPrompt(context); // The prompt plugin was named after the tool plugins, so its index carries them. expect(prompt).toContain("- list_processes: List the processes running in the session's environment."); expect(prompt).toContain("- project_fact: Look up a project convention: deploy."); @@ -89,9 +89,11 @@ describe("extras examples", () => { selection, toolTurn(pendingCall("project_fact", { key: "deploy" }, "call_fact")), ); - expect(JSON.parse(path[1]?.parts[0]?.data ?? "{}")).toMatchObject({ - status: "completed", - result: { content: [{ type: "text", text: "vercel" }] }, - }); + expect(JSON.parse(path.find((item) => item.entry.type === "assistant")?.parts[0]?.data ?? "{}")).toMatchObject( + { + status: "completed", + result: { content: [{ type: "text", text: "vercel" }] }, + }, + ); })); }); diff --git a/packages/harness/test/plugin.external.test.ts b/packages/harness/test/plugin.external.test.ts index 76d21576..bb13ceec 100644 --- a/packages/harness/test/plugin.external.test.ts +++ b/packages/harness/test/plugin.external.test.ts @@ -1,5 +1,5 @@ import "./utils/env.ts"; -import type { Message } from "@codeworksh/aikit"; +import { type Message, Transcript } from "@codeworksh/aikit"; import { Cause, Effect, Exit, Fiber, Schema, Stream } from "effect"; import { join } from "node:path"; import { writeFile } from "node:fs/promises"; @@ -57,7 +57,7 @@ const exchange = (input: { database: ":memory:", llm: (request, signal) => { contexts.push(request.context); - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, ...(input.plugins === undefined ? {} : { plugins: input.plugins }), @@ -125,8 +125,8 @@ describe("third-party plugins", () => { expect(contexts[0]?.tools?.map((tool) => tool.name)).toEqual(["acme_echo"]); expect(prompts[0]).toContain("- acme_echo: Echo a value back"); - expect(path.map((item) => item.entry.type)).toEqual(["user", "assistant", "assistant"]); - const settled = JSON.parse(path[1]?.parts[0]?.data ?? "{}"); + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant", "assistant"]); + const settled = JSON.parse(path.find((item) => item.entry.type === "assistant")?.parts[0]?.data ?? "{}"); expect(settled).toMatchObject({ status: "completed", result: { @@ -153,7 +153,9 @@ describe("third-party plugins", () => { pendingCall("acme_secret", { value: "allow" }, "call_allowed"), ), }); - const parts = (path[1]?.parts ?? []).map((part) => JSON.parse(part.data)); + const parts = (path.find((item) => item.entry.type === "assistant")?.parts ?? []).map((part) => + JSON.parse(part.data), + ); const blocked = parts.find((part) => part.callID === "call_blocked"); const allowed = parts.find((part) => part.callID === "call_allowed"); expect(blocked).toMatchObject({ @@ -174,7 +176,7 @@ describe("third-party plugins", () => { llm: toolTurn(pendingCall("bash", { command: "echo hi" }, "call_bash")), }); expect(prompts[0]).toContain("- bash: Run a command through the acme shell"); - const settled = JSON.parse(path[1]?.parts[0]?.data ?? "{}"); + const settled = JSON.parse(path.find((item) => item.entry.type === "assistant")?.parts[0]?.data ?? "{}"); expect(settled).toMatchObject({ status: "completed", result: { content: [{ type: "text", text: "acme-override:echo hi" }] }, @@ -292,7 +294,7 @@ describe("third-party plugins", () => { }); expect(contexts[0]?.tools?.[0]).toMatchObject({ name: "acme_echo", description: "Echo, relabelled by acme" }); expect(prompts[0]).toContain("- acme_echo: Echo a value back"); - const settled = JSON.parse(path[1]?.parts[0]?.data ?? "{}"); + const settled = JSON.parse(path.find((item) => item.entry.type === "assistant")?.parts[0]?.data ?? "{}"); expect(settled.result.content).toEqual([ { type: "text", text: "hi" }, { type: "text", text: "(acme-checked)" }, @@ -340,7 +342,7 @@ describe("third-party plugins", () => { }); expect(contexts[0]?.tools).toEqual([]); expect(prompts[0]).toContain("\n(none)\n"); - const settled = JSON.parse(path[1]?.parts[0]?.data ?? "{}"); + const settled = JSON.parse(path.find((item) => item.entry.type === "assistant")?.parts[0]?.data ?? "{}"); expect(settled).toMatchObject({ status: "error", result: { isError: true } }); expect(settled.result.content[0].text).toContain("Unknown Tool: bash"); })); diff --git a/packages/harness/test/plugin.git.live.test.ts b/packages/harness/test/plugin.git.live.test.ts index 664510e3..7fd81b56 100644 --- a/packages/harness/test/plugin.git.live.test.ts +++ b/packages/harness/test/plugin.git.live.test.ts @@ -1,5 +1,5 @@ import "./utils/env.ts"; -import type { Message } from "@codeworksh/aikit"; +import { type Message, Transcript } from "@codeworksh/aikit"; import { Effect } from "effect"; import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -125,8 +125,8 @@ live("plugins installed from git, as a third party ships them", () => { ]); // A prompt plugin runs after every tool plugin, so it indexed the tool above even // though its settings entry is written after it. - expect(contexts[0]?.systemPrompt).toContain(""); - expect(contexts[0]?.systemPrompt).toContain( + expect(Transcript.getCurrentSystemPrompt(contexts[0]!)).toContain(""); + expect(Transcript.getCurrentSystemPrompt(contexts[0]!)).toContain( "Tools available: bash, read, write, edit, spacex_launch_manifest.", ); }), @@ -173,7 +173,7 @@ live("plugins installed from git, as a third party ships them", () => { const { contexts } = await exchange(workspace); // The 0.4.0 section, which 0.1.0 does not render. - expect(contexts[0]?.systemPrompt).toContain(""); + expect(Transcript.getCurrentSystemPrompt(contexts[0]!)).toContain(""); }), ); diff --git a/packages/harness/test/plugin.host.test.ts b/packages/harness/test/plugin.host.test.ts index 93ae96fb..5b293433 100644 --- a/packages/harness/test/plugin.host.test.ts +++ b/packages/harness/test/plugin.host.test.ts @@ -1,5 +1,5 @@ import "./utils/env.ts"; -import { createAssistantMessageEventStream, type Model } from "@codeworksh/aikit"; +import { createAssistantMessageEventStream, type Model, Transcript } from "@codeworksh/aikit"; import { Deferred, Effect, Fiber, Schema } from "effect"; import { describe, expect, it } from "vite-plus/test"; import { join } from "node:path"; @@ -42,9 +42,14 @@ describe("plugin domains and exchange host", () => { buckets.registry.prompt.foundation.set("Base."); const snapshot = buckets.freeze("/work"); // The patched guideline reaches the harness-rendered ; no snippet, so no index line. - expect(snapshot.systemPrompt).toBe( - ["Base.", "\n(none)\n", "\n- one\n", "\n/work\n"].join("\n\n"), - ); + expect(snapshot.prompt).toEqual({ + foundation: "Base.", + sections: { + tools: "\n(none)\n", + rules: "\n- one\n", + cwd: "\n/work\n", + }, + }); expect(snapshot.tools.defs[0]?.description).toBe("patched"); expect(snapshot.tools.wire[0]?.description).toBe("patched"); expect((await Effect.runPromise(snapshot.tools.handle(pendingCall("echo")))).status).toBe("completed"); @@ -93,7 +98,7 @@ describe("plugin domains and exchange host", () => { database: ":memory:", llm: (input, signal) => { models.push(input.resolvedModel); - observed.push(input.context.systemPrompt ?? ""); + observed.push(Transcript.getCurrentSystemPrompt(input.context)); return open(input, signal); }, plugins: [ @@ -326,7 +331,7 @@ describe("plugin pipelines in the kernel loop", () => { ]); if (mode === "sequential") expect(trace.indexOf("a:done")).toBeLessThan(trace.indexOf("b:before")); const path = yield* session.history(); - expect(path).toHaveLength(3); + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant", "assistant"]); }).pipe( Effect.provide( Harness.layer({ diff --git a/packages/harness/test/plugin.instruction.test.ts b/packages/harness/test/plugin.instruction.test.ts index 096e5b42..a310d49e 100644 --- a/packages/harness/test/plugin.instruction.test.ts +++ b/packages/harness/test/plugin.instruction.test.ts @@ -1,3 +1,4 @@ +import { Transcript } from "@codeworksh/aikit"; import "./utils/env.ts"; import { Effect, Logger } from "effect"; import { execFile } from "node:child_process"; @@ -46,7 +47,7 @@ const prompt = async (input: { root: string; directory: string; hostCwd: string; hostCwd: input.hostCwd, database: ":memory:", llm: (request, signal) => { - observed.push(request.context.systemPrompt ?? ""); + observed.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), diff --git a/packages/harness/test/plugin.passes.test.ts b/packages/harness/test/plugin.passes.test.ts index a32ba1e8..9a4ef0a8 100644 --- a/packages/harness/test/plugin.passes.test.ts +++ b/packages/harness/test/plugin.passes.test.ts @@ -1,3 +1,4 @@ +import { Transcript } from "@codeworksh/aikit"; import "./utils/env.ts"; import { Effect, Option } from "effect"; import { createHash } from "node:crypto"; @@ -108,7 +109,7 @@ describe("a running session", () => { hostCwd: project, database: ":memory:", llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), @@ -280,7 +281,7 @@ describe("a running session and the store", () => { hostCwd: project, database: ":memory:", llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), @@ -349,7 +350,7 @@ describe("boot", () => { hostCwd: project, database: ":memory:", llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), @@ -407,7 +408,7 @@ describe("boot", () => { hostCwd: bin, database: ":memory:", llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), @@ -464,7 +465,7 @@ describe("reload", () => { hostCwd: input.hostCwd ?? input.project, database: ":memory:", llm: (request, signal) => { - input.prompts.push(request.context.systemPrompt ?? ""); + input.prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), @@ -717,7 +718,7 @@ describe("linking a session to a host directory", () => { hostCwd: elsewhere, database: ":memory:", llm: (request, signal) => { - prompts.push(request.context.systemPrompt ?? ""); + prompts.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), diff --git a/packages/harness/test/plugin.prompt.test.ts b/packages/harness/test/plugin.prompt.test.ts index 887c68c7..75d98a5f 100644 --- a/packages/harness/test/plugin.prompt.test.ts +++ b/packages/harness/test/plugin.prompt.test.ts @@ -1,3 +1,4 @@ +import { Transcript } from "@codeworksh/aikit"; import "./utils/env.ts"; import { Effect, Schema } from "effect"; import { describe, expect, it } from "vite-plus/test"; @@ -38,7 +39,7 @@ describe("codework.prompt.default", () => { hostCwd: root, database: ":memory:", llm: (input, signal) => { - observed.push(input.context.systemPrompt ?? ""); + observed.push(Transcript.getCurrentSystemPrompt(input.context)); return open(input, signal); }, ...harness, diff --git a/packages/harness/test/plugin.skill.test.ts b/packages/harness/test/plugin.skill.test.ts index 5400d35b..62afbae4 100644 --- a/packages/harness/test/plugin.skill.test.ts +++ b/packages/harness/test/plugin.skill.test.ts @@ -1,3 +1,4 @@ +import { Transcript } from "@codeworksh/aikit"; import "./utils/env.ts"; import { Effect, Logger, Schema } from "effect"; import { execFile } from "node:child_process"; @@ -59,7 +60,7 @@ const prompt = async (input: { database: ":memory:", ...(input.plugins === undefined ? {} : { plugins: input.plugins }), llm: (request, signal) => { - observed.push(request.context.systemPrompt ?? ""); + observed.push(Transcript.getCurrentSystemPrompt(request.context)); return open(request, signal); }, }), @@ -386,7 +387,7 @@ describe("codework.prompt.skill", () => { database: ":memory:", llm: (request, signal) => { requests.push({ - systemPrompt: request.context.systemPrompt ?? "", + systemPrompt: Transcript.getCurrentSystemPrompt(request.context), messages: request.context.messages, }); return open(request, signal); diff --git a/packages/harness/test/prompt.changes.live.test.ts b/packages/harness/test/prompt.changes.live.test.ts new file mode 100644 index 00000000..396551a8 --- /dev/null +++ b/packages/harness/test/prompt.changes.live.test.ts @@ -0,0 +1,95 @@ +import { Effect } from "effect"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vite-plus/test"; +import { ContextCodec } from "../src/context/codec.ts"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import { define, Section } from "../src/plugin/index.ts"; +import { SessionRuntime } from "../src/session/runtime.ts"; +import type { State } from "../src/state/state.ts"; +import { withSettings } from "./fixtures/settings.ts"; +import "./utils/live.ts"; + +/** + * A prompt section changed between two prompts reaches a real model. Codex takes system messages + * in place, so the second request still carries the first codeword and adds the change after it. + */ + +const Codeword = Section.define("codeword"); +const live = { provider: "openai-codex", id: "gpt-6-luna" } as const; + +describe("prompt sections against live openai-codex", () => { + it.skipIf(!process.env.OPENAI_CODEX_API_KEY?.trim())( + "answers from a section changed between prompts, sent in place", + { timeout: 300_000 }, + () => + withSettings(async ({ root, custom }) => { + await writeFile(join(custom, "settings.jsonc"), JSON.stringify({ model: live })); + const state = { word: "alpha" }; + const codeword = define({ + id: "acme.prompt.codeword", + kind: "prompt", + setup: (ctx) => + ctx.plugin.prompt.sections.append( + Codeword, + `The codeword is ${state.word}. Asked for the codeword, reply with it and nothing else.`, + ), + }); + const bodies: string[] = []; + const onPayload: NonNullable = async (payload) => { + bodies.push(JSON.stringify(payload)); + return payload; + }; + + const replies = await Effect.gen(function* () { + const handle = yield* Session.create({ directory: root }); + const runtime = yield* SessionRuntime.Service; + yield* runtime.update(handle.id, { onPayload }); + yield* handle.main.run("What is the codeword?"); + state.word = "bravo"; + yield* handle.main.run("What is the codeword now?"); + const messages = yield* Effect.forEach(yield* handle.main.history(), ContextCodec.decodeMessage); + return messages.flatMap((message) => + message.role === "assistant" + ? [ + message.parts + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join("") + .trim() + .toLowerCase(), + ] + : [], + ); + }).pipe( + Effect.provide( + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: ":memory:", + userConfigDir: custom, + plugins: [codeword], + }), + ), + Effect.scoped, + Effect.timeout("240 seconds"), + Effect.runPromise, + ); + + expect(bodies).toHaveLength(2); + expect(replies).toHaveLength(2); + expect(replies[0]).toContain("alpha"); + expect(replies[1]).toContain("bravo"); + const second = bodies[1]!; + const artifact = { + answers: replies.map((reply) => ["alpha", "bravo"].find((word) => reply.includes(word))), + promptChange: second.includes('Updated system prompt section \\"codeword\\"') ? "in place" : "folded", + earlierSectionSent: second.includes("The codeword is alpha."), + }; + expect(artifact.promptChange).toBe("in place"); + await expect(`${JSON.stringify(artifact, null, "\t")}\n`).toMatchFileSnapshot( + "./__artifacts__/prompt.changes.live.openai-codex.json", + ); + }), + ); +}); diff --git a/packages/harness/test/prompt.changes.test.ts b/packages/harness/test/prompt.changes.test.ts new file mode 100644 index 00000000..3e6ec274 --- /dev/null +++ b/packages/harness/test/prompt.changes.test.ts @@ -0,0 +1,118 @@ +import { type Message, Transcript } from "@codeworksh/aikit"; +import "./utils/env.ts"; +import { Effect } from "effect"; +import { describe, expect, it } from "vite-plus/test"; +import { join } from "node:path"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import { defaultPromptPlugin } from "../src/plugin/builtin/prompt/default.ts"; +import { define, Section } from "../src/plugin/index.ts"; +import { immediateOpen } from "./fixtures/llm.ts"; +import { withSettings } from "./fixtures/settings.ts"; + +const Mode = Section.define("mode"); + +/** A section's text only where the test sets it; the rest depend on the machine. */ +const shown = (sections: Message.SystemMessage["sections"]) => + Object.fromEntries( + Object.entries(sections ?? {}).map(([name, value]) => [name, name === Mode.name || value === null ? value : "…"]), + ); + +/** What each request carried: its roles, and the section patch of every system message in it. */ +const request = (context: Message.Context) => ({ + roles: context.messages.map((message) => message.role), + declared: context.messages.flatMap((message) => (message.role === "system" ? [shown(message.sections)] : [])), + mode: /\n([\s\S]*?)\n<\/mode>/.exec(Transcript.getCurrentSystemPrompt(context))?.[1] ?? null, +}); + +/** The developer messages a model with in-place system messages gets: the head, then each update. */ +const inPlace = (context: Message.Context) => { + const prompt = Transcript.resolveTranscriptPrompt(context, true); + const updates = context.messages + .filter((message): message is Message.SystemMessage => message.role === "system") + .slice(prompt.foldedSystemMessages) + .map(Transcript.renderSystemPromptUpdate); + return [prompt.systemPrompt, ...updates]; +}; + +const isPrefix = (head: ReadonlyArray, whole: ReadonlyArray) => + head.every((value, index) => whole[index] === value); + +describe("prompt sections across prompts", () => { + it("declares every section once, then only what changed, through resume and fork", () => + withSettings(async ({ root }) => { + const contexts: Message.Context[] = []; + // One open for both runtimes: its reply ids count on across them. + const open = immediateOpen(contexts); + const state: { mode: string | undefined } = { mode: "Plan only." }; + const runtime = () => + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: join(root, "sessions.db"), + llm: open, + plugins: [ + defaultPromptPlugin, + define({ + id: "acme.prompt.mode", + kind: "prompt", + setup: (ctx) => { + if (state.mode !== undefined) ctx.plugin.prompt.sections.append(Mode, state.mode); + }, + }), + ], + }); + const types = (history: ReadonlyArray<{ readonly entry: { readonly type: string } }>) => + history.map(({ entry }) => entry.type); + + const sessionId = await Effect.runPromise( + Effect.gen(function* () { + const { id, main } = yield* Session.create({ directory: root }); + yield* main.run("one"); + state.mode = "Build allowed."; + yield* main.run("two"); + state.mode = undefined; + yield* main.run("three"); + yield* main.run("four"); + return id; + }).pipe(Effect.provide(runtime()), Effect.scoped), + ); + + const threads = await Effect.runPromise( + Effect.gen(function* () { + const main = yield* Session.attach({ sessionId }); + // Resumed in a new process with nothing changed: the transcript already declares it all. + yield* main.run("five"); + // A removed section added again is declared again. + state.mode = "Plan again."; + yield* main.run("six"); + state.mode = undefined; + const history = yield* main.history(); + // Cut after the first reply, whose transcript still has the plan-only section. + const firstReply = history.find(({ entry }) => entry.type === "assistant")!; + const { thread: fork } = yield* main.fork({ entryId: firstReply.entry.id }); + yield* fork.run("seven"); + return { main: types(yield* main.history()), fork: types(yield* fork.history()) }; + }).pipe(Effect.provide(runtime()), Effect.scoped), + ); + + const requests = contexts.map(request); + // Each request extends the one before it, so a model taking changes in place keeps its prefix. + const prefixKept = contexts + .slice(1, 6) + .map((context, index) => isPrefix(inPlace(contexts[index]!), inPlace(context))); + expect(prefixKept).toEqual([true, true, true, true, true]); + expect(requests.map(({ mode }) => mode)).toEqual([ + "Plan only.", + "Build allowed.", + null, + null, + null, + "Plan again.", + null, + ]); + await expect(JSON.stringify({ requests, prefixKept, threads }, null, "\t")).toMatchFileSnapshot( + "./__artifacts__/prompt.changes.json", + ); + })); +}); diff --git a/packages/harness/test/runner.loop.test.ts b/packages/harness/test/runner.loop.test.ts index bf4f8ae1..a5c98be1 100644 --- a/packages/harness/test/runner.loop.test.ts +++ b/packages/harness/test/runner.loop.test.ts @@ -4,7 +4,7 @@ import { Global } from "../src/global.ts"; import { builtins as plugins } from "../src/plugin/builtin.ts"; import type { Info } from "../src/settings/schema.ts"; import { Settings } from "../src/settings/settings.ts"; -import { createAssistantMessageEventStream, Message } from "@codeworksh/aikit"; +import { createAssistantMessageEventStream, Message, Transcript } from "@codeworksh/aikit"; import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Schema } from "effect"; import { SqlClient, SqlSchema } from "effect/sql"; import { describe, expect } from "vite-plus/test"; @@ -173,6 +173,7 @@ describe("runner loop — aikit input/output", () => { expect(bySeq.map((row) => row.id)).toEqual(["msg_steer", "msg_follow"]); expect((yield* threads.line(threadId)).map((item) => item.entry.type)).toEqual([ "user", + "system", "assistant", "user", "assistant", @@ -181,8 +182,8 @@ describe("runner loop — aikit input/output", () => { // Every request is rebuilt from durable history. The second request sees // the first terminal assistant and the newly promoted follow-up. expect(contexts.map((context) => context.messages.map((message) => message.role))).toEqual([ - ["user"], - ["user", "assistant", "user"], + ["user", "system"], + ["user", "system", "assistant", "user"], ]); }), ); @@ -199,7 +200,11 @@ describe("runner loop — aikit input/output", () => { yield* execution.resume(threadId); expect(yield* delivered(threadId)).toEqual([{ id: "msg_once", promotedSeq: 3 }]); - expect((yield* threads.line(threadId)).map((item) => item.entry.type)).toEqual(["user", "assistant"]); + expect((yield* threads.line(threadId)).map((item) => item.entry.type)).toEqual([ + "user", + "system", + "assistant", + ]); }), ); @@ -222,6 +227,7 @@ describe("runner loop — aikit input/output", () => { "user", "user", "user", + "system", "assistant", "user", "assistant", @@ -343,14 +349,15 @@ describe("runner loop — tool continuation and lifecycle gate", () => { expect(contexts).toHaveLength(2); expect(snapshots).toBe(2); - const second = contexts[1]!.messages[1]!; + const second = contexts[1]!.messages.find((message) => message.role === "assistant"); const firstTool = - second.role === "assistant" ? second.parts.find((part) => part.type === "toolCall") : undefined; + second?.role === "assistant" ? second.parts.find((part) => part.type === "toolCall") : undefined; expect(firstTool).toMatchObject({ callID: "call_echo", status: "completed" }); const path = yield* threads.line(threadId); expect(path.map((item) => [item.entry.type, item.entry.state])).toEqual([ ["user", "committed"], + ["system", "committed"], ["assistant", "committed"], ["assistant", "committed"], ]); @@ -359,8 +366,8 @@ describe("runner loop — tool continuation and lifecycle gate", () => { `) as ReadonlyArray<{ readonly type: string; readonly seq: number }>; const firstStarted = eventRows.find((row) => row.type === "session.llm.started.1")!; const firstEnded = eventRows.find((row) => row.type === "session.llm.ended.1")!; - expect(path[1]!.entry.seq).toBe(firstStarted.seq); - expect(path[1]!.entry.seq).toBeLessThan(firstEnded.seq); + expect(path[2]!.entry.seq).toBe(firstStarted.seq); + expect(path[2]!.entry.seq).toBeLessThan(firstEnded.seq); expect(path.filter((item) => item.entry.id === "assistant_1")).toHaveLength(1); }), ); @@ -505,9 +512,10 @@ describe("runner loop — tool interruption", () => { expect(interrupted.data.reason).toBe("user"); const path = yield* threads.line(threadId); - expect(path[1]!.entry.state).toBe("committed"); - expect(path[1]!.parts.map((part) => Option.getOrThrow(part.status))).toEqual(["aborted", "aborted"]); - for (const part of path[1]!.parts) { + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); + expect(path[2]!.entry.state).toBe("committed"); + expect(path[2]!.parts.map((part) => Option.getOrThrow(part.status))).toEqual(["aborted", "aborted"]); + for (const part of path[2]!.parts) { expect(JSON.parse(part.data)).toMatchObject({ status: "aborted", result: { content: [{ type: "text", text: "Tool Execution Interrupted" }], isError: true }, @@ -570,10 +578,10 @@ describe("runner loop — provider interruption", () => { expect(Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause)).toBe(true); const path = yield* threads.line(threadId); - expect(path.map((item) => item.entry.type)).toEqual(["user", "assistant"]); - expect(path[1]!.entry.state).toBe("aborted"); - expect(JSON.parse(path[1]!.entry.data)).toMatchObject({ stopReason: "aborted" }); - expect(path[1]!.parts).toEqual([]); + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); + expect(path[2]!.entry.state).toBe("aborted"); + expect(JSON.parse(path[2]!.entry.data)).toMatchObject({ stopReason: "aborted" }); + expect(path[2]!.parts).toEqual([]); expect(Array.from(yield* execution.active)).toEqual([]); }), { timeout: 10_000 }, @@ -627,10 +635,10 @@ describe("runner loop — provider failure", () => { } const path = yield* threads.line(threadId); - expect(path.map((item) => item.entry.type)).toEqual(["user", "assistant"]); - expect(path[1]!.entry.state).toBe("aborted"); - expect(path[1]!.parts).toEqual([]); - expect(JSON.parse(path[1]!.entry.data)).toMatchObject({ + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); + expect(path[2]!.entry.state).toBe("aborted"); + expect(path[2]!.parts).toEqual([]); + expect(JSON.parse(path[2]!.entry.data)).toMatchObject({ stopReason: "error", errorMessage: "provider failed", failure: { _tag: "Authentication", reason: "missing", retryable: false }, @@ -721,6 +729,8 @@ describe("runner loop — retry", () => { Effect.sync(() => { if (!event.type.startsWith("session.")) return; if (event.type.startsWith("session.prompt.") || event.type.startsWith("session.execution.")) return; + // A system entry's id is random. + if (event.type === EventList.SystemChanged.type) return void seen.push({ type: event.type }); const data = event.data as Record; const picked = Object.fromEntries( ["messageId", "attempt", "maxRetries", "delayMs", "success", "reason", "cause", "message"] @@ -736,7 +746,7 @@ describe("runner loop — retry", () => { const pathOf = Effect.fnUntraced(function* (threadId: ThreadSchema.ID) { const threads = yield* Thread.Service; return (yield* threads.line(threadId)).map((item) => ({ - id: item.entry.id, + id: item.entry.type === "system" ? "system" : item.entry.id, type: item.entry.type, state: item.entry.state, parts: item.parts.length, @@ -765,7 +775,13 @@ describe("runner loop — retry", () => { yield* execution.resume(threadId); const path = yield* pathOf(threadId); - expect(path.map((entry) => entry.state)).toEqual(["committed", "aborted", "aborted", "committed"]); + expect(path.map((entry) => entry.state)).toEqual([ + "committed", + "committed", + "aborted", + "aborted", + "committed", + ]); // Every attempt is sent the original context: the failed drafts and their partial text never leak in. expect(contexts).toHaveLength(3); expect(contexts[1]).toEqual(contexts[0]); @@ -804,7 +820,11 @@ describe("runner loop — retry", () => { const exit = yield* Fiber.await(waiting); expect(Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause)).toBe(true); - expect((yield* pathOf(threadId)).map((entry) => entry.state)).toEqual(["committed", "aborted"]); + expect((yield* pathOf(threadId)).map((entry) => entry.state)).toEqual([ + "committed", + "committed", + "aborted", + ]); expect(seen.slice(-2)).toEqual([ { type: EventList.RetryFinished.type, attempt: 1, success: false, message: "interrupted" }, { type: EventList.TurnAborted.type, cause: { _tag: "interrupted" } }, @@ -835,7 +855,11 @@ describe("runner loop — retry", () => { const exit = yield* execution.resume(threadId).pipe(Effect.exit); expect(Exit.isFailure(exit)).toBe(true); expect(seen.some((event) => String(event.type).startsWith("session.retry."))).toBe(false); - expect((yield* pathOf(threadId)).map((entry) => entry.state)).toEqual(["committed", "aborted"]); + expect((yield* pathOf(threadId)).map((entry) => entry.state)).toEqual([ + "committed", + "committed", + "aborted", + ]); }), ); } @@ -1254,6 +1278,7 @@ describe("runner loop — resume after restart", () => { expect(path.map((entry) => [entry.entry.type, entry.entry.state])).toEqual([ ["user", "committed"], ["assistant", "aborted"], + ["system", "committed"], ["assistant", "committed"], ]); expect(yield* stateOf(threadId)).toEqual({ execution: null, restarts: 0 }); @@ -1535,9 +1560,10 @@ describe("runner loop — lifecycle hooks", () => { const path = yield* threads.line(threadId); expect(path.map((item) => [item.entry.type, item.entry.state])).toEqual([ ["user", "committed"], + ["system", "committed"], ["assistant", "committed"], ]); - const tool = path[1]!.parts.find((part) => part.type === "toolCall"); + const tool = path[2]!.parts.find((part) => part.type === "toolCall"); expect(tool).toBeDefined(); const rows = yield* delivered(threadId); expect(rows.find((row) => row.id === "msg_queued")?.promotedSeq).toBeNull(); @@ -1545,8 +1571,14 @@ describe("runner loop — lifecycle hooks", () => { // The next run answers the tool results first, then picks the queued follow-up up. yield* execution.resume(threadId); expect(contexts).toHaveLength(3); - expect(contexts[1]!.messages).toHaveLength(2); - expect(contexts[2]!.messages).toHaveLength(4); + expect(contexts[1]!.messages.map((message) => message.role)).toEqual(["user", "system", "assistant"]); + expect(contexts[2]!.messages.map((message) => message.role)).toEqual([ + "user", + "system", + "assistant", + "assistant", + "user", + ]); const next = yield* delivered(threadId); expect(next.find((row) => row.id === "msg_queued")?.promotedSeq).not.toBeNull(); }), @@ -1661,7 +1693,8 @@ describe("runner loop — lifecycle hooks", () => { runtime({ open: recording(contexts), hooks: hooksOf({ - prepareNextTurn: (_turn, next) => during.run.pipe(Effect.as({ ...next, systemPrompt: "replaced" })), + prepareNextTurn: (_turn, next) => + during.run.pipe(Effect.as({ ...next, prompt: { ...next.prompt, foundation: "replaced" } })), }), plugins: [...plugins, { id: "test.tool.echo", kind: "tool", setup: (ctx) => ctx.plugin.tools.add(echo) }], }), @@ -1831,9 +1864,11 @@ describe("runner loop — tool exposure", () => { expect(contexts).toHaveLength(2); const section = (name: string) => - new RegExp(`<${name}>\\n([\\s\\S]*?)\\n`).exec(contexts[0]!.systemPrompt ?? "")?.[1]; - const reply = contexts[1]!.messages[1]!; - const call = reply.role === "assistant" ? reply.parts.find((part) => part.type === "toolCall") : undefined; + new RegExp(`<${name}>\\n([\\s\\S]*?)\\n`).exec( + Transcript.getCurrentSystemPrompt(contexts[0]!), + )?.[1]; + const reply = contexts[1]!.messages.find((message) => message.role === "assistant"); + const call = reply?.role === "assistant" ? reply.parts.find((part) => part.type === "toolCall") : undefined; const result = { requestTools: contexts.map((context) => (context.tools ?? []).map((tool) => tool.name)), tools: section("tools"), diff --git a/packages/harness/test/sdk.test.ts b/packages/harness/test/sdk.test.ts index 5d6b44f2..3edb0faa 100644 --- a/packages/harness/test/sdk.test.ts +++ b/packages/harness/test/sdk.test.ts @@ -86,8 +86,10 @@ describe("Harness Effect SDK", () => { yield* Effect.gen(function* () { const session = yield* Session.attach({ sessionId }); yield* session.run("second"); + // The restarted runtime finds the prompt the first run declared, so declares nothing again. expect((yield* session.history()).map(({ entry }) => entry.type)).toEqual([ "user", + "system", "assistant", "user", "assistant", diff --git a/packages/harness/test/settings.loop.test.ts b/packages/harness/test/settings.loop.test.ts index c4b35dbc..e30fac4b 100644 --- a/packages/harness/test/settings.loop.test.ts +++ b/packages/harness/test/settings.loop.test.ts @@ -1,5 +1,5 @@ import "./utils/env.ts"; -import { createAssistantMessageEventStream } from "@codeworksh/aikit"; +import { createAssistantMessageEventStream, Transcript } from "@codeworksh/aikit"; import { Deferred, Effect, Fiber, Schema } from "effect"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -100,7 +100,9 @@ describe("settings at turn boundaries", () => { // The "high" edit landed during the tool batch, so the continuation turn reads it. expect(inputs.map((input) => input.thinkingLevel)).toEqual(["low", "high", "high"]); expect(inputs.map((input) => input.options?.headers?.revision)).toEqual(["low", "high", "high"]); - expect(inputs[1]?.context.systemPrompt).toBe(inputs[0]?.context.systemPrompt); + expect(Transcript.getCurrentSystemPrompt(inputs[1]!.context)).toBe( + Transcript.getCurrentSystemPrompt(inputs[0]!.context), + ); yield* Effect.promise(() => file(custom, "off")); yield* handle.run("idle steer"); expect(inputs.at(-1)?.thinkingLevel).toBe("off"); diff --git a/packages/harness/test/thread.test.ts b/packages/harness/test/thread.test.ts index c4e85ca2..60b2383d 100644 --- a/packages/harness/test/thread.test.ts +++ b/packages/harness/test/thread.test.ts @@ -306,8 +306,15 @@ describe("threads", () => { yield* run(sessionId, main, "Now add tests"); const line = yield* threads.line(main); - expect(line.map(({ entry }) => entry.type)).toEqual(["user", "assistant", "assistant", "user", "assistant"]); - const [, e2, e3] = line; + expect(line.map(({ entry }) => entry.type)).toEqual([ + "user", + "system", + "assistant", + "assistant", + "user", + "assistant", + ]); + const [, , e2, e3] = line; const forked = yield* threads.fork({ threadId: main, entryId: e3!.entry.id }); const t2 = ThreadSchema.ID.make(forked.thread.id); @@ -352,7 +359,9 @@ describe("threads", () => { const { sessionId, main } = yield* createSession; yield* run(sessionId, main, "first"); yield* run(sessionId, main, "second"); - const [e1, e2, e3, e4] = yield* threads.line(main); + const line = yield* threads.line(main); + expect(line.map(({ entry }) => entry.type)).toEqual(["user", "system", "assistant", "user", "assistant"]); + const [e1, , e2, e3, e4] = line; // An inherited entry re-points the origin to the thread that wrote it. const t2 = ThreadSchema.ID.make((yield* threads.fork({ threadId: main, entryId: e4!.entry.id })).thread.id); @@ -445,11 +454,13 @@ describe("threads", () => { const result = { bothActive: true, main: yield* own(main), fork: yield* own(t2) }; expect(result.main).toEqual([ "user committed", + "system committed", "assistant committed", "user committed", "assistant committed", ]); - expect(result.fork).toEqual(["user committed", "assistant committed"]); + // The fork cuts before main's system entry, so its first request declares its own. + expect(result.fork).toEqual(["user committed", "system committed", "assistant committed"]); // Seqs interleave between the two runs, so the artifact holds per-thread facts only. yield* artifact("thread.concurrent.json", result); }), @@ -475,7 +486,8 @@ describe("threads", () => { }); yield* run(sessionId, main, "after the reset"); const last = contexts.at(-1)!; - expect(last.roles).toEqual(["user", "user"]); + // The reset drops the earlier system entry too, so the request declares the prompt again. + expect(last.roles).toEqual(["user", "user", "system"]); expect(last.texts.includes("before the reset")).toBe(false); expect(last.texts.includes("keep the login work")).toBe(true); yield* artifact("thread.reset.json", { @@ -537,14 +549,14 @@ describe("threads", () => { const own = (yield* threads.line(t2)) .filter(({ entry }) => entry.threadId === t2) .map(({ entry }) => `${entry.type} ${entry.state}`); - expect(own).toEqual(["user committed", "assistant aborted", "assistant committed"]); + expect(own).toEqual(["user committed", "assistant aborted", "system committed", "assistant committed"]); const thread = Option.getOrThrow(yield* threads.get(t2)); expect(Option.getOrUndefined(thread.execution)).toBeUndefined(); expect(thread.restarts).toBe(0); // Only the thread that was cut off ran again, from its committed entries: the inherited // prompt and its own, without the aborted draft. expect(contexts.filter((one) => one.threadId === main)).toHaveLength(1); - expect(contexts.at(-1)!.roles).toEqual(["user", "user"]); + expect(contexts.at(-1)!.roles).toEqual(["user", "user", "system"]); yield* artifact("thread.recovery.json", yield* snapshot(sessionId)); }), ); diff --git a/packages/harness/test/tools.toolsearch.test.ts b/packages/harness/test/tools.toolsearch.test.ts index b698a32a..1ed32fcb 100644 --- a/packages/harness/test/tools.toolsearch.test.ts +++ b/packages/harness/test/tools.toolsearch.test.ts @@ -31,7 +31,7 @@ import { State } from "../src/state/state.ts"; import { ThreadSchema } from "../src/thread/schema.ts"; import { Thread } from "../src/thread/thread.ts"; import * as Tool from "../src/tool/tool.ts"; -import { assistant } from "./fixtures/llm.ts"; +import { assistant, sectionNames } from "./fixtures/llm.ts"; import { pooled } from "./fixtures/pool.ts"; import { seedSpace } from "./fixtures/space.ts"; import { testEffect } from "./utils/effect.ts"; @@ -233,13 +233,17 @@ const run = (sessionId: SessionSchema.ID, threadId: ThreadSchema.ID, prompt: str yield* control.run({ sessionId, threadId, prompt: PromptSchema.Prompt.fromText(prompt), delivery: "followUp" }); }); -/** A request as the model saw it: its declared tools, and the tool changes its system messages carry. */ +/** + * A request as the model saw it: its declared tools, and the section names and tool changes its + * system messages carry. + */ const sent = (context: Message.Context) => ({ tools: (context.tools ?? []).map((tool) => tool.name), system: context.messages.flatMap((message) => message.role === "system" ? [ { + sections: sectionNames(message.sections), added: (message.toolsAdded ?? []).map((tool) => tool.name), removed: (message.toolsRemoved ?? []).map((tool) => tool.name), }, @@ -274,6 +278,12 @@ const results = Effect.fnUntraced(function* (threadId: ThreadSchema.ID) { ); }); +/** A system entry's data, its sections by name only. */ +const systemData = (data: string) => { + const { sections, ...rest } = JSON.parse(data) as { readonly sections?: Record }; + return { ...(sections === undefined ? {} : { sections: sectionNames(sections) }), ...rest }; +}; + /** The thread's events, entries and `agent` revisions, by seq; ids aliased in order seen. */ const ledger = Effect.fnUntraced(function* (sessionId: SessionSchema.ID, threadId: ThreadSchema.ID) { const sql = yield* SqlClient.SqlClient; @@ -304,13 +314,16 @@ const ledger = Effect.fnUntraced(function* (sessionId: SessionSchema.ID, threadI seq: entry.seq, type: entry.type, state: entry.state, - ...(entry.type === "system" ? { data: JSON.parse(entry.data) as unknown } : {}), + ...(entry.type === "system" ? { data: systemData(entry.data) } : {}), })), agent: revisions.map((revision) => ({ seq: revision.seq, value: JSON.parse(revision.value) as unknown })), }; }); type Ledger = Effect.Success>; +/** The first request's system message: every prompt section, no tool change. */ +const prompt = { sections: ["tools", "rules", "cwd"], added: [], removed: [] }; + const systemEntries = (book: Ledger) => book.entries.filter((entry) => entry.type === "system"); const settledSeqs = (book: Ledger) => book.events.filter((event) => event.type.startsWith(`${EventList.ToolSettled.type}.`)).map((event) => event.seq); @@ -352,15 +365,17 @@ describe("toolsearch", () => { expect(first!.tools).toContain(TOOL_SEARCH_NAME); expect(first!.tools).not.toContain("weather_forecast"); expect(second!.tools).toEqual(first!.tools); - expect(first!.system).toEqual([]); - expect(second!.system).toEqual([{ added: ["weather_forecast"], removed: [] }]); - // The third request replays the one system entry; nothing new is declared. + // The first request declares the prompt sections; loading a tool leaves them as they are. + expect(first!.system).toEqual([prompt]); + expect(second!.system).toEqual([prompt, { sections: [], added: ["weather_forecast"], removed: [] }]); + // The third request replays both system entries; nothing new is declared. expect(third!.tools).toEqual(first!.tools); expect(third!.system).toEqual(second!.system); - expect(systemEntries(book)).toHaveLength(1); - expect(systemEntries(book)[0]!.data).toMatchObject({ - toolsAdded: [{ name: "weather_forecast" }], + expect(systemEntries(book)).toHaveLength(2); + expect(systemEntries(book)[1]!.data).toEqual({ + toolsAdded: [expect.objectContaining({ name: "weather_forecast" })], toolsRemoved: [], + hash: expect.any(String), }); // The loaded tool runs. expect(outcomes[1]).toMatchObject({ @@ -397,14 +412,14 @@ describe("toolsearch", () => { ]); for (const failed of outcomes.slice(0, 3)) expect(failed.details).toMatchObject({ _tag: "ToolSearchFailed" }); expect(outcomes[3]!.details).toEqual({ loaded: [] }); - // Nothing loaded: no `agent` change and no system entry. + // Nothing loaded: no `agent` change and no system entry past the first request's prompt. expect(book.agent.some((revision) => JSON.stringify(revision.value).includes("modelTools"))).toBe(false); - expect(systemEntries(book)).toEqual([]); + expect(systemEntries(book)).toHaveLength(1); expect( requestsOf(main) .map(sent) .map((one) => one.system), - ).toEqual([[], []]); + ).toEqual([[prompt], [prompt]]); yield* artifact("tools.toolsearch.refused.json", { results: outcomes, agent: book.agent }); }), ); @@ -429,8 +444,7 @@ describe("toolsearch", () => { expect(add.slice(0, 2)).toEqual(loadedOf(limited!.details)); expect(new Set(add)).toEqual(new Set([...loadedOf(limited!.details), ...loadedOf(all!.details)])); const [, next] = requestsOf(main).map(sent); - expect(next!.system).toHaveLength(1); - expect(next!.system[0]!.added).toEqual(add); + expect(next!.system).toEqual([prompt, { sections: [], added: add, removed: [] }]); yield* artifact("tools.toolsearch.limit.json", { results: [limited, all], @@ -449,11 +463,11 @@ describe("toolsearch", () => { const { sessionId, main } = yield* createSession; yield* run(sessionId, main, "Load the forecast tool", [call(search("weather forecast", 1)), reply]); const line = yield* threads.line(main); - expect(line.map(({ entry }) => entry.type)).toEqual(["user", "assistant", "system", "assistant"]); - const [prompt, searched] = line; + expect(line.map(({ entry }) => entry.type)).toEqual(["user", "system", "assistant", "system", "assistant"]); + const [asked, , searched] = line; const before = ThreadSchema.ID.make( - (yield* threads.fork({ threadId: main, entryId: prompt!.entry.id })).thread.id, + (yield* threads.fork({ threadId: main, entryId: asked!.entry.id })).thread.id, ); const after = ThreadSchema.ID.make( (yield* threads.fork({ threadId: main, entryId: searched!.entry.id })).thread.id, @@ -471,14 +485,15 @@ describe("toolsearch", () => { const [afterFirst] = requestsOf(after).map(sent); const beforeResults = yield* results(before); const afterResults = yield* results(after); - expect(beforeFirst!.system).toEqual([]); + // Cut before main's first system entry, the fork declares the prompt itself. + expect(beforeFirst!.system).toEqual([prompt]); expect(beforeResults[0]).toMatchObject({ status: "error", text: 'Tool "weather_forecast" is not available in this thread.', details: { _tag: "ToolUnavailable", name: "weather_forecast" }, }); - // The history up to the cut has no system entry; the fork declares the tool itself. - expect(afterFirst!.system).toEqual([{ added: ["weather_forecast"], removed: [] }]); + // The history up to the cut declares the prompt but not the tool; the fork declares the tool itself. + expect(afterFirst!.system).toEqual([prompt, { sections: [], added: ["weather_forecast"], removed: [] }]); expect(afterResults[0]).toMatchObject({ status: "completed", text: "weather_forecast ran" }); const afterBook = yield* ledger(sessionId, after); expect(systemEntries(afterBook)).toHaveLength(1); @@ -536,7 +551,10 @@ describe("toolsearch", () => { }); // The removal is declared to the model as a system entry. expect(systemEntries(book)).toHaveLength(1); - expect(systemEntries(book)[0]!.data).toMatchObject({ toolsRemoved: [{ name: "notes_read" }] }); + expect(systemEntries(book)[0]!.data).toMatchObject({ + sections: prompt.sections, + toolsRemoved: [{ name: "notes_read" }], + }); }), ); }); From a6dfc37adb82cfb060c02e19cd8b71550ba26e92 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 31 Oct 2026 00:47:05 +0530 Subject: [PATCH 02/11] feat(harness): declare prompt section changes ahead of the prompt The system entry now lands before the prompt it comes with, so the model reads the setup, then the request. The finished-reply guard and fork-before skip system entries, so a declaration left by a cancelled prompt is inert. Co-Authored-By: Claude Opus 5.5 --- packages/harness/src/context/assemble.ts | 3 +- packages/harness/src/runner/loop.ts | 10 +++- packages/harness/src/thread/thread.ts | 3 +- .../test/__artifacts__/prompt.changes.json | 58 +++++++++---------- .../prompt.changes.live.openai-codex.json | 3 +- .../test/__artifacts__/runner.retry.json | 12 ++-- .../test/__artifacts__/thread.concurrent.json | 3 +- .../test/__artifacts__/thread.fork.json | 10 ++-- .../test/__artifacts__/thread.recovery.json | 20 +++---- .../test/__artifacts__/thread.reset.json | 22 +++---- .../__artifacts__/thread.walkthrough.json | 10 ++-- .../test/__artifacts__/tools.read.vision.json | 18 +++--- .../__artifacts__/tools.toolsearch.fork.json | 18 +++--- .../__artifacts__/tools.toolsearch.load.json | 16 ++--- .../tools.toolsearch.unavailable.json | 12 ++-- packages/harness/test/plugin.external.test.ts | 2 +- packages/harness/test/plugin.host.test.ts | 2 +- .../harness/test/prompt.changes.live.test.ts | 7 ++- packages/harness/test/runner.loop.test.ts | 26 ++++----- packages/harness/test/sdk.test.ts | 2 +- packages/harness/test/thread.test.ts | 22 +++---- .../harness/test/tools.toolsearch.test.ts | 4 +- 22 files changed, 147 insertions(+), 136 deletions(-) diff --git a/packages/harness/src/context/assemble.ts b/packages/harness/src/context/assemble.ts index 148c126e..58ad0f4e 100644 --- a/packages/harness/src/context/assemble.ts +++ b/packages/harness/src/context/assemble.ts @@ -8,6 +8,7 @@ import { ContextDecodeError } from "./errors.ts"; export interface Snapshot { readonly threadId: string; + /** The newest entry other than `system`: a declaration left by a cancelled prompt is nothing to answer. */ readonly leafEntryId: string | null; readonly messages: ReadonlyArray; readonly lastAssistant?: { @@ -70,7 +71,7 @@ export const assembleLine = Effect.fn("Context.assembleLine")(function* ( threadId: string, line: ReadonlyArray, ): Effect.fn.Return { - const leafEntryId = line.at(-1)?.entry.id ?? null; + const leafEntryId = line.findLast((hydrated) => hydrated.entry.type !== "system")?.entry.id ?? null; const committedPath = line.filter((hydrated) => hydrated.entry.state === "committed"); let lastAssistant: Snapshot["lastAssistant"]; let latestStart: diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index f26d2ac2..3145d724 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -279,7 +279,7 @@ export const layer = (options: Options = {}) => * The request keeps the foundation as `systemPrompt` and the declared tools as `tools`. The * prompt sections, and what the thread offers beyond those tools (or withholds), are declared * by a `system` entry, published when the transcript's state differs from the snapshot's: the - * first request declares every section. Returns its message, for the request at hand. + * first request declares every section. Returns its message, for a request not reassembled after it. */ const declareSystemChanges = Effect.fn("Loop.declareSystemChanges")(function* ( snapshot: State.Snapshot, @@ -315,6 +315,10 @@ export const layer = (options: Options = {}) => continuing: boolean, ) { const { sessionId, threadId } = snapshot; + // Declared ahead of the input it comes with, so the model reads the setup, then the prompt. + // With no input to promote it goes last, once a request is sure to follow. + const declaredFirst = promotion !== undefined && (yield* inputs.hasPending(threadId, promotion)); + if (declaredFirst) yield* declareSystemChanges(snapshot, (yield* context.assemble(threadId)).messages); if (promotion !== undefined) { const cutoff = yield* events.latestSequence(sessionId); const followUp = promotion === "followUp" && (yield* inputs.promoteFollowUp(threadId)); @@ -330,7 +334,9 @@ export const layer = (options: Options = {}) => if (leaf !== undefined && (leaf.stopReason === "stop" || leaf.stopReason === "length")) { return undefined; } - const messages = [...assembled.messages, ...(yield* declareSystemChanges(snapshot, assembled.messages))]; + const messages = declaredFirst + ? assembled.messages + : [...assembled.messages, ...(yield* declareSystemChanges(snapshot, assembled.messages))]; // The latest attempt's publisher; a retried turn has one per provider request. let publisher: LLMEventPublisher.Publisher | undefined; diff --git a/packages/harness/src/thread/thread.ts b/packages/harness/src/thread/thread.ts index ce5bf81b..f6b8b881 100644 --- a/packages/harness/src/thread/thread.ts +++ b/packages/harness/src/thread/thread.ts @@ -674,7 +674,8 @@ export const layer = Layer.effect( if (target.state !== "committed") return yield* reject("entry_not_committed"); if (input.mode !== "before") return { point: target, text: undefined }; if (target.type !== "user") return yield* reject("not_a_user_entry"); - const before = entries.slice(0, index).findLast((one) => one.state === "committed"); + // A `system` entry only declares setup for the prompt after it: the prompt's own resend declares again. + const before = entries.slice(0, index).findLast((one) => one.state === "committed" && one.type !== "system"); if (before === undefined) return yield* reject("nothing_before_entry"); const parts = yield* partsFor([target]); const text = parts diff --git a/packages/harness/test/__artifacts__/prompt.changes.json b/packages/harness/test/__artifacts__/prompt.changes.json index a8a65896..0f0842ac 100644 --- a/packages/harness/test/__artifacts__/prompt.changes.json +++ b/packages/harness/test/__artifacts__/prompt.changes.json @@ -2,8 +2,8 @@ "requests": [ { "roles": [ - "user", - "system" + "system", + "user" ], "declared": [ { @@ -17,11 +17,11 @@ }, { "roles": [ - "user", "system", - "assistant", "user", - "system" + "assistant", + "system", + "user" ], "declared": [ { @@ -38,14 +38,14 @@ }, { "roles": [ - "user", "system", - "assistant", "user", - "system", "assistant", + "system", "user", - "system" + "assistant", + "system", + "user" ], "declared": [ { @@ -65,14 +65,14 @@ }, { "roles": [ - "user", "system", - "assistant", "user", - "system", "assistant", + "system", "user", + "assistant", "system", + "user", "assistant", "user" ], @@ -94,14 +94,14 @@ }, { "roles": [ - "user", "system", - "assistant", "user", - "system", "assistant", + "system", "user", + "assistant", "system", + "user", "assistant", "user", "assistant", @@ -125,21 +125,21 @@ }, { "roles": [ - "user", "system", - "assistant", "user", - "system", "assistant", - "user", "system", + "user", "assistant", + "system", "user", "assistant", "user", "assistant", "user", - "system" + "assistant", + "system", + "user" ], "declared": [ { @@ -162,11 +162,11 @@ }, { "roles": [ - "user", "system", - "assistant", "user", - "system" + "assistant", + "system", + "user" ], "declared": [ { @@ -191,29 +191,29 @@ ], "threads": { "main": [ - "user", "system", - "assistant", "user", - "system", "assistant", - "user", "system", + "user", "assistant", + "system", "user", "assistant", "user", "assistant", "user", + "assistant", "system", + "user", "assistant" ], "fork": [ - "user", "system", - "assistant", "user", + "assistant", "system", + "user", "assistant" ] } diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json b/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json index 9fb6e344..fe1f7638 100644 --- a/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json +++ b/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json @@ -4,5 +4,6 @@ "bravo" ], "promptChange": "in place", - "earlierSectionSent": true + "earlierSectionSent": true, + "changeBeforePrompt": true } diff --git a/packages/harness/test/__artifacts__/runner.retry.json b/packages/harness/test/__artifacts__/runner.retry.json index 56dbd33f..8256804a 100644 --- a/packages/harness/test/__artifacts__/runner.retry.json +++ b/packages/harness/test/__artifacts__/runner.retry.json @@ -65,18 +65,18 @@ } ], "path": [ - { - "id": "msg_retry", - "type": "user", - "state": "committed", - "parts": 1 - }, { "id": "system", "type": "system", "state": "committed", "parts": 0 }, + { + "id": "msg_retry", + "type": "user", + "state": "committed", + "parts": 1 + }, { "id": "assistant_1", "type": "assistant", diff --git a/packages/harness/test/__artifacts__/thread.concurrent.json b/packages/harness/test/__artifacts__/thread.concurrent.json index 54a60090..42839aea 100644 --- a/packages/harness/test/__artifacts__/thread.concurrent.json +++ b/packages/harness/test/__artifacts__/thread.concurrent.json @@ -1,15 +1,14 @@ { "bothActive": true, "main": [ - "user committed", "system committed", + "user committed", "assistant committed", "user committed", "assistant committed" ], "fork": [ "user committed", - "system committed", "assistant committed" ] } diff --git a/packages/harness/test/__artifacts__/thread.fork.json b/packages/harness/test/__artifacts__/thread.fork.json index 07735f90..1ab5f53f 100644 --- a/packages/harness/test/__artifacts__/thread.fork.json +++ b/packages/harness/test/__artifacts__/thread.fork.json @@ -9,8 +9,8 @@ "events": [ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", - "3 session.prompt.promoted.1 T1", - "4 session.thread.system.changed.1 T1", + "3 session.thread.system.changed.1 T1", + "4 session.prompt.promoted.1 T1", "5 session.llm.started.1 T1", "6 session.llm.ended.1 T1", "7 session.turn.ended.1 T1", @@ -42,7 +42,7 @@ }, { "thread": "T3", - "origin": "fork T1 at E1", + "origin": "fork T1 at E2", "createdSeq": 14, "title": null }, @@ -54,8 +54,8 @@ } ], "entries": [ - "E1 T1 user seq=3 committed=3 committed", - "E2 T1 system seq=4 committed=4 committed", + "E1 T1 system seq=3 committed=3 committed", + "E2 T1 user seq=4 committed=4 committed", "E3 T1 assistant seq=5 committed=7 committed", "E4 T1 user seq=9 committed=9 committed", "E5 T1 assistant seq=10 committed=12 committed", diff --git a/packages/harness/test/__artifacts__/thread.recovery.json b/packages/harness/test/__artifacts__/thread.recovery.json index f5bdc287..ebfa8b1d 100644 --- a/packages/harness/test/__artifacts__/thread.recovery.json +++ b/packages/harness/test/__artifacts__/thread.recovery.json @@ -2,8 +2,8 @@ "events": [ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", - "3 session.prompt.promoted.1 T1", - "4 session.thread.system.changed.1 T1", + "3 session.thread.system.changed.1 T1", + "4 session.prompt.promoted.1 T1", "5 session.llm.started.1 T1", "6 session.llm.ended.1 T1", "7 session.turn.ended.1 T1", @@ -12,10 +12,9 @@ "10 session.prompt.promoted.1 T2", "11 session.llm.started.1 T2", "12 session.llm.failed.1 T2", - "13 session.thread.system.changed.1 T2", - "14 session.llm.started.1 T2", - "15 session.llm.ended.1 T2", - "16 session.turn.ended.1 T2" + "13 session.llm.started.1 T2", + "14 session.llm.ended.1 T2", + "15 session.turn.ended.1 T2" ], "threads": [ { @@ -26,19 +25,18 @@ }, { "thread": "T2", - "origin": "fork T1 at E1", + "origin": "fork T1 at E2", "createdSeq": 8, "title": "cut off" } ], "entries": [ - "E1 T1 user seq=3 committed=3 committed", - "E2 T1 system seq=4 committed=4 committed", + "E1 T1 system seq=3 committed=3 committed", + "E2 T1 user seq=4 committed=4 committed", "E3 T1 assistant seq=5 committed=7 committed", "E4 T2 user seq=10 committed=10 committed", "E5 T2 assistant seq=11 committed=12 aborted", - "E6 T2 system seq=13 committed=13 committed", - "E7 T2 assistant seq=14 committed=16 committed" + "E6 T2 assistant seq=13 committed=15 committed" ], "documents": [ { diff --git a/packages/harness/test/__artifacts__/thread.reset.json b/packages/harness/test/__artifacts__/thread.reset.json index f26abe59..34c8f007 100644 --- a/packages/harness/test/__artifacts__/thread.reset.json +++ b/packages/harness/test/__artifacts__/thread.reset.json @@ -2,22 +2,22 @@ "context": { "roles": [ "user", - "user", - "system" + "system", + "user" ], - "texts": "The conversation was started over. Carried over from before:\n\n\nkeep the login work\n | after the reset | " + "texts": "The conversation was started over. Carried over from before:\n\n\nkeep the login work\n | | after the reset" }, "events": [ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", - "3 session.prompt.promoted.1 T1", - "4 session.thread.system.changed.1 T1", + "3 session.thread.system.changed.1 T1", + "4 session.prompt.promoted.1 T1", "5 session.llm.started.1 T1", "6 session.llm.ended.1 T1", "7 session.turn.ended.1 T1", "9 session.prompt.admitted.1 T1", - "10 session.prompt.promoted.1 T1", - "11 session.thread.system.changed.1 T1", + "10 session.thread.system.changed.1 T1", + "11 session.prompt.promoted.1 T1", "12 session.llm.started.1 T1", "13 session.llm.ended.1 T1", "14 session.turn.ended.1 T1" @@ -31,12 +31,12 @@ } ], "entries": [ - "E1 T1 user seq=3 committed=3 committed", - "E2 T1 system seq=4 committed=4 committed", + "E1 T1 system seq=3 committed=3 committed", + "E2 T1 user seq=4 committed=4 committed", "E3 T1 assistant seq=5 committed=7 committed", "E4 T1 reset seq=8 committed=8 committed", - "E5 T1 user seq=10 committed=10 committed", - "E6 T1 system seq=11 committed=11 committed", + "E5 T1 system seq=10 committed=10 committed", + "E6 T1 user seq=11 committed=11 committed", "E7 T1 assistant seq=12 committed=14 committed" ], "documents": [ diff --git a/packages/harness/test/__artifacts__/thread.walkthrough.json b/packages/harness/test/__artifacts__/thread.walkthrough.json index fc5be405..e687f42c 100644 --- a/packages/harness/test/__artifacts__/thread.walkthrough.json +++ b/packages/harness/test/__artifacts__/thread.walkthrough.json @@ -3,8 +3,8 @@ "1 session.created.1 T1", "2 session.thread.configured.1 T1", "3 session.prompt.admitted.1 T1", - "4 session.prompt.promoted.1 T1", - "5 session.thread.system.changed.1 T1", + "4 session.thread.system.changed.1 T1", + "5 session.prompt.promoted.1 T1", "6 session.llm.started.1 T1", "7 session.llm.ended.1 T1", "8 session.thread.configured.1 T1", @@ -48,8 +48,8 @@ } ], "entries": [ - "E1 T1 user seq=4 committed=4 committed", - "E2 T1 system seq=5 committed=5 committed", + "E1 T1 system seq=4 committed=4 committed", + "E2 T1 user seq=5 committed=5 committed", "E3 T1 assistant seq=6 committed=10 committed", "E4 T1 assistant seq=11 committed=13 committed", "E5 T1 user seq=15 committed=15 committed", @@ -84,8 +84,8 @@ } ], "t2Context": [ - "user", "system", + "user", "assistant", "assistant", "user" diff --git a/packages/harness/test/__artifacts__/tools.read.vision.json b/packages/harness/test/__artifacts__/tools.read.vision.json index f62e6c80..be222844 100644 --- a/packages/harness/test/__artifacts__/tools.read.vision.json +++ b/packages/harness/test/__artifacts__/tools.read.vision.json @@ -2,29 +2,29 @@ { "model": "text-only", "messages": [ + { + "role": "system" + }, { "role": "user", "content": [ "read the image" ] - }, - { - "role": "system" } ] }, { "model": "text-only", "messages": [ + { + "role": "system" + }, { "role": "user", "content": [ "read the image" ] }, - { - "role": "system" - }, { "role": "assistant", "parts": [ @@ -48,15 +48,15 @@ { "model": "vision", "messages": [ + { + "role": "system" + }, { "role": "user", "content": [ "read the image" ] }, - { - "role": "system" - }, { "role": "assistant", "parts": [ diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.fork.json b/packages/harness/test/__artifacts__/tools.toolsearch.fork.json index c585146b..14e7e83b 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.fork.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.fork.json @@ -87,13 +87,7 @@ "entries": [ { "id": "E1", - "seq": 26, - "type": "user", - "state": "committed" - }, - { - "id": "E2", - "seq": 27, + "seq": 25, "type": "system", "state": "committed", "data": { @@ -131,15 +125,21 @@ "hash": "ec3094bdf14770d4650a88a2307556654cabf8623185fefb11702ba2e9a9c6e8" } }, + { + "id": "E2", + "seq": 26, + "type": "user", + "state": "committed" + }, { "id": "E3", - "seq": 28, + "seq": 27, "type": "assistant", "state": "committed" }, { "id": "E4", - "seq": 32, + "seq": 31, "type": "assistant", "state": "committed" } diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.load.json b/packages/harness/test/__artifacts__/tools.toolsearch.load.json index 330d91e8..a29ab1fc 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.load.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.load.json @@ -114,11 +114,11 @@ }, { "seq": 3, - "type": "session.prompt.promoted.1" + "type": "session.thread.system.changed.1" }, { "seq": 4, - "type": "session.thread.system.changed.1" + "type": "session.prompt.promoted.1" }, { "seq": 5, @@ -173,12 +173,6 @@ { "id": "E1", "seq": 3, - "type": "user", - "state": "committed" - }, - { - "id": "E2", - "seq": 4, "type": "system", "state": "committed", "data": { @@ -192,6 +186,12 @@ "hash": "2aedbc0fffd70a2b41a7ce78f786f9809d727be73bf31b672ac2d603d3817fbb" } }, + { + "id": "E2", + "seq": 4, + "type": "user", + "state": "committed" + }, { "id": "E3", "seq": 5, diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json b/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json index c8151453..06cf4391 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json @@ -60,12 +60,6 @@ { "id": "E1", "seq": 4, - "type": "user", - "state": "committed" - }, - { - "id": "E2", - "seq": 5, "type": "system", "state": "committed", "data": { @@ -83,6 +77,12 @@ "hash": "d6d66014ddda5da53e6e8e6520f0e6e1851a1a9c246d23802463da98342397a1" } }, + { + "id": "E2", + "seq": 5, + "type": "user", + "state": "committed" + }, { "id": "E3", "seq": 6, diff --git a/packages/harness/test/plugin.external.test.ts b/packages/harness/test/plugin.external.test.ts index bb13ceec..9c70695d 100644 --- a/packages/harness/test/plugin.external.test.ts +++ b/packages/harness/test/plugin.external.test.ts @@ -125,7 +125,7 @@ describe("third-party plugins", () => { expect(contexts[0]?.tools?.map((tool) => tool.name)).toEqual(["acme_echo"]); expect(prompts[0]).toContain("- acme_echo: Echo a value back"); - expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant", "assistant"]); + expect(path.map((item) => item.entry.type)).toEqual(["system", "user", "assistant", "assistant"]); const settled = JSON.parse(path.find((item) => item.entry.type === "assistant")?.parts[0]?.data ?? "{}"); expect(settled).toMatchObject({ status: "completed", diff --git a/packages/harness/test/plugin.host.test.ts b/packages/harness/test/plugin.host.test.ts index 5b293433..bd4a22bd 100644 --- a/packages/harness/test/plugin.host.test.ts +++ b/packages/harness/test/plugin.host.test.ts @@ -331,7 +331,7 @@ describe("plugin pipelines in the kernel loop", () => { ]); if (mode === "sequential") expect(trace.indexOf("a:done")).toBeLessThan(trace.indexOf("b:before")); const path = yield* session.history(); - expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant", "assistant"]); + expect(path.map((item) => item.entry.type)).toEqual(["system", "user", "assistant", "assistant"]); }).pipe( Effect.provide( Harness.layer({ diff --git a/packages/harness/test/prompt.changes.live.test.ts b/packages/harness/test/prompt.changes.live.test.ts index 396551a8..6b6aaf61 100644 --- a/packages/harness/test/prompt.changes.live.test.ts +++ b/packages/harness/test/prompt.changes.live.test.ts @@ -13,7 +13,8 @@ import "./utils/live.ts"; /** * A prompt section changed between two prompts reaches a real model. Codex takes system messages - * in place, so the second request still carries the first codeword and adds the change after it. + * in place, so the second request still carries the first codeword and adds the change after it, + * ahead of the prompt it came with. */ const Codeword = Section.define("codeword"); @@ -85,8 +86,12 @@ describe("prompt sections against live openai-codex", () => { answers: replies.map((reply) => ["alpha", "bravo"].find((word) => reply.includes(word))), promptChange: second.includes('Updated system prompt section \\"codeword\\"') ? "in place" : "folded", earlierSectionSent: second.includes("The codeword is alpha."), + changeBeforePrompt: + second.indexOf("The codeword is bravo.") >= 0 && + second.indexOf("The codeword is bravo.") < second.indexOf("What is the codeword now?"), }; expect(artifact.promptChange).toBe("in place"); + expect(artifact.changeBeforePrompt).toBe(true); await expect(`${JSON.stringify(artifact, null, "\t")}\n`).toMatchFileSnapshot( "./__artifacts__/prompt.changes.live.openai-codex.json", ); diff --git a/packages/harness/test/runner.loop.test.ts b/packages/harness/test/runner.loop.test.ts index a5c98be1..1c283f02 100644 --- a/packages/harness/test/runner.loop.test.ts +++ b/packages/harness/test/runner.loop.test.ts @@ -172,8 +172,8 @@ describe("runner loop — aikit input/output", () => { const bySeq = [...rows].sort((left, right) => (left.promotedSeq ?? 0) - (right.promotedSeq ?? 0)); expect(bySeq.map((row) => row.id)).toEqual(["msg_steer", "msg_follow"]); expect((yield* threads.line(threadId)).map((item) => item.entry.type)).toEqual([ - "user", "system", + "user", "assistant", "user", "assistant", @@ -182,8 +182,8 @@ describe("runner loop — aikit input/output", () => { // Every request is rebuilt from durable history. The second request sees // the first terminal assistant and the newly promoted follow-up. expect(contexts.map((context) => context.messages.map((message) => message.role))).toEqual([ - ["user", "system"], - ["user", "system", "assistant", "user"], + ["system", "user"], + ["system", "user", "assistant", "user"], ]); }), ); @@ -199,10 +199,10 @@ describe("runner loop — aikit input/output", () => { yield* execution.resume(threadId); yield* execution.resume(threadId); - expect(yield* delivered(threadId)).toEqual([{ id: "msg_once", promotedSeq: 3 }]); + expect(yield* delivered(threadId)).toEqual([{ id: "msg_once", promotedSeq: 4 }]); expect((yield* threads.line(threadId)).map((item) => item.entry.type)).toEqual([ - "user", "system", + "user", "assistant", ]); }), @@ -224,10 +224,10 @@ describe("runner loop — aikit input/output", () => { const path = yield* threads.line(threadId); expect(path.map((item) => item.entry.type)).toEqual([ + "system", "user", "user", "user", - "system", "assistant", "user", "assistant", @@ -356,8 +356,8 @@ describe("runner loop — tool continuation and lifecycle gate", () => { const path = yield* threads.line(threadId); expect(path.map((item) => [item.entry.type, item.entry.state])).toEqual([ - ["user", "committed"], ["system", "committed"], + ["user", "committed"], ["assistant", "committed"], ["assistant", "committed"], ]); @@ -512,7 +512,7 @@ describe("runner loop — tool interruption", () => { expect(interrupted.data.reason).toBe("user"); const path = yield* threads.line(threadId); - expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); + expect(path.map((item) => item.entry.type)).toEqual(["system", "user", "assistant"]); expect(path[2]!.entry.state).toBe("committed"); expect(path[2]!.parts.map((part) => Option.getOrThrow(part.status))).toEqual(["aborted", "aborted"]); for (const part of path[2]!.parts) { @@ -578,7 +578,7 @@ describe("runner loop — provider interruption", () => { expect(Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause)).toBe(true); const path = yield* threads.line(threadId); - expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); + expect(path.map((item) => item.entry.type)).toEqual(["system", "user", "assistant"]); expect(path[2]!.entry.state).toBe("aborted"); expect(JSON.parse(path[2]!.entry.data)).toMatchObject({ stopReason: "aborted" }); expect(path[2]!.parts).toEqual([]); @@ -635,7 +635,7 @@ describe("runner loop — provider failure", () => { } const path = yield* threads.line(threadId); - expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); + expect(path.map((item) => item.entry.type)).toEqual(["system", "user", "assistant"]); expect(path[2]!.entry.state).toBe("aborted"); expect(path[2]!.parts).toEqual([]); expect(JSON.parse(path[2]!.entry.data)).toMatchObject({ @@ -1559,8 +1559,8 @@ describe("runner loop — lifecycle hooks", () => { const threads = yield* Thread.Service; const path = yield* threads.line(threadId); expect(path.map((item) => [item.entry.type, item.entry.state])).toEqual([ - ["user", "committed"], ["system", "committed"], + ["user", "committed"], ["assistant", "committed"], ]); const tool = path[2]!.parts.find((part) => part.type === "toolCall"); @@ -1571,10 +1571,10 @@ describe("runner loop — lifecycle hooks", () => { // The next run answers the tool results first, then picks the queued follow-up up. yield* execution.resume(threadId); expect(contexts).toHaveLength(3); - expect(contexts[1]!.messages.map((message) => message.role)).toEqual(["user", "system", "assistant"]); + expect(contexts[1]!.messages.map((message) => message.role)).toEqual(["system", "user", "assistant"]); expect(contexts[2]!.messages.map((message) => message.role)).toEqual([ - "user", "system", + "user", "assistant", "assistant", "user", diff --git a/packages/harness/test/sdk.test.ts b/packages/harness/test/sdk.test.ts index 3edb0faa..5682259b 100644 --- a/packages/harness/test/sdk.test.ts +++ b/packages/harness/test/sdk.test.ts @@ -88,8 +88,8 @@ describe("Harness Effect SDK", () => { yield* session.run("second"); // The restarted runtime finds the prompt the first run declared, so declares nothing again. expect((yield* session.history()).map(({ entry }) => entry.type)).toEqual([ - "user", "system", + "user", "assistant", "user", "assistant", diff --git a/packages/harness/test/thread.test.ts b/packages/harness/test/thread.test.ts index 60b2383d..9d747dfd 100644 --- a/packages/harness/test/thread.test.ts +++ b/packages/harness/test/thread.test.ts @@ -307,8 +307,8 @@ describe("threads", () => { const line = yield* threads.line(main); expect(line.map(({ entry }) => entry.type)).toEqual([ - "user", "system", + "user", "assistant", "assistant", "user", @@ -360,8 +360,8 @@ describe("threads", () => { yield* run(sessionId, main, "first"); yield* run(sessionId, main, "second"); const line = yield* threads.line(main); - expect(line.map(({ entry }) => entry.type)).toEqual(["user", "system", "assistant", "user", "assistant"]); - const [e1, , e2, e3, e4] = line; + expect(line.map(({ entry }) => entry.type)).toEqual(["system", "user", "assistant", "user", "assistant"]); + const [, e1, e2, e3, e4] = line; // An inherited entry re-points the origin to the thread that wrote it. const t2 = ThreadSchema.ID.make((yield* threads.fork({ threadId: main, entryId: e4!.entry.id })).thread.id); @@ -422,7 +422,7 @@ describe("threads", () => { const execution = yield* RunnerExecution.Service; const { sessionId, main } = yield* createSession; yield* run(sessionId, main, "base"); - const [first] = yield* threads.line(main); + const first = (yield* threads.line(main)).find(({ entry }) => entry.type === "user"); const t2 = ThreadSchema.ID.make((yield* threads.fork({ threadId: main, entryId: first!.entry.id })).thread.id); const gates = yield* Effect.forEach([main, t2], (threadId, index) => @@ -453,14 +453,14 @@ describe("threads", () => { ); const result = { bothActive: true, main: yield* own(main), fork: yield* own(t2) }; expect(result.main).toEqual([ - "user committed", "system committed", + "user committed", "assistant committed", "user committed", "assistant committed", ]); - // The fork cuts before main's system entry, so its first request declares its own. - expect(result.fork).toEqual(["user committed", "system committed", "assistant committed"]); + // The fork inherits main's system entry, so its first request declares nothing again. + expect(result.fork).toEqual(["user committed", "assistant committed"]); // Seqs interleave between the two runs, so the artifact holds per-thread facts only. yield* artifact("thread.concurrent.json", result); }), @@ -487,7 +487,7 @@ describe("threads", () => { yield* run(sessionId, main, "after the reset"); const last = contexts.at(-1)!; // The reset drops the earlier system entry too, so the request declares the prompt again. - expect(last.roles).toEqual(["user", "user", "system"]); + expect(last.roles).toEqual(["user", "system", "user"]); expect(last.texts.includes("before the reset")).toBe(false); expect(last.texts.includes("keep the login work")).toBe(true); yield* artifact("thread.reset.json", { @@ -506,7 +506,7 @@ describe("threads", () => { const events = yield* Event.Service; const { sessionId, main } = yield* createSession; yield* run(sessionId, main, "base"); - const [first] = yield* threads.line(main); + const first = (yield* threads.line(main)).find(({ entry }) => entry.type === "user"); const t2 = ThreadSchema.ID.make((yield* threads.fork({ threadId: main, entryId: first!.entry.id })).thread.id); // What a process dying mid-turn leaves: a promoted prompt, an open draft, the run still marked. @@ -549,14 +549,14 @@ describe("threads", () => { const own = (yield* threads.line(t2)) .filter(({ entry }) => entry.threadId === t2) .map(({ entry }) => `${entry.type} ${entry.state}`); - expect(own).toEqual(["user committed", "assistant aborted", "system committed", "assistant committed"]); + expect(own).toEqual(["user committed", "assistant aborted", "assistant committed"]); const thread = Option.getOrThrow(yield* threads.get(t2)); expect(Option.getOrUndefined(thread.execution)).toBeUndefined(); expect(thread.restarts).toBe(0); // Only the thread that was cut off ran again, from its committed entries: the inherited // prompt and its own, without the aborted draft. expect(contexts.filter((one) => one.threadId === main)).toHaveLength(1); - expect(contexts.at(-1)!.roles).toEqual(["user", "user", "system"]); + expect(contexts.at(-1)!.roles).toEqual(["system", "user", "user"]); yield* artifact("thread.recovery.json", yield* snapshot(sessionId)); }), ); diff --git a/packages/harness/test/tools.toolsearch.test.ts b/packages/harness/test/tools.toolsearch.test.ts index 1ed32fcb..b060246b 100644 --- a/packages/harness/test/tools.toolsearch.test.ts +++ b/packages/harness/test/tools.toolsearch.test.ts @@ -463,8 +463,8 @@ describe("toolsearch", () => { const { sessionId, main } = yield* createSession; yield* run(sessionId, main, "Load the forecast tool", [call(search("weather forecast", 1)), reply]); const line = yield* threads.line(main); - expect(line.map(({ entry }) => entry.type)).toEqual(["user", "system", "assistant", "system", "assistant"]); - const [asked, , searched] = line; + expect(line.map(({ entry }) => entry.type)).toEqual(["system", "user", "assistant", "system", "assistant"]); + const [, asked, searched] = line; const before = ThreadSchema.ID.make( (yield* threads.fork({ threadId: main, entryId: asked!.entry.id })).thread.id, From 9550f166c45ca32e7881e30ea6ec0a9c5909db6d Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 31 Oct 2026 01:15:16 +0530 Subject: [PATCH 03/11] feat(aikit): send prompt changes in place on Anthropic and OpenAI Responses Anthropic (Opus 4.8/5/5.5, Sonnet/Haiku 5.5, Fable/Mythos 5/5.1) and direct OpenAI Responses models now take later system messages in place through the AI SDK adapters, so a prompt section change keeps the cached prefix. On Anthropic a later system message is held for the next assistant message, as the API takes one only after a user message; one ahead of the conversation joins the leading prompt, since a trailing system message's cache entry is never read once a reply follows it. Co-Authored-By: Claude Opus 5.5 --- packages/aikit/src/cli/modelgen.ts | 9 +++ packages/aikit/src/llm/stream.ts | 4 +- packages/aikit/src/llm/transform.ts | 38 ++++++++--- .../aikit/test/__artifacts__/modelgen.json | 44 +++++++++++- .../prompt.changes.live.anthropic.json | 10 +++ .../prompt.changes.live.openai-codex.json | 3 +- .../prompt.changes.live.openai.json | 10 +++ .../harness/test/prompt.changes.live.test.ts | 67 ++++++++++++------- 8 files changed, 146 insertions(+), 39 deletions(-) create mode 100644 packages/harness/test/__artifacts__/prompt.changes.live.anthropic.json create mode 100644 packages/harness/test/__artifacts__/prompt.changes.live.openai.json diff --git a/packages/aikit/src/cli/modelgen.ts b/packages/aikit/src/cli/modelgen.ts index 68fe348d..ed671278 100644 --- a/packages/aikit/src/cli/modelgen.ts +++ b/packages/aikit/src/cli/modelgen.ts @@ -353,6 +353,9 @@ function applyOpenAiMetadata(model: Model.Info, source: ModelsDevModel | undefin } // models.dev reports the input sub-limit as gpt-5-pro's output. if (id === "gpt-5-pro") model.maxTokens = OPENAI_MAX_TOKENS; + if (model.api?.method === Model.APIMethodEnum.responses && OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS.has(id)) { + mergeCompat(model, { supportsMidConvoSystemMessages: true }); + } } model.cost.serviceTierMultipliers = serviceTierMultipliers(model, source); @@ -391,6 +394,9 @@ function isAnthropicMessages(model: Model.Info): boolean { // The Messages API rejects `temperature` on these. const ANTHROPIC_NO_TEMPERATURE = /opus-4[-.][78]|opus-5|(sonnet|haiku)-5[-.]5/; +// These take system messages inside the conversation (pi). +const ANTHROPIC_MID_CONVO_SYSTEM_MESSAGES = + /^claude-(?:opus-(?:4[.-]8|5(?:[.-]5)?)|(?:sonnet|haiku)-5[.-]5|(?:fable|mythos)-5(?:[.-]1)?)(?:-\d{8})?$/; // Checked live against the Messages API: effort levels each adaptive family // accepts, the 5.5 and Fable models that reject `thinking: {type: "disabled"}`, and @@ -422,6 +428,9 @@ function applyAnthropicMetadata(model: Model.Info): void { mergeThinkingLevelMap(model, { off: null }); } if (ANTHROPIC_NO_TEMPERATURE.test(id)) mergeCompat(model, { supportsTemperature: false }); + if (model.provider.id === "anthropic" && ANTHROPIC_MID_CONVO_SYSTEM_MESSAGES.test(id)) { + mergeCompat(model, { supportsMidConvoSystemMessages: true }); + } } // diff --git a/packages/aikit/src/llm/stream.ts b/packages/aikit/src/llm/stream.ts index b3116ca0..cea730dc 100644 --- a/packages/aikit/src/llm/stream.ts +++ b/packages/aikit/src/llm/stream.ts @@ -433,7 +433,9 @@ export const stream: Protocol.StreamFunction>[0] = compact({ model: languageModel, // Our own Responses transport receives the system prompt among the messages. - system: isOwnResponses(model, runtimeOptions.method) ? undefined : leadingSystemPrompt(context), + system: isOwnResponses(model, runtimeOptions.method) + ? undefined + : leadingSystemPrompt(context, model, runtimeOptions.method), messages, // Prompt and tool changes sent in place travel as system messages. allowSystemInMessages: messages.some((message) => message.role === "system") || undefined, diff --git a/packages/aikit/src/llm/transform.ts b/packages/aikit/src/llm/transform.ts index bf161bc6..497390b0 100644 --- a/packages/aikit/src/llm/transform.ts +++ b/packages/aikit/src/llm/transform.ts @@ -485,16 +485,28 @@ export function transcriptToolSupport( } /** - * Whether prompt changes go in place as developer messages. Only our own Responses transport - * sends them; the AI SDK adapters take the current prompt as the leading system prompt. + * Whether prompt changes go in place as system messages. Our own Responses transport and the + * AI SDK's OpenAI Responses and Anthropic Messages adapters keep them where they stand; Chat + * Completions takes the current prompt as the leading system prompt. */ export function supportsMidConvoSystemMessages(model: Model.Info, method: Model.APIMethodEnum | undefined): boolean { - return isOwnResponses(model, method) && model.compat?.supportsMidConvoSystemMessages === true; + if (model.compat?.supportsMidConvoSystemMessages !== true) return false; + return isOwnResponses(model, method) || !isChatCompletions(model, resolveMethod(model, method)); } -/** The prompt the AI SDK adapters take as `system`: the current one, every change folded in. */ -export function leadingSystemPrompt(context: Message.Context): string | undefined { - return Transcript.getCurrentSystemPrompt(context) || undefined; +/** + * The prompt the AI SDK adapters take as `system`: the leading one when changes go in place, + * else the current one, every change folded in. + */ +export function leadingSystemPrompt( + context: Message.Context, + model: Model.Info, + method: Model.APIMethodEnum | undefined, +): string | undefined { + return ( + Transcript.resolveTranscriptPrompt(context, supportsMidConvoSystemMessages(model, method)).systemPrompt || + undefined + ); } /** A tool as the language model receives it, for declaring it inside the prompt. */ @@ -557,6 +569,12 @@ export function convertMessages( messages.push({ role: "system", content: sanitizeSurrogates(prompt.systemPrompt) }); } + // Anthropic takes a later system message only after a user message, and never between a tool + // call and its result: each is held for the next assistant message, or the end (pi). One ahead + // of the conversation joins the leading prompt instead, where it caches like the rest of it; a + // trailing system message's cache entry is never read once a reply follows it. + const anthropic = isAnthropicWire(model, method); + const held: ModelMessage[] = []; let systemIndex = 0; for (const msg of transformedMessages) { if (msg.role === "system") { @@ -571,17 +589,19 @@ export function convertMessages( model, method, ); - if (change) messages.push(change); + if (change) (anthropic && messages.some((one) => one.role !== "system") ? held : messages).push(change); continue; } if (msg.role === "user") { messages.push(...userContent(msg.parts)); continue; } - messages.push(...assistantMessages(msg, model, method)); + const converted = assistantMessages(msg, model, method); + if (converted.length > 0) messages.push(...held.splice(0)); + messages.push(...converted); } - return messages; + return [...messages, ...held]; } /** diff --git a/packages/aikit/test/__artifacts__/modelgen.json b/packages/aikit/test/__artifacts__/modelgen.json index 5bbf7420..9deb6d93 100644 --- a/packages/aikit/test/__artifacts__/modelgen.json +++ b/packages/aikit/test/__artifacts__/modelgen.json @@ -8,7 +8,8 @@ }, "baseUrl": "https://api.anthropic.com/v1", "compat": { - "forceAdaptiveThinking": true + "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true }, "contextWindow": 1000000, "cost": { @@ -50,7 +51,8 @@ }, "baseUrl": "https://api.anthropic.com/v1", "compat": { - "forceAdaptiveThinking": true + "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true }, "contextWindow": 1000000, "cost": { @@ -161,6 +163,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -367,6 +370,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -409,6 +413,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -451,6 +456,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -647,6 +653,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -1585,6 +1592,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.25, @@ -1641,6 +1651,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 400000, "cost": { "cacheRead": 0.075, @@ -1735,6 +1748,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 1050000, "cost": { "cacheRead": 0, @@ -1791,6 +1807,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.5, @@ -1903,6 +1922,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.02, @@ -1959,6 +1981,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.4, @@ -2015,6 +2040,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.2, @@ -2071,6 +2099,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 1, @@ -2128,6 +2159,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.01, @@ -2184,6 +2218,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.2, @@ -2240,6 +2277,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.1, diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.anthropic.json b/packages/harness/test/__artifacts__/prompt.changes.live.anthropic.json new file mode 100644 index 00000000..b96adf55 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.live.anthropic.json @@ -0,0 +1,10 @@ +{ + "answers": [ + "alpha", + "bravo" + ], + "promptChange": "in place", + "earlierSectionSent": true, + "change": "after", + "cached": true +} diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json b/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json index fe1f7638..ba834819 100644 --- a/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json +++ b/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json @@ -5,5 +5,6 @@ ], "promptChange": "in place", "earlierSectionSent": true, - "changeBeforePrompt": true + "change": "before", + "cached": true } diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.openai.json b/packages/harness/test/__artifacts__/prompt.changes.live.openai.json new file mode 100644 index 00000000..ba834819 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.live.openai.json @@ -0,0 +1,10 @@ +{ + "answers": [ + "alpha", + "bravo" + ], + "promptChange": "in place", + "earlierSectionSent": true, + "change": "before", + "cached": true +} diff --git a/packages/harness/test/prompt.changes.live.test.ts b/packages/harness/test/prompt.changes.live.test.ts index 6b6aaf61..d237c36f 100644 --- a/packages/harness/test/prompt.changes.live.test.ts +++ b/packages/harness/test/prompt.changes.live.test.ts @@ -12,30 +12,43 @@ import { withSettings } from "./fixtures/settings.ts"; import "./utils/live.ts"; /** - * A prompt section changed between two prompts reaches a real model. Codex takes system messages - * in place, so the second request still carries the first codeword and adds the change after it, - * ahead of the prompt it came with. + * A prompt section changed between two prompts reaches a real model. These models take system + * messages in place, so the second request still carries the first codeword and adds the change + * after it: ahead of the prompt it came with, or on Anthropic, which takes one only after a user + * message, right after that prompt. The second request reads the first one's sections from the cache. */ const Codeword = Section.define("codeword"); -const live = { provider: "openai-codex", id: "gpt-6-luna" } as const; +const Reference = Section.define("reference"); +/** A fixed section past every provider's minimum cacheable prefix. */ +const reference = Array.from({ length: 300 }, (_, i) => `Reference line ${i}: the archive holds item ${i * 7}.`).join( + "\n", +); +process.env.CODEWORK_CACHE_RETENTION ??= "short"; +const CASES = [ + { name: "openai-codex", provider: "openai-codex", id: "gpt-6-luna", env: "OPENAI_CODEX_API_KEY", change: "before" }, + { name: "openai", provider: "openai", id: "gpt-6-luna", env: "OPENAI_API_KEY", change: "before" }, + { name: "anthropic", provider: "anthropic", id: "claude-sonnet-5-5", env: "ANTHROPIC_API_KEY", change: "after" }, +] as const; -describe("prompt sections against live openai-codex", () => { - it.skipIf(!process.env.OPENAI_CODEX_API_KEY?.trim())( +describe.each(CASES)("prompt sections against live $name", ({ name, provider, id, env, change }) => { + it.skipIf(!process.env[env]?.trim())( "answers from a section changed between prompts, sent in place", { timeout: 300_000 }, () => withSettings(async ({ root, custom }) => { - await writeFile(join(custom, "settings.jsonc"), JSON.stringify({ model: live })); + await writeFile(join(custom, "settings.jsonc"), JSON.stringify({ model: { provider, id } })); const state = { word: "alpha" }; const codeword = define({ id: "acme.prompt.codeword", kind: "prompt", - setup: (ctx) => + setup: (ctx) => { + ctx.plugin.prompt.sections.append(Reference, reference); ctx.plugin.prompt.sections.append( Codeword, `The codeword is ${state.word}. Asked for the codeword, reply with it and nothing else.`, - ), + ); + }, }); const bodies: string[] = []; const onPayload: NonNullable = async (payload) => { @@ -43,7 +56,7 @@ describe("prompt sections against live openai-codex", () => { return payload; }; - const replies = await Effect.gen(function* () { + const { replies, cacheRead } = await Effect.gen(function* () { const handle = yield* Session.create({ directory: root }); const runtime = yield* SessionRuntime.Service; yield* runtime.update(handle.id, { onPayload }); @@ -51,17 +64,17 @@ describe("prompt sections against live openai-codex", () => { state.word = "bravo"; yield* handle.main.run("What is the codeword now?"); const messages = yield* Effect.forEach(yield* handle.main.history(), ContextCodec.decodeMessage); - return messages.flatMap((message) => - message.role === "assistant" - ? [ - message.parts - .flatMap((part) => (part.type === "text" ? [part.text] : [])) - .join("") - .trim() - .toLowerCase(), - ] - : [], - ); + const assistants = messages.flatMap((message) => (message.role === "assistant" ? [message] : [])); + return { + replies: assistants.map((message) => + message.parts + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join("") + .trim() + .toLowerCase(), + ), + cacheRead: assistants.at(-1)?.usage.cacheRead ?? 0, + }; }).pipe( Effect.provide( Harness.layer({ @@ -82,18 +95,20 @@ describe("prompt sections against live openai-codex", () => { expect(replies[0]).toContain("alpha"); expect(replies[1]).toContain("bravo"); const second = bodies[1]!; + const bravo = second.indexOf("The codeword is bravo."); const artifact = { answers: replies.map((reply) => ["alpha", "bravo"].find((word) => reply.includes(word))), promptChange: second.includes('Updated system prompt section \\"codeword\\"') ? "in place" : "folded", earlierSectionSent: second.includes("The codeword is alpha."), - changeBeforePrompt: - second.indexOf("The codeword is bravo.") >= 0 && - second.indexOf("The codeword is bravo.") < second.indexOf("What is the codeword now?"), + change: bravo < 0 ? "missing" : bravo < second.indexOf("What is the codeword now?") ? "before" : "after", + // The change kept the prefix: the second request reads the first one's sections from the cache. + cached: cacheRead > 0, }; expect(artifact.promptChange).toBe("in place"); - expect(artifact.changeBeforePrompt).toBe(true); + expect(artifact.change).toBe(change); + expect(artifact.cached).toBe(true); await expect(`${JSON.stringify(artifact, null, "\t")}\n`).toMatchFileSnapshot( - "./__artifacts__/prompt.changes.live.openai-codex.json", + `./__artifacts__/prompt.changes.live.${name}.json`, ); }), ); From 47afdf7e75439806a75b898cb8bf76fbdc9bd0a5 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 31 Oct 2026 01:42:08 +0530 Subject: [PATCH 04/11] refactor: tighten in-place prompt changes after review - Transcript.diffSections replaces the harness copy of the section diff. - Copilot Claude models take system messages in place, as direct Anthropic. - An Anthropic system message ahead of the conversation is never held, so it stays in the leading prompt where it caches. - An Anthropic wire test covers holding a change past a tool result and the folded fallback; the live test covers Copilot and checks the cache only where caching is explicit. Co-Authored-By: Claude Opus 5.5 --- packages/aikit/src/cli/modelgen.ts | 5 +- packages/aikit/src/llm/transform.ts | 14 +++-- packages/aikit/src/message/transcript.ts | 14 +++++ packages/aikit/test/transcript.test.ts | 62 ++++++++++++++++++- packages/harness/src/plugin/prompt/render.ts | 18 +----- packages/harness/src/runner/loop.ts | 3 +- .../prompt.changes.live.openai-codex.json | 3 +- .../prompt.changes.live.openai.json | 3 +- .../harness/test/prompt.changes.live.test.ts | 28 +++++++-- packages/harness/test/utils/live.ts | 21 ++++--- 10 files changed, 127 insertions(+), 44 deletions(-) diff --git a/packages/aikit/src/cli/modelgen.ts b/packages/aikit/src/cli/modelgen.ts index ed671278..3c645331 100644 --- a/packages/aikit/src/cli/modelgen.ts +++ b/packages/aikit/src/cli/modelgen.ts @@ -394,7 +394,7 @@ function isAnthropicMessages(model: Model.Info): boolean { // The Messages API rejects `temperature` on these. const ANTHROPIC_NO_TEMPERATURE = /opus-4[-.][78]|opus-5|(sonnet|haiku)-5[-.]5/; -// These take system messages inside the conversation (pi). +// These take system messages inside the conversation, verified live against the Messages API. const ANTHROPIC_MID_CONVO_SYSTEM_MESSAGES = /^claude-(?:opus-(?:4[.-]8|5(?:[.-]5)?)|(?:sonnet|haiku)-5[.-]5|(?:fable|mythos)-5(?:[.-]1)?)(?:-\d{8})?$/; @@ -710,6 +710,9 @@ export function githubCopilotBuiltInModels(provider: ModelsDevProvider | undefin if (method === Model.APIMethodEnum.responses && OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS.has(id)) { mergeCompat(info, { supportsMidConvoSystemMessages: true }); } + if (method === Model.APIMethodEnum.messages && ANTHROPIC_MID_CONVO_SYSTEM_MESSAGES.test(id)) { + mergeCompat(info, { supportsMidConvoSystemMessages: true }); + } if (method === Model.APIMethodEnum.responses && GITHUB_COPILOT_ADDITIONAL_TOOLS_MODEL_IDS.has(id)) { mergeCompat(info, { supportsAdditionalTools: true }); } diff --git a/packages/aikit/src/llm/transform.ts b/packages/aikit/src/llm/transform.ts index 497390b0..d1acf1d0 100644 --- a/packages/aikit/src/llm/transform.ts +++ b/packages/aikit/src/llm/transform.ts @@ -569,12 +569,15 @@ export function convertMessages( messages.push({ role: "system", content: sanitizeSurrogates(prompt.systemPrompt) }); } - // Anthropic takes a later system message only after a user message, and never between a tool - // call and its result: each is held for the next assistant message, or the end (pi). One ahead - // of the conversation joins the leading prompt instead, where it caches like the rest of it; a - // trailing system message's cache entry is never read once a reply follows it. + // Every system message goes as an update, the first declaration too: a section a model read + // whole does not yield to a later update of it, one it read as an update does. Anthropic takes + // a later system message only after a user message, and never between a tool call and its + // result: each is held for the next assistant message, or the end. One ahead of the + // conversation joins the leading prompt there, where it caches; a trailing system message's + // cache entry is never read once a reply follows it. const anthropic = isAnthropicWire(model, method); const held: ModelMessage[] = []; + let started = false; let systemIndex = 0; for (const msg of transformedMessages) { if (msg.role === "system") { @@ -589,9 +592,10 @@ export function convertMessages( model, method, ); - if (change) (anthropic && messages.some((one) => one.role !== "system") ? held : messages).push(change); + if (change) (anthropic && started ? held : messages).push(change); continue; } + started = true; if (msg.role === "user") { messages.push(...userContent(msg.parts)); continue; diff --git a/packages/aikit/src/message/transcript.ts b/packages/aikit/src/message/transcript.ts index 84b71d58..a604d6db 100644 --- a/packages/aikit/src/message/transcript.ts +++ b/packages/aikit/src/message/transcript.ts @@ -60,6 +60,20 @@ export function getCurrentSections(messages: readonly Message[]): Record>, + current: Readonly>, +): Record | undefined { + const patch: Record = {}; + for (const [name, text] of Object.entries(current)) if (previous[name] !== text) patch[name] = text; + for (const name of Object.keys(previous)) if (!Object.hasOwn(current, name)) patch[name] = null; + return Object.keys(patch).length === 0 ? undefined : patch; +} + /** The prompt after every transcript change in order: content accumulates, sections as {@link getCurrentSections}. */ export function getCurrentSystemPrompt(context: Pick): string { const content: string[] = [context.systemPrompt ?? ""]; diff --git a/packages/aikit/test/transcript.test.ts b/packages/aikit/test/transcript.test.ts index 13216232..e0804fb4 100644 --- a/packages/aikit/test/transcript.test.ts +++ b/packages/aikit/test/transcript.test.ts @@ -1,11 +1,12 @@ -/** Prompt and tool changes carried by transcript system messages, from the context down to the Responses wire. */ +/** Prompt and tool changes carried by transcript system messages, from the context down to the Responses and Anthropic wires. */ import Type from "typebox"; import { describe, expect, it } from "vite-plus/test"; import * as Message from "../src/message/message.ts"; import * as Transcript from "../src/message/transcript.ts"; import type * as Model from "../src/model/model.ts"; import { stream } from "../src/stream.ts"; -import { makeUserMessage } from "./utils/fixtures.ts"; +import { fakeAnthropic, textReply } from "./utils/anthropic.ts"; +import { makeAssistantMessage, makeCompletedToolCall, makeModel, makeUserMessage } from "./utils/fixtures.ts"; import { createOpenAICodexMockFetch, OPENAI_CODEX_TEST_API_KEY, @@ -363,3 +364,60 @@ describe("Transcript", () => { ).toEqual([changed, lateTool]); }); }); + +describe("transcript prompt changes on the Anthropic wire", () => { + type AnthropicBody = { + system?: Array<{ text: string }>; + messages: Array<{ role: string; content: Array<{ type: string; text?: string }> }>; + }; + const claude = (inPlace: boolean) => + makeModel({ npm: "@ai-sdk/anthropic", compat: { supportsMidConvoSystemMessages: inPlace } }); + const wire = async (model: Model.Info) => { + const server = fakeAnthropic(textReply); + await stream( + model, + { + systemPrompt: "base prompt", + messages: [ + system({ sections: { codeword: "alpha" } }), + makeUserMessage("first"), + makeAssistantMessage(model, { stopReason: "toolUse", parts: [makeCompletedToolCall("call-1")] }), + system({ sections: { codeword: "bravo" } }), + makeUserMessage("second"), + ], + }, + { apiKey: "test-key", factoryOptions: { fetch: server.fetch } }, + ).result(); + const body = server.body() as AnthropicBody; + return { + system: body.system?.map((part) => part.text), + messages: body.messages.map((message) => [ + message.role, + message.content.map((part) => part.text ?? part.type).join(" | "), + ]), + }; + }; + + it("joins the first declaration to the leading prompt and holds a later change past the tool result and prompt", async () => { + expect(await wire(claude(true))).toEqual({ + system: ["base prompt", 'Updated system prompt section "codeword":\n\nalpha'], + messages: [ + ["user", "first"], + ["assistant", "tool_use"], + ["user", "tool_result | second"], + ["system", 'Updated system prompt section "codeword":\n\nbravo'], + ], + }); + }); + + it("folds every change into the leading prompt on a model without mid-conversation system messages", async () => { + expect(await wire(claude(false))).toEqual({ + system: ["base prompt\n\nbravo"], + messages: [ + ["user", "first"], + ["assistant", "tool_use"], + ["user", "tool_result | second"], + ], + }); + }); +}); diff --git a/packages/harness/src/plugin/prompt/render.ts b/packages/harness/src/plugin/prompt/render.ts index 38957446..65e7af75 100644 --- a/packages/harness/src/plugin/prompt/render.ts +++ b/packages/harness/src/plugin/prompt/render.ts @@ -66,22 +66,8 @@ export const render = (input: Input): Rendered => { sections.cwd = tag("cwd", input.directory); for (const name of Array.from(written.keys())) push(name); - // A blank foundation would leave models that take system messages in place with no leading - // prompt, and their transport would promote a later section update in its stead. + // A blank foundation would leave the Responses transport without instructions, and it would + // promote the first system message in its stead. const foundation = input.prompt.foundation?.trim() ? input.prompt.foundation : fallback; return { foundation, sections }; }; - -/** - * The patch that brings the sections the transcript declares to `current`, `null` removing one; - * `undefined` when they match. - */ -export const diff = ( - previous: Readonly>, - current: Readonly>, -): Record | undefined => { - const patch: Record = {}; - for (const [name, text] of Object.entries(current)) if (previous[name] !== text) patch[name] = text; - for (const name of Object.keys(previous)) if (!Object.hasOwn(current, name)) patch[name] = null; - return Object.keys(patch).length === 0 ? undefined : patch; -}; diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index 3145d724..ecaa900c 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -8,7 +8,6 @@ import { type Message, Transcript } from "@codeworksh/aikit"; import { Cause, DateTime, Duration, Effect, Exit, Layer, Option, Schema } from "effect"; import { ContextCodec } from "../context/codec.ts"; -import { diff } from "../plugin/prompt/render.ts"; import { Context } from "../context/context.ts"; import { Event } from "../event/event.ts"; import { EventList } from "../event/list.ts"; @@ -290,7 +289,7 @@ export const layer = (options: Options = {}) => "none", ).requestTools; const tools = Transcript.getToolStateChanges(previous, snapshot.tools.offered); - const sections = diff(Transcript.getCurrentSections(messages), snapshot.prompt.sections); + const sections = Transcript.diffSections(Transcript.getCurrentSections(messages), snapshot.prompt.sections); if (sections === undefined && tools.toolsAdded.length === 0 && tools.toolsRemoved.length === 0) return []; const changes = { ...(sections === undefined ? {} : { sections }), ...tools }; const messageId = SessionMessageSchema.ID.create(); diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json b/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json index ba834819..e2115b80 100644 --- a/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json +++ b/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json @@ -5,6 +5,5 @@ ], "promptChange": "in place", "earlierSectionSent": true, - "change": "before", - "cached": true + "change": "before" } diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.openai.json b/packages/harness/test/__artifacts__/prompt.changes.live.openai.json index ba834819..e2115b80 100644 --- a/packages/harness/test/__artifacts__/prompt.changes.live.openai.json +++ b/packages/harness/test/__artifacts__/prompt.changes.live.openai.json @@ -5,6 +5,5 @@ ], "promptChange": "in place", "earlierSectionSent": true, - "change": "before", - "cached": true + "change": "before" } diff --git a/packages/harness/test/prompt.changes.live.test.ts b/packages/harness/test/prompt.changes.live.test.ts index d237c36f..6456e4f4 100644 --- a/packages/harness/test/prompt.changes.live.test.ts +++ b/packages/harness/test/prompt.changes.live.test.ts @@ -15,7 +15,8 @@ import "./utils/live.ts"; * A prompt section changed between two prompts reaches a real model. These models take system * messages in place, so the second request still carries the first codeword and adds the change * after it: ahead of the prompt it came with, or on Anthropic, which takes one only after a user - * message, right after that prompt. The second request reads the first one's sections from the cache. + * message, right after that prompt. Where caching is explicit (Anthropic), the second request + * reads the first one's sections from the cache; OpenAI's is best effort, so it is not checked there. */ const Codeword = Section.define("codeword"); @@ -28,10 +29,27 @@ process.env.CODEWORK_CACHE_RETENTION ??= "short"; const CASES = [ { name: "openai-codex", provider: "openai-codex", id: "gpt-6-luna", env: "OPENAI_CODEX_API_KEY", change: "before" }, { name: "openai", provider: "openai", id: "gpt-6-luna", env: "OPENAI_API_KEY", change: "before" }, - { name: "anthropic", provider: "anthropic", id: "claude-sonnet-5-5", env: "ANTHROPIC_API_KEY", change: "after" }, + { + name: "anthropic", + provider: "anthropic", + id: "claude-sonnet-5-5", + env: "ANTHROPIC_API_KEY", + change: "after", + cache: true, + }, + { + name: "github-copilot", + provider: "github-copilot", + id: "claude-sonnet-5.5", + env: "COPILOT_GITHUB_TOKEN", + change: "after", + cache: true, + }, ] as const; -describe.each(CASES)("prompt sections against live $name", ({ name, provider, id, env, change }) => { +describe.each(CASES)("prompt sections against live $name", (c) => { + const { name, provider, id, env, change } = c; + const cache = "cache" in c && c.cache; it.skipIf(!process.env[env]?.trim())( "answers from a section changed between prompts, sent in place", { timeout: 300_000 }, @@ -102,11 +120,11 @@ describe.each(CASES)("prompt sections against live $name", ({ name, provider, id earlierSectionSent: second.includes("The codeword is alpha."), change: bravo < 0 ? "missing" : bravo < second.indexOf("What is the codeword now?") ? "before" : "after", // The change kept the prefix: the second request reads the first one's sections from the cache. - cached: cacheRead > 0, + ...(cache ? { cached: cacheRead > 0 } : {}), }; expect(artifact.promptChange).toBe("in place"); expect(artifact.change).toBe(change); - expect(artifact.cached).toBe(true); + if (cache) expect(artifact.cached).toBe(true); await expect(`${JSON.stringify(artifact, null, "\t")}\n`).toMatchFileSnapshot( `./__artifacts__/prompt.changes.live.${name}.json`, ); diff --git a/packages/harness/test/utils/live.ts b/packages/harness/test/utils/live.ts index 06787483..7af06c76 100644 --- a/packages/harness/test/utils/live.ts +++ b/packages/harness/test/utils/live.ts @@ -5,22 +5,25 @@ import { join } from "node:path"; import "./env.ts"; /** - * The Codex login's current access token, for `OPENAI_CODEX_API_KEY`. The login file itself is - * never used by a test: a refresh would rotate the refresh token and sign the developer out. An - * expired token skips the Codex cases rather than refreshing. + * A login's current access token, for its key variable. The login file itself is never used by a + * test: a refresh would rotate the refresh token and sign the developer out. An expired token + * skips the provider's cases rather than refreshing. */ -const codexToken = (): string | undefined => { +const loginToken = (provider: string): string | undefined => { try { - const login = JSON.parse(readFileSync(join(homedir(), ".codework", "aikit", "auth.json"), "utf8"))[ - "openai-codex" - ]; + const login = JSON.parse(readFileSync(join(homedir(), ".codework", "aikit", "auth.json"), "utf8"))[provider]; return typeof login?.access === "string" && login.expires > Date.now() + 10 * 60_000 ? login.access : undefined; } catch { return undefined; } }; -const codex = process.env.OPENAI_CODEX_API_KEY ?? codexToken(); -if (codex !== undefined) process.env.OPENAI_CODEX_API_KEY = codex; +for (const [env, provider] of [ + ["OPENAI_CODEX_API_KEY", "openai-codex"], + ["COPILOT_GITHUB_TOKEN", "github-copilot"], +] as const) { + const token = process.env[env] ?? loginToken(provider); + if (token !== undefined) process.env[env] = token; +} /** Every live case runs against each of these; `env` names the key that enables it. */ export const LIVE = [ From e8feb99697a477d462181d886e457ae5e03384cd Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 31 Oct 2026 01:42:46 +0530 Subject: [PATCH 05/11] chore(aikit): refresh baseline catalog Co-Authored-By: Claude Opus 5.5 --- packages/aikit/src/model/baseline.json | 1363 +++++++++++------------- 1 file changed, 650 insertions(+), 713 deletions(-) diff --git a/packages/aikit/src/model/baseline.json b/packages/aikit/src/model/baseline.json index d925d91f..916087af 100644 --- a/packages/aikit/src/model/baseline.json +++ b/packages/aikit/src/model/baseline.json @@ -8,7 +8,8 @@ }, "baseUrl": "https://api.anthropic.com/v1", "compat": { - "forceAdaptiveThinking": true + "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true }, "contextWindow": 1000000, "cost": { @@ -50,7 +51,8 @@ }, "baseUrl": "https://api.anthropic.com/v1", "compat": { - "forceAdaptiveThinking": true + "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true }, "contextWindow": 1000000, "cost": { @@ -161,6 +163,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -367,6 +370,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -409,6 +413,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -451,6 +456,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -647,6 +653,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, "contextWindow": 1000000, @@ -1348,53 +1355,6 @@ "xhigh": "xhigh" } }, - "gpt-5.2-chat-latest": { - "api": { - "id": "gpt-5.2-chat-latest", - "method": "responses", - "url": "https://api.openai.com/v1" - }, - "baseUrl": "https://api.openai.com/v1", - "contextWindow": 128000, - "cost": { - "cacheRead": 0.175, - "cacheWrite": 0, - "input": 1.75, - "output": 14, - "serviceTierMultipliers": { - "flex": 0.5, - "priority": 2 - } - }, - "id": "gpt-5.2-chat-latest", - "input": [ - "text", - "image" - ], - "maxTokens": 16384, - "name": "GPT-5.2 Chat", - "npm": "@ai-sdk/openai", - "protocol": "openai", - "provider": { - "env": [ - "OPENAI_API_KEY" - ], - "id": "openai", - "name": "OpenAI", - "source": "api" - }, - "providerOptionsKey": "openai", - "reasoning": true, - "thinkingLevelMap": { - "high": null, - "low": null, - "max": null, - "medium": "medium", - "minimal": null, - "off": null, - "xhigh": "xhigh" - } - }, "gpt-5.2-pro": { "api": { "id": "gpt-5.2-pro", @@ -1442,48 +1402,6 @@ "xhigh": "xhigh" } }, - "gpt-5.3-chat-latest": { - "api": { - "id": "gpt-5.3-chat-latest", - "method": "responses", - "url": "https://api.openai.com/v1" - }, - "baseUrl": "https://api.openai.com/v1", - "contextWindow": 128000, - "cost": { - "cacheRead": 0.175, - "cacheWrite": 0, - "input": 1.75, - "output": 14, - "serviceTierMultipliers": { - "flex": 0.5, - "priority": 2 - } - }, - "id": "gpt-5.3-chat-latest", - "input": [ - "text", - "image" - ], - "maxTokens": 16384, - "name": "GPT-5.3 Chat (latest)", - "npm": "@ai-sdk/openai", - "protocol": "openai", - "provider": { - "env": [ - "OPENAI_API_KEY" - ], - "id": "openai", - "name": "OpenAI", - "source": "api" - }, - "providerOptionsKey": "openai", - "reasoning": false, - "thinkingLevelMap": { - "off": null, - "xhigh": "xhigh" - } - }, "gpt-5.3-codex": { "api": { "id": "gpt-5.3-codex", @@ -1585,6 +1503,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.25, @@ -1641,6 +1562,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 400000, "cost": { "cacheRead": 0.075, @@ -1735,6 +1659,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 1050000, "cost": { "cacheRead": 0, @@ -1791,6 +1718,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.5, @@ -1903,6 +1833,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.02, @@ -1959,6 +1892,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.4, @@ -2015,6 +1951,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.2, @@ -2071,6 +2010,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 1, @@ -2128,6 +2070,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.01, @@ -2184,6 +2129,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.2, @@ -2240,6 +2188,9 @@ "url": "https://api.openai.com/v1" }, "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, "contextWindow": 272000, "cost": { "cacheRead": 0.1, @@ -2290,6 +2241,65 @@ "xhigh": "xhigh" } }, + "gpt-6.1-terra": { + "api": { + "id": "gpt-6.1-terra", + "method": "responses", + "url": "https://api.openai.com/v1" + }, + "baseUrl": "https://api.openai.com/v1", + "compat": { + "supportsMidConvoSystemMessages": true + }, + "contextWindow": 1050000, + "cost": { + "cacheRead": 0.05, + "cacheWrite": 1.25, + "input": 1, + "output": 5, + "serviceTierMultipliers": { + "flex": 0.5, + "priority": 2 + }, + "tiers": [ + { + "cacheRead": 0.1, + "cacheWrite": 2.5, + "input": 2, + "inputTokensAbove": 272000, + "output": 7.5 + } + ] + }, + "id": "gpt-6.1-terra", + "input": [ + "text", + "image" + ], + "maxTokens": 128000, + "name": "GPT-6.1 Terra", + "npm": "@ai-sdk/openai", + "protocol": "openai", + "provider": { + "env": [ + "OPENAI_API_KEY" + ], + "id": "openai", + "name": "OpenAI", + "source": "api" + }, + "providerOptionsKey": "openai", + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": "max", + "medium": "medium", + "minimal": null, + "off": "none", + "xhigh": "xhigh" + } + }, "gpt-daybreak-blue-latest": { "api": { "id": "gpt-daybreak-blue-latest", @@ -3711,42 +3721,6 @@ "xhigh": "xhigh" } }, - "arcee-ai/trinity-large-thinking": { - "api": { - "id": "arcee-ai/trinity-large-thinking", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 262144, - "cost": { - "cacheRead": 0.06, - "cacheWrite": 0, - "input": 0.25, - "output": 0.8 - }, - "id": "arcee-ai/trinity-large-thinking", - "input": [ - "text" - ], - "maxTokens": 80000, - "name": "Arcee AI: Trinity Large Thinking", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "off": null - } - }, "bytedance-seed/seed-1.6": { "api": { "id": "bytedance-seed/seed-1.6", @@ -4163,18 +4137,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 163840, + "contextWindow": 128000, "cost": { "cacheRead": 0, "cacheWrite": 0, - "input": 0.32, - "output": 0.89 + "input": 0.2574, + "output": 1.0287 }, "id": "deepseek/deepseek-chat", "input": [ "text" ], - "maxTokens": 16384, + "maxTokens": 16000, "name": "DeepSeek: DeepSeek V3", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -4240,7 +4214,7 @@ "input": [ "text" ], - "maxTokens": 32768, + "maxTokens": 147456, "name": "DeepSeek: DeepSeek V3.1", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -4255,42 +4229,6 @@ "providerOptionsKey": "openrouter", "reasoning": true }, - "deepseek/deepseek-r1": { - "api": { - "id": "deepseek/deepseek-r1", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 64000, - "cost": { - "cacheRead": 0, - "cacheWrite": 0, - "input": 0.7, - "output": 2.5 - }, - "id": "deepseek/deepseek-r1", - "input": [ - "text" - ], - "maxTokens": 16000, - "name": "DeepSeek: R1", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "off": null - } - }, "deepseek/deepseek-r1-0528": { "api": { "id": "deepseek/deepseek-r1-0528", @@ -4367,18 +4305,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 131072, + "contextWindow": 163840, "cost": { - "cacheRead": 0.028, + "cacheRead": 0.13, "cacheWrite": 0, - "input": 0.28, - "output": 0.42 + "input": 0.26, + "output": 0.38 }, "id": "deepseek/deepseek-v3.2", "input": [ "text" ], - "maxTokens": 65536, + "maxTokens": 147456, "name": "DeepSeek: DeepSeek V3.2", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -4477,10 +4415,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.042, + "cacheRead": 0.0238, "cacheWrite": 0, - "input": 0.15, - "output": 0.15 + "input": 0.119, + "output": 0.238 }, "id": "deepseek/deepseek-v4-flash-0731", "input": [ @@ -4560,18 +4498,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 1024000, + "contextWindow": 1048576, "cost": { - "cacheRead": 0.079605, + "cacheRead": 0.135, "cacheWrite": 0, - "input": 0.95526, - "output": 1.91052 + "input": 1.6, + "output": 3.2 }, "id": "deepseek/deepseek-v4-pro", "input": [ "text" ], - "maxTokens": 384000, + "maxTokens": 393216, "name": "DeepSeek: DeepSeek V4 Pro 0423", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -4602,7 +4540,7 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 1024000, + "contextWindow": 1048576, "cost": { "cacheRead": 0.022, "cacheWrite": 0, @@ -4613,7 +4551,7 @@ "input": [ "text" ], - "maxTokens": 384000, + "maxTokens": 393216, "name": "DeepSeek: DeepSeek V4 Pro 0813", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -4646,10 +4584,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.0016, + "cacheRead": 0.006, "cacheWrite": 0, - "input": 0.0464, - "output": 0.08 + "input": 0.045, + "output": 0.5 }, "id": "deepseek/deepseek-v4.1-flash", "input": [ @@ -5781,7 +5719,7 @@ "cacheRead": 0.015, "cacheWrite": 0, "input": 0.06, - "output": 0.25 + "output": 0.15 }, "id": "ibm-granite/granite-4.2-8b", "input": [ @@ -5895,26 +5833,68 @@ "xhigh": null } }, - "inclusionai/ling-3.0-flash": { + "inception/mercury-3": { "api": { - "id": "inclusionai/ling-3.0-flash", + "id": "inception/mercury-3", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 262144, + "contextWindow": 1000000, "cost": { - "cacheRead": 0.0042, + "cacheRead": 0.01, "cacheWrite": 0, - "input": 0.021, - "output": 0.063 + "input": 0.1, + "output": 0.35 }, - "id": "inclusionai/ling-3.0-flash", + "id": "inception/mercury-3", "input": [ "text" ], - "maxTokens": 32768, - "name": "inclusionAI: Ling 3.0 Flash", + "maxTokens": 131072, + "name": "Inception: Mercury 3", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": null, + "medium": "medium", + "minimal": null, + "off": "none", + "xhigh": null + } + }, + "inclusionai/ling-3.0-flash": { + "api": { + "id": "inclusionai/ling-3.0-flash", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 262144, + "cost": { + "cacheRead": 0.0042, + "cacheWrite": 0, + "input": 0.021, + "output": 0.063 + }, + "id": "inclusionai/ling-3.0-flash", + "input": [ + "text" + ], + "maxTokens": 32768, + "name": "inclusionAI: Ling 3.0 Flash", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", "provider": { @@ -5969,7 +5949,7 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 262144, + "contextWindow": 1000000, "cost": { "cacheRead": 0.015, "cacheWrite": 0, @@ -5980,7 +5960,7 @@ "input": [ "text" ], - "maxTokens": 235929, + "maxTokens": 262144, "name": "inclusionAI: Ling 3.1 Flash", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -5995,6 +5975,42 @@ "providerOptionsKey": "openrouter", "reasoning": true }, + "liquid/lfm-2.5-2.6b": { + "api": { + "id": "liquid/lfm-2.5-2.6b", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 128000, + "cost": { + "cacheRead": 0.04, + "cacheWrite": 0, + "input": 0.35, + "output": 0.5 + }, + "id": "liquid/lfm-2.5-2.6b", + "input": [ + "text" + ], + "maxTokens": 8192, + "name": "LiquidAI: LFM2.5-2.6B", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "off": null + } + }, "liquid/lfm-2.5-2.6b:free": { "api": { "id": "liquid/lfm-2.5-2.6b:free", @@ -6544,7 +6560,7 @@ "text", "image" ], - "maxTokens": 943718, + "maxTokens": 131072, "name": "Meta: Muse Spark Open", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -6723,18 +6739,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 198000, + "contextWindow": 204800, "cost": { "cacheRead": 0.03, "cacheWrite": 0, - "input": 0.27, - "output": 0.95 + "input": 0.3, + "output": 1.2 }, "id": "minimax/minimax-m2.5", "input": [ "text" ], - "maxTokens": 32768, + "maxTokens": 131072, "name": "MiniMax: MiniMax M2.5", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -6822,6 +6838,49 @@ "providerOptionsKey": "openrouter", "reasoning": true }, + "minimax/minimax-m3.1-flash": { + "api": { + "id": "minimax/minimax-m3.1-flash", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 1048576, + "cost": { + "cacheRead": 0.0075, + "cacheWrite": 0, + "input": 0.075, + "output": 0.3 + }, + "id": "minimax/minimax-m3.1-flash", + "input": [ + "text", + "image" + ], + "maxTokens": 524288, + "name": "MiniMax: MiniMax M3.1 Flash", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": "max", + "medium": "medium", + "minimal": null, + "off": null, + "xhigh": "xhigh" + } + }, "mistralai/codestral-2508": { "api": { "id": "mistralai/codestral-2508", @@ -7101,8 +7160,8 @@ "cost": { "cacheRead": 0.07, "cacheWrite": 0, - "input": 0.68, - "output": 2.09 + "input": 0.7, + "output": 2.1 }, "id": "mistralai/mistral-large-4-0", "input": [ @@ -7394,12 +7453,12 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 256000, + "contextWindow": 128000, "cost": { "cacheRead": 0, "cacheWrite": 0, - "input": 0.09375, - "output": 0.25 + "input": 0.075, + "output": 0.2 }, "id": "mistralai/mistral-small-3.2-24b-instruct", "input": [ @@ -7666,10 +7725,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0.18, + "cacheRead": 0.15, "cacheWrite": 0, - "input": 0.6712, - "output": 3.35 + "input": 0.6, + "output": 3.5 }, "id": "moonshotai/kimi-k2.7-code", "input": [ @@ -7703,10 +7762,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.21, + "cacheRead": 0.182, "cacheWrite": 0, - "input": 0.239, - "output": 11.936 + "input": 0.192, + "output": 15 }, "id": "moonshotai/kimi-k3", "input": [ @@ -7737,6 +7796,49 @@ "xhigh": null } }, + "moonshotai/kimi-k3.1-pro": { + "api": { + "id": "moonshotai/kimi-k3.1-pro", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 1048576, + "cost": { + "cacheRead": 0.2, + "cacheWrite": 0, + "input": 2, + "output": 10 + }, + "id": "moonshotai/kimi-k3.1-pro", + "input": [ + "text", + "image" + ], + "maxTokens": 943718, + "name": "MoonshotAI: Kimi K3.1 Pro", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": "max", + "medium": null, + "minimal": null, + "off": "none", + "xhigh": null + } + }, "nex-agi/nex-n2.5-pro": { "api": { "id": "nex-agi/nex-n2.5-pro", @@ -7789,7 +7891,7 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0, + "cacheRead": 0.012, "cacheWrite": 0, "input": 0.06, "output": 0.24 @@ -7854,18 +7956,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 262144, + "contextWindow": 1048576, "cost": { - "cacheRead": 0, + "cacheRead": 0.15, "cacheWrite": 0, - "input": 0.08, - "output": 0.45 + "input": 0.3, + "output": 2.4 }, "id": "nvidia/nemotron-3-super-120b-a12b", "input": [ "text" ], - "maxTokens": 235929, + "maxTokens": 262144, "name": "NVIDIA: Nemotron 3 Super", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -7889,48 +7991,6 @@ "xhigh": null } }, - "nvidia/nemotron-3-super-120b-a12b:free": { - "api": { - "id": "nvidia/nemotron-3-super-120b-a12b:free", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 262144, - "cost": { - "cacheRead": 0, - "cacheWrite": 0, - "input": 0, - "output": 0 - }, - "id": "nvidia/nemotron-3-super-120b-a12b:free", - "input": [ - "text" - ], - "maxTokens": 235929, - "name": "NVIDIA: Nemotron 3 Super (free)", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "high": null, - "low": "low", - "max": null, - "medium": "medium", - "minimal": null, - "off": "none", - "xhigh": null - } - }, "nvidia/nemotron-3-ultra-550b-a55b": { "api": { "id": "nvidia/nemotron-3-ultra-550b-a55b", @@ -8090,10 +8150,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0.08, + "cacheRead": 0.02, "cacheWrite": 0, - "input": 0.1, - "output": 0.4 + "input": 0.15, + "output": 0.5 }, "id": "nvidia/nemotron-3.5-super-vl", "input": [ @@ -8149,26 +8209,26 @@ "providerOptionsKey": "openrouter", "reasoning": true }, - "openai/gpt-3.5-turbo": { + "openai/gpt-3.5-turbo-0613": { "api": { - "id": "openai/gpt-3.5-turbo", + "id": "openai/gpt-3.5-turbo-0613", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 16385, + "contextWindow": 4095, "cost": { "cacheRead": 0, "cacheWrite": 0, - "input": 0.5, - "output": 1.5 + "input": 1, + "output": 2 }, - "id": "openai/gpt-3.5-turbo", + "id": "openai/gpt-3.5-turbo-0613", "input": [ "text" ], - "maxTokens": 4096, - "name": "OpenAI: GPT-3.5 Turbo", + "maxTokens": 3685, + "name": "OpenAI: GPT-3.5 Turbo (older v0613)", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", "provider": { @@ -8182,54 +8242,21 @@ "providerOptionsKey": "openrouter", "reasoning": false }, - "openai/gpt-3.5-turbo-0613": { + "openai/gpt-3.5-turbo-16k": { "api": { - "id": "openai/gpt-3.5-turbo-0613", + "id": "openai/gpt-3.5-turbo-16k", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 4095, + "contextWindow": 16385, "cost": { "cacheRead": 0, "cacheWrite": 0, - "input": 1, - "output": 2 + "input": 3, + "output": 4 }, - "id": "openai/gpt-3.5-turbo-0613", - "input": [ - "text" - ], - "maxTokens": 3685, - "name": "OpenAI: GPT-3.5 Turbo (older v0613)", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": false - }, - "openai/gpt-3.5-turbo-16k": { - "api": { - "id": "openai/gpt-3.5-turbo-16k", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 16385, - "cost": { - "cacheRead": 0, - "cacheWrite": 0, - "input": 3, - "output": 4 - }, - "id": "openai/gpt-3.5-turbo-16k", + "id": "openai/gpt-3.5-turbo-16k", "input": [ "text" ], @@ -8281,40 +8308,6 @@ "providerOptionsKey": "openrouter", "reasoning": false }, - "openai/gpt-4-turbo": { - "api": { - "id": "openai/gpt-4-turbo", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 128000, - "cost": { - "cacheRead": 0, - "cacheWrite": 0, - "input": 10, - "output": 30 - }, - "id": "openai/gpt-4-turbo", - "input": [ - "text", - "image" - ], - "maxTokens": 4096, - "name": "OpenAI: GPT-4 Turbo", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": false - }, "openai/gpt-4.1": { "api": { "id": "openai/gpt-4.1", @@ -8392,7 +8385,7 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1047576, "cost": { - "cacheRead": 0.025, + "cacheRead": 0.03, "cacheWrite": 0, "input": 0.1, "output": 0.4 @@ -8402,7 +8395,7 @@ "image", "text" ], - "maxTokens": 32768, + "maxTokens": 942818, "name": "OpenAI: GPT-4.1 Nano", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -10193,6 +10186,110 @@ "xhigh": "xhigh" } }, + "openai/gpt-6.1-terra": { + "api": { + "id": "openai/gpt-6.1-terra", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 1050000, + "cost": { + "cacheRead": 0.05, + "cacheWrite": 1.25, + "input": 1, + "output": 5, + "tiers": [ + { + "cacheRead": 0.1, + "cacheWrite": 2.5, + "input": 2, + "inputTokensAbove": 272000, + "output": 7.5 + } + ] + }, + "id": "openai/gpt-6.1-terra", + "input": [ + "image", + "text" + ], + "maxTokens": 128000, + "name": "OpenAI: GPT-6.1 Terra", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": "max", + "medium": "medium", + "minimal": null, + "off": "none", + "xhigh": "xhigh" + } + }, + "openai/gpt-6.1-terra-pro": { + "api": { + "id": "openai/gpt-6.1-terra-pro", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 1050000, + "cost": { + "cacheRead": 0.05, + "cacheWrite": 1.25, + "input": 1, + "output": 5, + "tiers": [ + { + "cacheRead": 0.1, + "cacheWrite": 2.5, + "input": 2, + "inputTokensAbove": 272000, + "output": 7.5 + } + ] + }, + "id": "openai/gpt-6.1-terra-pro", + "input": [ + "image", + "text" + ], + "maxTokens": 128000, + "name": "OpenAI: GPT-6.1 Terra Pro", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": "max", + "medium": "medium", + "minimal": null, + "off": null, + "xhigh": "xhigh" + } + }, "openai/gpt-audio": { "api": { "id": "openai/gpt-audio", @@ -10413,43 +10510,6 @@ "off": null } }, - "openai/o1": { - "api": { - "id": "openai/o1", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, - "cost": { - "cacheRead": 7.5, - "cacheWrite": 0, - "input": 15, - "output": 60 - }, - "id": "openai/o1", - "input": [ - "text", - "image" - ], - "maxTokens": 100000, - "name": "OpenAI: o1", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "off": null - } - }, "openai/o3": { "api": { "id": "openai/o3", @@ -10487,84 +10547,6 @@ "off": null } }, - "openai/o3-mini": { - "api": { - "id": "openai/o3-mini", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, - "cost": { - "cacheRead": 0.55, - "cacheWrite": 0, - "input": 1.1, - "output": 4.4 - }, - "id": "openai/o3-mini", - "input": [ - "text" - ], - "maxTokens": 100000, - "name": "OpenAI: o3 Mini", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "off": null - } - }, - "openai/o3-mini-high": { - "api": { - "id": "openai/o3-mini-high", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, - "cost": { - "cacheRead": 0.55, - "cacheWrite": 0, - "input": 1.1, - "output": 4.4 - }, - "id": "openai/o3-mini-high", - "input": [ - "text" - ], - "maxTokens": 100000, - "name": "OpenAI: o3 Mini High", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "high": "high", - "low": null, - "max": null, - "medium": null, - "minimal": null, - "off": null, - "xhigh": null - } - }, "openai/o3-pro": { "api": { "id": "openai/o3-pro", @@ -10602,64 +10584,27 @@ "off": null } }, - "openai/o4-mini": { - "api": { - "id": "openai/o4-mini", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, - "cost": { - "cacheRead": 0.275, - "cacheWrite": 0, - "input": 1.1, - "output": 4.4 - }, - "id": "openai/o4-mini", - "input": [ - "image", - "text" - ], - "maxTokens": 100000, - "name": "OpenAI: o4 Mini", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "off": null - } - }, - "openai/o4-mini-high": { + "openrouter/auto": { "api": { - "id": "openai/o4-mini-high", + "id": "openrouter/auto", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, + "contextWindow": 2000000, "cost": { - "cacheRead": 0.275, + "cacheRead": 0, "cacheWrite": 0, - "input": 1.1, - "output": 4.4 + "input": 0, + "output": 0 }, - "id": "openai/o4-mini-high", + "id": "openrouter/auto", "input": [ - "image", - "text" + "text", + "image" ], - "maxTokens": 100000, - "name": "OpenAI: o4 Mini High", + "maxTokens": 4096, + "name": "Auto Router", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", "provider": { @@ -10671,20 +10616,11 @@ "source": "api" }, "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "high": "high", - "low": null, - "max": null, - "medium": null, - "minimal": null, - "off": null, - "xhigh": null - } + "reasoning": true }, - "openrouter/auto": { + "openrouter/auto-beta": { "api": { - "id": "openrouter/auto", + "id": "openrouter/auto-beta", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, @@ -10696,13 +10632,13 @@ "input": 0, "output": 0 }, - "id": "openrouter/auto", + "id": "openrouter/auto-beta", "input": [ "text", "image" ], - "maxTokens": 4096, - "name": "Auto Router", + "maxTokens": 65536, + "name": "Auto Router (Beta)", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", "provider": { @@ -10716,27 +10652,27 @@ "providerOptionsKey": "openrouter", "reasoning": true }, - "openrouter/auto-beta": { + "openrouter/free": { "api": { - "id": "openrouter/auto-beta", + "id": "openrouter/free", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 2000000, + "contextWindow": 200000, "cost": { "cacheRead": 0, "cacheWrite": 0, "input": 0, "output": 0 }, - "id": "openrouter/auto-beta", + "id": "openrouter/free", "input": [ "text", "image" ], "maxTokens": 4096, - "name": "Auto Router (Beta)", + "name": "Free Models Router", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", "provider": { @@ -10750,27 +10686,26 @@ "providerOptionsKey": "openrouter", "reasoning": true }, - "openrouter/free": { + "openrouter/pareto-code": { "api": { - "id": "openrouter/free", + "id": "openrouter/pareto-code", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, + "contextWindow": 2000000, "cost": { "cacheRead": 0, "cacheWrite": 0, "input": 0, "output": 0 }, - "id": "openrouter/free", + "id": "openrouter/pareto-code", "input": [ - "text", - "image" + "text" ], "maxTokens": 4096, - "name": "Free Models Router", + "name": "Pareto Code Router", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", "provider": { @@ -10784,26 +10719,27 @@ "providerOptionsKey": "openrouter", "reasoning": true }, - "openrouter/pareto-code": { + "particle-ai/particle-1": { "api": { - "id": "openrouter/pareto-code", + "id": "particle-ai/particle-1", "method": "languageModel", "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 2000000, + "contextWindow": 1048576, "cost": { - "cacheRead": 0, + "cacheRead": 0.003, "cacheWrite": 0, - "input": 0, - "output": 0 + "input": 0.05, + "output": 0.5 }, - "id": "openrouter/pareto-code", + "id": "particle-ai/particle-1", "input": [ - "text" + "text", + "image" ], - "maxTokens": 4096, - "name": "Pareto Code Router", + "maxTokens": 131072, + "name": "Particle AI: Particle-1", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", "provider": { @@ -10815,7 +10751,16 @@ "source": "api" }, "providerOptionsKey": "openrouter", - "reasoning": true + "reasoning": true, + "thinkingLevelMap": { + "high": "high", + "low": "low", + "max": "max", + "medium": "medium", + "minimal": "minimal", + "off": "none", + "xhigh": "xhigh" + } }, "perceptron/perceptron-mk1.5": { "api": { @@ -11035,6 +10980,40 @@ "xhigh": "xhigh" } }, + "prism-ml/ternary-bonsai-2-9b": { + "api": { + "id": "prism-ml/ternary-bonsai-2-9b", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 262144, + "cost": { + "cacheRead": 0.04, + "cacheWrite": 0, + "input": 0.08, + "output": 0.13 + }, + "id": "prism-ml/ternary-bonsai-2-9b", + "input": [ + "text", + "image" + ], + "maxTokens": 65536, + "name": "PrismML: Ternary Bonsai 2 9B", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true + }, "qwen/qwen-2.5-72b-instruct": { "api": { "id": "qwen/qwen-2.5-72b-instruct", @@ -11287,9 +11266,9 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0, + "cacheRead": 0.1, "cacheWrite": 0, - "input": 0.09, + "input": 0.1, "output": 0.3 }, "id": "qwen/qwen3-30b-a3b-instruct-2507", @@ -11525,42 +11504,6 @@ "providerOptionsKey": "openrouter", "reasoning": false }, - "qwen/qwen3-next-80b-a3b-thinking": { - "api": { - "id": "qwen/qwen3-next-80b-a3b-thinking", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 262144, - "cost": { - "cacheRead": 0, - "cacheWrite": 0, - "input": 0.15, - "output": 1.2 - }, - "id": "qwen/qwen3-next-80b-a3b-thinking", - "input": [ - "text" - ], - "maxTokens": 235929, - "name": "Qwen: Qwen3 Next 80B A3B Thinking", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "off": null - } - }, "qwen/qwen3-vl-235b-a22b-instruct": { "api": { "id": "qwen/qwen3-vl-235b-a22b-instruct", @@ -11745,15 +11688,15 @@ "cost": { "cacheRead": 0, "cacheWrite": 0, - "input": 0.195, - "output": 1.56 + "input": 0.26, + "output": 2.6 }, "id": "qwen/qwen3.5-27b", "input": [ "text", "image" ], - "maxTokens": 65536, + "maxTokens": 81920, "name": "Qwen: Qwen3.5-27B", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -11779,7 +11722,7 @@ "cost": { "cacheRead": 0.05, "cacheWrite": 0, - "input": 0.15, + "input": 0.14, "output": 1 }, "id": "qwen/qwen3.5-35b-a3b", @@ -11787,7 +11730,7 @@ "text", "image" ], - "maxTokens": 235929, + "maxTokens": 81920, "name": "Qwen: Qwen3.5-35B-A3B", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -11843,19 +11786,19 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 256000, + "contextWindow": 262144, "cost": { "cacheRead": 0, "cacheWrite": 0, - "input": 0.1, - "output": 0.15 + "input": 0.17, + "output": 0.25 }, "id": "qwen/qwen3.5-9b", "input": [ "text", "image" ], - "maxTokens": 32768, + "maxTokens": 235929, "name": "Qwen: Qwen3.5-9B", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -11999,17 +11942,17 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0.025, + "cacheRead": 0, "cacheWrite": 0, - "input": 0.25, - "output": 2 + "input": 0.32, + "output": 3.2 }, "id": "qwen/qwen3.6-27b", "input": [ "text", "image" ], - "maxTokens": 65536, + "maxTokens": 81920, "name": "Qwen: Qwen3.6 27B", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -12033,10 +11976,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0.05, + "cacheRead": 0.025, "cacheWrite": 0, - "input": 0.15, - "output": 1 + "input": 0.1, + "output": 0.9 }, "id": "qwen/qwen3.6-35b-a3b", "input": [ @@ -12321,10 +12264,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0.05, + "cacheRead": 0.0375, "cacheWrite": 0, - "input": 0.24, - "output": 2.2 + "input": 0.15, + "output": 1.875 }, "id": "qwen/qwen3.8-27b", "input": [ @@ -12509,6 +12452,42 @@ "providerOptionsKey": "openrouter", "reasoning": true }, + "recursive/flint-v1.0:free": { + "api": { + "id": "recursive/flint-v1.0:free", + "method": "languageModel", + "url": "https://openrouter.ai/api/v1" + }, + "baseUrl": "https://openrouter.ai/api/v1", + "contextWindow": 262144, + "cost": { + "cacheRead": 0, + "cacheWrite": 0, + "input": 0, + "output": 0 + }, + "id": "recursive/flint-v1.0:free", + "input": [ + "text" + ], + "maxTokens": 235929, + "name": "Recursive: Flint (free)", + "npm": "@openrouter/ai-sdk-provider", + "protocol": "openrouter", + "provider": { + "env": [ + "OPENROUTER_API_KEY" + ], + "id": "openrouter", + "name": "OpenRouter", + "source": "api" + }, + "providerOptionsKey": "openrouter", + "reasoning": true, + "thinkingLevelMap": { + "off": null + } + }, "reflection/beam-1.0-501b-a23b": { "api": { "id": "reflection/beam-1.0-501b-a23b", @@ -12518,16 +12497,16 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.1, + "cacheRead": 0.05, "cacheWrite": 0, - "input": 1, - "output": 3 + "input": 0.25, + "output": 1 }, "id": "reflection/beam-1.0-501b-a23b", "input": [ "text" ], - "maxTokens": 943718, + "maxTokens": 131072, "name": "Reflection: Beam 1.0 501B A23B", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -12939,10 +12918,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0.033, + "cacheRead": 0.020625, "cacheWrite": 0, - "input": 0.132, - "output": 0.528 + "input": 0.0825, + "output": 0.33 }, "id": "tencent/hy3", "input": [ @@ -13023,10 +13002,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.042, + "cacheRead": 0.0378, "cacheWrite": 0, - "input": 0.834, - "output": 2.501 + "input": 0.7506, + "output": 2.2509 }, "id": "tencent/hy4-preview", "input": [ @@ -13247,7 +13226,7 @@ "image", "text" ], - "maxTokens": 4096, + "maxTokens": 65536, "name": "TypeSafe: Jev Router", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -13372,48 +13351,6 @@ "xhigh": "xhigh" } }, - "upstage/solar-pro-3": { - "api": { - "id": "upstage/solar-pro-3", - "method": "languageModel", - "url": "https://openrouter.ai/api/v1" - }, - "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 131072, - "cost": { - "cacheRead": 0.015, - "cacheWrite": 0, - "input": 0.15, - "output": 0.6 - }, - "id": "upstage/solar-pro-3", - "input": [ - "text" - ], - "maxTokens": 117964, - "name": "Upstage: Solar Pro 3", - "npm": "@openrouter/ai-sdk-provider", - "protocol": "openrouter", - "provider": { - "env": [ - "OPENROUTER_API_KEY" - ], - "id": "openrouter", - "name": "OpenRouter", - "source": "api" - }, - "providerOptionsKey": "openrouter", - "reasoning": true, - "thinkingLevelMap": { - "high": "high", - "low": "low", - "max": null, - "medium": "medium", - "minimal": "minimal", - "off": "none", - "xhigh": null - } - }, "upstage/solar-pro4": { "api": { "id": "upstage/solar-pro4", @@ -13423,10 +13360,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 524288, "cost": { - "cacheRead": 0.018, + "cacheRead": 0.03, "cacheWrite": 0, - "input": 0.09, - "output": 0.36 + "input": 0.15, + "output": 0.6 }, "id": "upstage/solar-pro4", "input": [ @@ -13760,19 +13697,19 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 1048576, + "contextWindow": 1000000, "cost": { - "cacheRead": 0.0028, + "cacheRead": 0.08, "cacheWrite": 0, - "input": 0.14, - "output": 0.28 + "input": 0.4, + "output": 2 }, "id": "xiaomi/mimo-v2.5", "input": [ "text", "image" ], - "maxTokens": 131072, + "maxTokens": 65536, "name": "Xiaomi: MiMo-V2.5", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -13794,18 +13731,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 1048576, + "contextWindow": 262144, "cost": { - "cacheRead": 0.0036, + "cacheRead": 0.096, "cacheWrite": 0, - "input": 0.435, - "output": 0.87 + "input": 0.48, + "output": 1.8 }, "id": "xiaomi/mimo-v2.5-pro", "input": [ "text" ], - "maxTokens": 131072, + "maxTokens": 235929, "name": "Xiaomi: MiMo-V2.5-Pro", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -14261,18 +14198,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 200000, + "contextWindow": 202752, "cost": { - "cacheRead": 0.1794, + "cacheRead": 0.26, "cacheWrite": 0, - "input": 0.966, - "output": 3.036 + "input": 1.4, + "output": 4.4 }, "id": "z-ai/glm-5.1", "input": [ "text" ], - "maxTokens": 128000, + "maxTokens": 131072, "name": "Z.ai: GLM 5.1", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -14296,10 +14233,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.14, + "cacheRead": 0.005, "cacheWrite": 0, - "input": 0.62, - "output": 2.42 + "input": 0.006, + "output": 7 }, "id": "z-ai/glm-5.2", "input": [ @@ -14338,10 +14275,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.078, + "cacheRead": 0.076, "cacheWrite": 0, - "input": 0.079, - "output": 1.4 + "input": 0.08, + "output": 3.6 }, "id": "z-ai/glm-5.3", "input": [ @@ -14380,17 +14317,17 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.006, + "cacheRead": 0.0197, "cacheWrite": 0, - "input": 0.048, - "output": 0.2 + "input": 0.0197, + "output": 0.1324 }, "id": "z-ai/glm-5.3-flash", "input": [ "text", "image" ], - "maxTokens": 943718, + "maxTokens": 131072, "name": "Z.ai: GLM 5.3 Flash", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -14723,10 +14660,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.007, + "cacheRead": 0.004, "cacheWrite": 0, - "input": 0.058, - "output": 0.6 + "input": 0.022, + "output": 0.192 }, "id": "~deepseek/deepseek-flash-latest", "input": [ @@ -14766,10 +14703,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.23, + "cacheRead": 0.1, "cacheWrite": 0, - "input": 0.225, - "output": 8 + "input": 0.1035, + "output": 4 }, "id": "~deepseek/deepseek-pro-latest", "input": [ @@ -14808,10 +14745,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.00336, + "cacheRead": 0.0442, "cacheWrite": 0, - "input": 0.01208, - "output": 0.12 + "input": 0.0452, + "output": 0.3 }, "id": "~deepseek/deepseek-v4-flash-latest", "input": [ @@ -14936,10 +14873,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.228, + "cacheRead": 0.15, "cacheWrite": 0, - "input": 0.238, - "output": 16 + "input": 0.1539, + "output": 10.5539 }, "id": "~moonshotai/kimi-latest", "input": [ @@ -15178,17 +15115,17 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1050000, "cost": { - "cacheRead": 0.2, - "cacheWrite": 2.5, - "input": 2, - "output": 12, + "cacheRead": 0.05, + "cacheWrite": 1.25, + "input": 1, + "output": 5, "tiers": [ { - "cacheRead": 0.4, - "cacheWrite": 5, - "input": 4, + "cacheRead": 0.1, + "cacheWrite": 2.5, + "input": 2, "inputTokensAbove": 272000, - "output": 18 + "output": 7.5 } ] }, @@ -15316,17 +15253,17 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.006, + "cacheRead": 0.0197, "cacheWrite": 0, - "input": 0.048, - "output": 0.2 + "input": 0.0197, + "output": 0.1324 }, "id": "~z-ai/glm-flash-latest", "input": [ "text", "image" ], - "maxTokens": 943718, + "maxTokens": 131072, "name": "Z.ai: GLM Flash Latest", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -15359,9 +15296,9 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.078, + "cacheRead": 0.0689, "cacheWrite": 0, - "input": 0.079, + "input": 0.0699, "output": 1.4 }, "id": "~z-ai/glm-latest", From fbfcf6f35040e3bda719c1d9b47df7928de53f70 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 31 Oct 2026 02:01:37 +0530 Subject: [PATCH 06/11] refactor: declare prompt section changes after the prompt again Anthropic takes a mid-conversation system message only after a user message, so its wire puts a change after the prompt whatever the stored order, and OpenAI and Codex answer from either order. Declaring after the prompt drops the pending-input pre-check, the second assembly and the system-entry skips in the leaf guard and fork-before. On Anthropic, a system message before the first reply now joins the leading prompt, so the first declaration still caches. The live helper reads the GitHub Copilot login, whose token never expires, and the live test covers Copilot Claude. Co-Authored-By: Claude Opus 5.5 --- packages/aikit/src/llm/transform.ts | 23 ++++---- packages/aikit/test/transcript.test.ts | 4 +- packages/harness/src/context/assemble.ts | 3 +- packages/harness/src/runner/loop.ts | 10 +--- packages/harness/src/thread/thread.ts | 3 +- .../test/__artifacts__/prompt.changes.json | 58 +++++++++---------- .../prompt.changes.live.github-copilot.json | 10 ++++ .../prompt.changes.live.openai-codex.json | 2 +- .../prompt.changes.live.openai.json | 2 +- .../test/__artifacts__/runner.retry.json | 12 ++-- .../test/__artifacts__/thread.concurrent.json | 3 +- .../test/__artifacts__/thread.fork.json | 10 ++-- .../test/__artifacts__/thread.recovery.json | 20 ++++--- .../test/__artifacts__/thread.reset.json | 22 +++---- .../__artifacts__/thread.walkthrough.json | 10 ++-- .../test/__artifacts__/tools.read.vision.json | 18 +++--- .../__artifacts__/tools.toolsearch.fork.json | 18 +++--- .../__artifacts__/tools.toolsearch.load.json | 16 ++--- .../tools.toolsearch.unavailable.json | 12 ++-- packages/harness/test/plugin.external.test.ts | 2 +- packages/harness/test/plugin.host.test.ts | 2 +- .../harness/test/prompt.changes.live.test.ts | 7 +-- packages/harness/test/runner.loop.test.ts | 26 ++++----- packages/harness/test/sdk.test.ts | 2 +- packages/harness/test/thread.test.ts | 22 +++---- .../harness/test/tools.toolsearch.test.ts | 4 +- packages/harness/test/utils/live.ts | 5 +- 27 files changed, 167 insertions(+), 159 deletions(-) create mode 100644 packages/harness/test/__artifacts__/prompt.changes.live.github-copilot.json diff --git a/packages/aikit/src/llm/transform.ts b/packages/aikit/src/llm/transform.ts index d1acf1d0..bde9edf2 100644 --- a/packages/aikit/src/llm/transform.ts +++ b/packages/aikit/src/llm/transform.ts @@ -570,14 +570,15 @@ export function convertMessages( } // Every system message goes as an update, the first declaration too: a section a model read - // whole does not yield to a later update of it, one it read as an update does. Anthropic takes - // a later system message only after a user message, and never between a tool call and its - // result: each is held for the next assistant message, or the end. One ahead of the - // conversation joins the leading prompt there, where it caches; a trailing system message's - // cache entry is never read once a reply follows it. + // whole does not yield to a later update of it, one it read as an update does. On Anthropic, + // one before the first reply joins the leading prompt, where it caches: a trailing system + // message's cache entry is never read once a reply follows it. Anthropic takes a later one + // only after a user message, and never between a tool call and its result: each is held for + // the next assistant message, or the end. const anthropic = isAnthropicWire(model, method); + const leading: ModelMessage[] = []; const held: ModelMessage[] = []; - let started = false; + let replied = false; let systemIndex = 0; for (const msg of transformedMessages) { if (msg.role === "system") { @@ -592,20 +593,22 @@ export function convertMessages( model, method, ); - if (change) (anthropic && started ? held : messages).push(change); + if (change) (!anthropic ? messages : replied ? held : leading).push(change); continue; } - started = true; if (msg.role === "user") { messages.push(...userContent(msg.parts)); continue; } const converted = assistantMessages(msg, model, method); - if (converted.length > 0) messages.push(...held.splice(0)); + if (converted.length > 0) { + replied = true; + messages.push(...held.splice(0)); + } messages.push(...converted); } - return [...messages, ...held]; + return [...leading, ...messages, ...held]; } /** diff --git a/packages/aikit/test/transcript.test.ts b/packages/aikit/test/transcript.test.ts index e0804fb4..d4f931cc 100644 --- a/packages/aikit/test/transcript.test.ts +++ b/packages/aikit/test/transcript.test.ts @@ -379,8 +379,8 @@ describe("transcript prompt changes on the Anthropic wire", () => { { systemPrompt: "base prompt", messages: [ - system({ sections: { codeword: "alpha" } }), makeUserMessage("first"), + system({ sections: { codeword: "alpha" } }), makeAssistantMessage(model, { stopReason: "toolUse", parts: [makeCompletedToolCall("call-1")] }), system({ sections: { codeword: "bravo" } }), makeUserMessage("second"), @@ -398,7 +398,7 @@ describe("transcript prompt changes on the Anthropic wire", () => { }; }; - it("joins the first declaration to the leading prompt and holds a later change past the tool result and prompt", async () => { + it("joins a declaration before the first reply to the leading prompt and holds a later one past the tool result and prompt", async () => { expect(await wire(claude(true))).toEqual({ system: ["base prompt", 'Updated system prompt section "codeword":\n\nalpha'], messages: [ diff --git a/packages/harness/src/context/assemble.ts b/packages/harness/src/context/assemble.ts index 58ad0f4e..148c126e 100644 --- a/packages/harness/src/context/assemble.ts +++ b/packages/harness/src/context/assemble.ts @@ -8,7 +8,6 @@ import { ContextDecodeError } from "./errors.ts"; export interface Snapshot { readonly threadId: string; - /** The newest entry other than `system`: a declaration left by a cancelled prompt is nothing to answer. */ readonly leafEntryId: string | null; readonly messages: ReadonlyArray; readonly lastAssistant?: { @@ -71,7 +70,7 @@ export const assembleLine = Effect.fn("Context.assembleLine")(function* ( threadId: string, line: ReadonlyArray, ): Effect.fn.Return { - const leafEntryId = line.findLast((hydrated) => hydrated.entry.type !== "system")?.entry.id ?? null; + const leafEntryId = line.at(-1)?.entry.id ?? null; const committedPath = line.filter((hydrated) => hydrated.entry.state === "committed"); let lastAssistant: Snapshot["lastAssistant"]; let latestStart: diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index ecaa900c..bbf001ff 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -278,7 +278,7 @@ export const layer = (options: Options = {}) => * The request keeps the foundation as `systemPrompt` and the declared tools as `tools`. The * prompt sections, and what the thread offers beyond those tools (or withholds), are declared * by a `system` entry, published when the transcript's state differs from the snapshot's: the - * first request declares every section. Returns its message, for a request not reassembled after it. + * first request declares every section. Returns its message, for the request at hand. */ const declareSystemChanges = Effect.fn("Loop.declareSystemChanges")(function* ( snapshot: State.Snapshot, @@ -314,10 +314,6 @@ export const layer = (options: Options = {}) => continuing: boolean, ) { const { sessionId, threadId } = snapshot; - // Declared ahead of the input it comes with, so the model reads the setup, then the prompt. - // With no input to promote it goes last, once a request is sure to follow. - const declaredFirst = promotion !== undefined && (yield* inputs.hasPending(threadId, promotion)); - if (declaredFirst) yield* declareSystemChanges(snapshot, (yield* context.assemble(threadId)).messages); if (promotion !== undefined) { const cutoff = yield* events.latestSequence(sessionId); const followUp = promotion === "followUp" && (yield* inputs.promoteFollowUp(threadId)); @@ -333,9 +329,7 @@ export const layer = (options: Options = {}) => if (leaf !== undefined && (leaf.stopReason === "stop" || leaf.stopReason === "length")) { return undefined; } - const messages = declaredFirst - ? assembled.messages - : [...assembled.messages, ...(yield* declareSystemChanges(snapshot, assembled.messages))]; + const messages = [...assembled.messages, ...(yield* declareSystemChanges(snapshot, assembled.messages))]; // The latest attempt's publisher; a retried turn has one per provider request. let publisher: LLMEventPublisher.Publisher | undefined; diff --git a/packages/harness/src/thread/thread.ts b/packages/harness/src/thread/thread.ts index f6b8b881..ce5bf81b 100644 --- a/packages/harness/src/thread/thread.ts +++ b/packages/harness/src/thread/thread.ts @@ -674,8 +674,7 @@ export const layer = Layer.effect( if (target.state !== "committed") return yield* reject("entry_not_committed"); if (input.mode !== "before") return { point: target, text: undefined }; if (target.type !== "user") return yield* reject("not_a_user_entry"); - // A `system` entry only declares setup for the prompt after it: the prompt's own resend declares again. - const before = entries.slice(0, index).findLast((one) => one.state === "committed" && one.type !== "system"); + const before = entries.slice(0, index).findLast((one) => one.state === "committed"); if (before === undefined) return yield* reject("nothing_before_entry"); const parts = yield* partsFor([target]); const text = parts diff --git a/packages/harness/test/__artifacts__/prompt.changes.json b/packages/harness/test/__artifacts__/prompt.changes.json index 0f0842ac..a8a65896 100644 --- a/packages/harness/test/__artifacts__/prompt.changes.json +++ b/packages/harness/test/__artifacts__/prompt.changes.json @@ -2,8 +2,8 @@ "requests": [ { "roles": [ - "system", - "user" + "user", + "system" ], "declared": [ { @@ -17,11 +17,11 @@ }, { "roles": [ - "system", "user", - "assistant", "system", - "user" + "assistant", + "user", + "system" ], "declared": [ { @@ -38,14 +38,14 @@ }, { "roles": [ - "system", "user", - "assistant", "system", - "user", "assistant", + "user", "system", - "user" + "assistant", + "user", + "system" ], "declared": [ { @@ -65,14 +65,14 @@ }, { "roles": [ - "system", "user", - "assistant", "system", - "user", "assistant", + "user", "system", + "assistant", "user", + "system", "assistant", "user" ], @@ -94,14 +94,14 @@ }, { "roles": [ - "system", "user", - "assistant", "system", - "user", "assistant", + "user", "system", + "assistant", "user", + "system", "assistant", "user", "assistant", @@ -125,21 +125,21 @@ }, { "roles": [ - "system", "user", - "assistant", "system", - "user", "assistant", + "user", "system", + "assistant", "user", + "system", "assistant", "user", "assistant", "user", "assistant", - "system", - "user" + "user", + "system" ], "declared": [ { @@ -162,11 +162,11 @@ }, { "roles": [ - "system", "user", - "assistant", "system", - "user" + "assistant", + "user", + "system" ], "declared": [ { @@ -191,29 +191,29 @@ ], "threads": { "main": [ - "system", "user", - "assistant", "system", - "user", "assistant", + "user", "system", + "assistant", "user", + "system", "assistant", "user", "assistant", "user", "assistant", - "system", "user", + "system", "assistant" ], "fork": [ - "system", "user", - "assistant", "system", + "assistant", "user", + "system", "assistant" ] } diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.github-copilot.json b/packages/harness/test/__artifacts__/prompt.changes.live.github-copilot.json new file mode 100644 index 00000000..b96adf55 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.live.github-copilot.json @@ -0,0 +1,10 @@ +{ + "answers": [ + "alpha", + "bravo" + ], + "promptChange": "in place", + "earlierSectionSent": true, + "change": "after", + "cached": true +} diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json b/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json index e2115b80..663e5327 100644 --- a/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json +++ b/packages/harness/test/__artifacts__/prompt.changes.live.openai-codex.json @@ -5,5 +5,5 @@ ], "promptChange": "in place", "earlierSectionSent": true, - "change": "before" + "change": "after" } diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.openai.json b/packages/harness/test/__artifacts__/prompt.changes.live.openai.json index e2115b80..663e5327 100644 --- a/packages/harness/test/__artifacts__/prompt.changes.live.openai.json +++ b/packages/harness/test/__artifacts__/prompt.changes.live.openai.json @@ -5,5 +5,5 @@ ], "promptChange": "in place", "earlierSectionSent": true, - "change": "before" + "change": "after" } diff --git a/packages/harness/test/__artifacts__/runner.retry.json b/packages/harness/test/__artifacts__/runner.retry.json index 8256804a..56dbd33f 100644 --- a/packages/harness/test/__artifacts__/runner.retry.json +++ b/packages/harness/test/__artifacts__/runner.retry.json @@ -65,18 +65,18 @@ } ], "path": [ - { - "id": "system", - "type": "system", - "state": "committed", - "parts": 0 - }, { "id": "msg_retry", "type": "user", "state": "committed", "parts": 1 }, + { + "id": "system", + "type": "system", + "state": "committed", + "parts": 0 + }, { "id": "assistant_1", "type": "assistant", diff --git a/packages/harness/test/__artifacts__/thread.concurrent.json b/packages/harness/test/__artifacts__/thread.concurrent.json index 42839aea..54a60090 100644 --- a/packages/harness/test/__artifacts__/thread.concurrent.json +++ b/packages/harness/test/__artifacts__/thread.concurrent.json @@ -1,14 +1,15 @@ { "bothActive": true, "main": [ - "system committed", "user committed", + "system committed", "assistant committed", "user committed", "assistant committed" ], "fork": [ "user committed", + "system committed", "assistant committed" ] } diff --git a/packages/harness/test/__artifacts__/thread.fork.json b/packages/harness/test/__artifacts__/thread.fork.json index 1ab5f53f..07735f90 100644 --- a/packages/harness/test/__artifacts__/thread.fork.json +++ b/packages/harness/test/__artifacts__/thread.fork.json @@ -9,8 +9,8 @@ "events": [ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", - "3 session.thread.system.changed.1 T1", - "4 session.prompt.promoted.1 T1", + "3 session.prompt.promoted.1 T1", + "4 session.thread.system.changed.1 T1", "5 session.llm.started.1 T1", "6 session.llm.ended.1 T1", "7 session.turn.ended.1 T1", @@ -42,7 +42,7 @@ }, { "thread": "T3", - "origin": "fork T1 at E2", + "origin": "fork T1 at E1", "createdSeq": 14, "title": null }, @@ -54,8 +54,8 @@ } ], "entries": [ - "E1 T1 system seq=3 committed=3 committed", - "E2 T1 user seq=4 committed=4 committed", + "E1 T1 user seq=3 committed=3 committed", + "E2 T1 system seq=4 committed=4 committed", "E3 T1 assistant seq=5 committed=7 committed", "E4 T1 user seq=9 committed=9 committed", "E5 T1 assistant seq=10 committed=12 committed", diff --git a/packages/harness/test/__artifacts__/thread.recovery.json b/packages/harness/test/__artifacts__/thread.recovery.json index ebfa8b1d..f5bdc287 100644 --- a/packages/harness/test/__artifacts__/thread.recovery.json +++ b/packages/harness/test/__artifacts__/thread.recovery.json @@ -2,8 +2,8 @@ "events": [ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", - "3 session.thread.system.changed.1 T1", - "4 session.prompt.promoted.1 T1", + "3 session.prompt.promoted.1 T1", + "4 session.thread.system.changed.1 T1", "5 session.llm.started.1 T1", "6 session.llm.ended.1 T1", "7 session.turn.ended.1 T1", @@ -12,9 +12,10 @@ "10 session.prompt.promoted.1 T2", "11 session.llm.started.1 T2", "12 session.llm.failed.1 T2", - "13 session.llm.started.1 T2", - "14 session.llm.ended.1 T2", - "15 session.turn.ended.1 T2" + "13 session.thread.system.changed.1 T2", + "14 session.llm.started.1 T2", + "15 session.llm.ended.1 T2", + "16 session.turn.ended.1 T2" ], "threads": [ { @@ -25,18 +26,19 @@ }, { "thread": "T2", - "origin": "fork T1 at E2", + "origin": "fork T1 at E1", "createdSeq": 8, "title": "cut off" } ], "entries": [ - "E1 T1 system seq=3 committed=3 committed", - "E2 T1 user seq=4 committed=4 committed", + "E1 T1 user seq=3 committed=3 committed", + "E2 T1 system seq=4 committed=4 committed", "E3 T1 assistant seq=5 committed=7 committed", "E4 T2 user seq=10 committed=10 committed", "E5 T2 assistant seq=11 committed=12 aborted", - "E6 T2 assistant seq=13 committed=15 committed" + "E6 T2 system seq=13 committed=13 committed", + "E7 T2 assistant seq=14 committed=16 committed" ], "documents": [ { diff --git a/packages/harness/test/__artifacts__/thread.reset.json b/packages/harness/test/__artifacts__/thread.reset.json index 34c8f007..f26abe59 100644 --- a/packages/harness/test/__artifacts__/thread.reset.json +++ b/packages/harness/test/__artifacts__/thread.reset.json @@ -2,22 +2,22 @@ "context": { "roles": [ "user", - "system", - "user" + "user", + "system" ], - "texts": "The conversation was started over. Carried over from before:\n\n\nkeep the login work\n | | after the reset" + "texts": "The conversation was started over. Carried over from before:\n\n\nkeep the login work\n | after the reset | " }, "events": [ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", - "3 session.thread.system.changed.1 T1", - "4 session.prompt.promoted.1 T1", + "3 session.prompt.promoted.1 T1", + "4 session.thread.system.changed.1 T1", "5 session.llm.started.1 T1", "6 session.llm.ended.1 T1", "7 session.turn.ended.1 T1", "9 session.prompt.admitted.1 T1", - "10 session.thread.system.changed.1 T1", - "11 session.prompt.promoted.1 T1", + "10 session.prompt.promoted.1 T1", + "11 session.thread.system.changed.1 T1", "12 session.llm.started.1 T1", "13 session.llm.ended.1 T1", "14 session.turn.ended.1 T1" @@ -31,12 +31,12 @@ } ], "entries": [ - "E1 T1 system seq=3 committed=3 committed", - "E2 T1 user seq=4 committed=4 committed", + "E1 T1 user seq=3 committed=3 committed", + "E2 T1 system seq=4 committed=4 committed", "E3 T1 assistant seq=5 committed=7 committed", "E4 T1 reset seq=8 committed=8 committed", - "E5 T1 system seq=10 committed=10 committed", - "E6 T1 user seq=11 committed=11 committed", + "E5 T1 user seq=10 committed=10 committed", + "E6 T1 system seq=11 committed=11 committed", "E7 T1 assistant seq=12 committed=14 committed" ], "documents": [ diff --git a/packages/harness/test/__artifacts__/thread.walkthrough.json b/packages/harness/test/__artifacts__/thread.walkthrough.json index e687f42c..fc5be405 100644 --- a/packages/harness/test/__artifacts__/thread.walkthrough.json +++ b/packages/harness/test/__artifacts__/thread.walkthrough.json @@ -3,8 +3,8 @@ "1 session.created.1 T1", "2 session.thread.configured.1 T1", "3 session.prompt.admitted.1 T1", - "4 session.thread.system.changed.1 T1", - "5 session.prompt.promoted.1 T1", + "4 session.prompt.promoted.1 T1", + "5 session.thread.system.changed.1 T1", "6 session.llm.started.1 T1", "7 session.llm.ended.1 T1", "8 session.thread.configured.1 T1", @@ -48,8 +48,8 @@ } ], "entries": [ - "E1 T1 system seq=4 committed=4 committed", - "E2 T1 user seq=5 committed=5 committed", + "E1 T1 user seq=4 committed=4 committed", + "E2 T1 system seq=5 committed=5 committed", "E3 T1 assistant seq=6 committed=10 committed", "E4 T1 assistant seq=11 committed=13 committed", "E5 T1 user seq=15 committed=15 committed", @@ -84,8 +84,8 @@ } ], "t2Context": [ - "system", "user", + "system", "assistant", "assistant", "user" diff --git a/packages/harness/test/__artifacts__/tools.read.vision.json b/packages/harness/test/__artifacts__/tools.read.vision.json index be222844..f62e6c80 100644 --- a/packages/harness/test/__artifacts__/tools.read.vision.json +++ b/packages/harness/test/__artifacts__/tools.read.vision.json @@ -2,29 +2,29 @@ { "model": "text-only", "messages": [ - { - "role": "system" - }, { "role": "user", "content": [ "read the image" ] + }, + { + "role": "system" } ] }, { "model": "text-only", "messages": [ - { - "role": "system" - }, { "role": "user", "content": [ "read the image" ] }, + { + "role": "system" + }, { "role": "assistant", "parts": [ @@ -48,15 +48,15 @@ { "model": "vision", "messages": [ - { - "role": "system" - }, { "role": "user", "content": [ "read the image" ] }, + { + "role": "system" + }, { "role": "assistant", "parts": [ diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.fork.json b/packages/harness/test/__artifacts__/tools.toolsearch.fork.json index 14e7e83b..c585146b 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.fork.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.fork.json @@ -87,7 +87,13 @@ "entries": [ { "id": "E1", - "seq": 25, + "seq": 26, + "type": "user", + "state": "committed" + }, + { + "id": "E2", + "seq": 27, "type": "system", "state": "committed", "data": { @@ -125,21 +131,15 @@ "hash": "ec3094bdf14770d4650a88a2307556654cabf8623185fefb11702ba2e9a9c6e8" } }, - { - "id": "E2", - "seq": 26, - "type": "user", - "state": "committed" - }, { "id": "E3", - "seq": 27, + "seq": 28, "type": "assistant", "state": "committed" }, { "id": "E4", - "seq": 31, + "seq": 32, "type": "assistant", "state": "committed" } diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.load.json b/packages/harness/test/__artifacts__/tools.toolsearch.load.json index a29ab1fc..330d91e8 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.load.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.load.json @@ -114,11 +114,11 @@ }, { "seq": 3, - "type": "session.thread.system.changed.1" + "type": "session.prompt.promoted.1" }, { "seq": 4, - "type": "session.prompt.promoted.1" + "type": "session.thread.system.changed.1" }, { "seq": 5, @@ -173,6 +173,12 @@ { "id": "E1", "seq": 3, + "type": "user", + "state": "committed" + }, + { + "id": "E2", + "seq": 4, "type": "system", "state": "committed", "data": { @@ -186,12 +192,6 @@ "hash": "2aedbc0fffd70a2b41a7ce78f786f9809d727be73bf31b672ac2d603d3817fbb" } }, - { - "id": "E2", - "seq": 4, - "type": "user", - "state": "committed" - }, { "id": "E3", "seq": 5, diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json b/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json index 06cf4391..c8151453 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json @@ -60,6 +60,12 @@ { "id": "E1", "seq": 4, + "type": "user", + "state": "committed" + }, + { + "id": "E2", + "seq": 5, "type": "system", "state": "committed", "data": { @@ -77,12 +83,6 @@ "hash": "d6d66014ddda5da53e6e8e6520f0e6e1851a1a9c246d23802463da98342397a1" } }, - { - "id": "E2", - "seq": 5, - "type": "user", - "state": "committed" - }, { "id": "E3", "seq": 6, diff --git a/packages/harness/test/plugin.external.test.ts b/packages/harness/test/plugin.external.test.ts index 9c70695d..bb13ceec 100644 --- a/packages/harness/test/plugin.external.test.ts +++ b/packages/harness/test/plugin.external.test.ts @@ -125,7 +125,7 @@ describe("third-party plugins", () => { expect(contexts[0]?.tools?.map((tool) => tool.name)).toEqual(["acme_echo"]); expect(prompts[0]).toContain("- acme_echo: Echo a value back"); - expect(path.map((item) => item.entry.type)).toEqual(["system", "user", "assistant", "assistant"]); + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant", "assistant"]); const settled = JSON.parse(path.find((item) => item.entry.type === "assistant")?.parts[0]?.data ?? "{}"); expect(settled).toMatchObject({ status: "completed", diff --git a/packages/harness/test/plugin.host.test.ts b/packages/harness/test/plugin.host.test.ts index bd4a22bd..5b293433 100644 --- a/packages/harness/test/plugin.host.test.ts +++ b/packages/harness/test/plugin.host.test.ts @@ -331,7 +331,7 @@ describe("plugin pipelines in the kernel loop", () => { ]); if (mode === "sequential") expect(trace.indexOf("a:done")).toBeLessThan(trace.indexOf("b:before")); const path = yield* session.history(); - expect(path.map((item) => item.entry.type)).toEqual(["system", "user", "assistant", "assistant"]); + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant", "assistant"]); }).pipe( Effect.provide( Harness.layer({ diff --git a/packages/harness/test/prompt.changes.live.test.ts b/packages/harness/test/prompt.changes.live.test.ts index 6456e4f4..9f1fc45e 100644 --- a/packages/harness/test/prompt.changes.live.test.ts +++ b/packages/harness/test/prompt.changes.live.test.ts @@ -14,8 +14,7 @@ import "./utils/live.ts"; /** * A prompt section changed between two prompts reaches a real model. These models take system * messages in place, so the second request still carries the first codeword and adds the change - * after it: ahead of the prompt it came with, or on Anthropic, which takes one only after a user - * message, right after that prompt. Where caching is explicit (Anthropic), the second request + * after it, right after the prompt it came with. Where caching is explicit (Anthropic), the second request * reads the first one's sections from the cache; OpenAI's is best effort, so it is not checked there. */ @@ -27,8 +26,8 @@ const reference = Array.from({ length: 300 }, (_, i) => `Reference line ${i}: th ); process.env.CODEWORK_CACHE_RETENTION ??= "short"; const CASES = [ - { name: "openai-codex", provider: "openai-codex", id: "gpt-6-luna", env: "OPENAI_CODEX_API_KEY", change: "before" }, - { name: "openai", provider: "openai", id: "gpt-6-luna", env: "OPENAI_API_KEY", change: "before" }, + { name: "openai-codex", provider: "openai-codex", id: "gpt-6-luna", env: "OPENAI_CODEX_API_KEY", change: "after" }, + { name: "openai", provider: "openai", id: "gpt-6-luna", env: "OPENAI_API_KEY", change: "after" }, { name: "anthropic", provider: "anthropic", diff --git a/packages/harness/test/runner.loop.test.ts b/packages/harness/test/runner.loop.test.ts index 1c283f02..a5c98be1 100644 --- a/packages/harness/test/runner.loop.test.ts +++ b/packages/harness/test/runner.loop.test.ts @@ -172,8 +172,8 @@ describe("runner loop — aikit input/output", () => { const bySeq = [...rows].sort((left, right) => (left.promotedSeq ?? 0) - (right.promotedSeq ?? 0)); expect(bySeq.map((row) => row.id)).toEqual(["msg_steer", "msg_follow"]); expect((yield* threads.line(threadId)).map((item) => item.entry.type)).toEqual([ - "system", "user", + "system", "assistant", "user", "assistant", @@ -182,8 +182,8 @@ describe("runner loop — aikit input/output", () => { // Every request is rebuilt from durable history. The second request sees // the first terminal assistant and the newly promoted follow-up. expect(contexts.map((context) => context.messages.map((message) => message.role))).toEqual([ - ["system", "user"], - ["system", "user", "assistant", "user"], + ["user", "system"], + ["user", "system", "assistant", "user"], ]); }), ); @@ -199,10 +199,10 @@ describe("runner loop — aikit input/output", () => { yield* execution.resume(threadId); yield* execution.resume(threadId); - expect(yield* delivered(threadId)).toEqual([{ id: "msg_once", promotedSeq: 4 }]); + expect(yield* delivered(threadId)).toEqual([{ id: "msg_once", promotedSeq: 3 }]); expect((yield* threads.line(threadId)).map((item) => item.entry.type)).toEqual([ - "system", "user", + "system", "assistant", ]); }), @@ -224,10 +224,10 @@ describe("runner loop — aikit input/output", () => { const path = yield* threads.line(threadId); expect(path.map((item) => item.entry.type)).toEqual([ - "system", "user", "user", "user", + "system", "assistant", "user", "assistant", @@ -356,8 +356,8 @@ describe("runner loop — tool continuation and lifecycle gate", () => { const path = yield* threads.line(threadId); expect(path.map((item) => [item.entry.type, item.entry.state])).toEqual([ - ["system", "committed"], ["user", "committed"], + ["system", "committed"], ["assistant", "committed"], ["assistant", "committed"], ]); @@ -512,7 +512,7 @@ describe("runner loop — tool interruption", () => { expect(interrupted.data.reason).toBe("user"); const path = yield* threads.line(threadId); - expect(path.map((item) => item.entry.type)).toEqual(["system", "user", "assistant"]); + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); expect(path[2]!.entry.state).toBe("committed"); expect(path[2]!.parts.map((part) => Option.getOrThrow(part.status))).toEqual(["aborted", "aborted"]); for (const part of path[2]!.parts) { @@ -578,7 +578,7 @@ describe("runner loop — provider interruption", () => { expect(Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause)).toBe(true); const path = yield* threads.line(threadId); - expect(path.map((item) => item.entry.type)).toEqual(["system", "user", "assistant"]); + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); expect(path[2]!.entry.state).toBe("aborted"); expect(JSON.parse(path[2]!.entry.data)).toMatchObject({ stopReason: "aborted" }); expect(path[2]!.parts).toEqual([]); @@ -635,7 +635,7 @@ describe("runner loop — provider failure", () => { } const path = yield* threads.line(threadId); - expect(path.map((item) => item.entry.type)).toEqual(["system", "user", "assistant"]); + expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); expect(path[2]!.entry.state).toBe("aborted"); expect(path[2]!.parts).toEqual([]); expect(JSON.parse(path[2]!.entry.data)).toMatchObject({ @@ -1559,8 +1559,8 @@ describe("runner loop — lifecycle hooks", () => { const threads = yield* Thread.Service; const path = yield* threads.line(threadId); expect(path.map((item) => [item.entry.type, item.entry.state])).toEqual([ - ["system", "committed"], ["user", "committed"], + ["system", "committed"], ["assistant", "committed"], ]); const tool = path[2]!.parts.find((part) => part.type === "toolCall"); @@ -1571,10 +1571,10 @@ describe("runner loop — lifecycle hooks", () => { // The next run answers the tool results first, then picks the queued follow-up up. yield* execution.resume(threadId); expect(contexts).toHaveLength(3); - expect(contexts[1]!.messages.map((message) => message.role)).toEqual(["system", "user", "assistant"]); + expect(contexts[1]!.messages.map((message) => message.role)).toEqual(["user", "system", "assistant"]); expect(contexts[2]!.messages.map((message) => message.role)).toEqual([ - "system", "user", + "system", "assistant", "assistant", "user", diff --git a/packages/harness/test/sdk.test.ts b/packages/harness/test/sdk.test.ts index 5682259b..3edb0faa 100644 --- a/packages/harness/test/sdk.test.ts +++ b/packages/harness/test/sdk.test.ts @@ -88,8 +88,8 @@ describe("Harness Effect SDK", () => { yield* session.run("second"); // The restarted runtime finds the prompt the first run declared, so declares nothing again. expect((yield* session.history()).map(({ entry }) => entry.type)).toEqual([ - "system", "user", + "system", "assistant", "user", "assistant", diff --git a/packages/harness/test/thread.test.ts b/packages/harness/test/thread.test.ts index 9d747dfd..60b2383d 100644 --- a/packages/harness/test/thread.test.ts +++ b/packages/harness/test/thread.test.ts @@ -307,8 +307,8 @@ describe("threads", () => { const line = yield* threads.line(main); expect(line.map(({ entry }) => entry.type)).toEqual([ - "system", "user", + "system", "assistant", "assistant", "user", @@ -360,8 +360,8 @@ describe("threads", () => { yield* run(sessionId, main, "first"); yield* run(sessionId, main, "second"); const line = yield* threads.line(main); - expect(line.map(({ entry }) => entry.type)).toEqual(["system", "user", "assistant", "user", "assistant"]); - const [, e1, e2, e3, e4] = line; + expect(line.map(({ entry }) => entry.type)).toEqual(["user", "system", "assistant", "user", "assistant"]); + const [e1, , e2, e3, e4] = line; // An inherited entry re-points the origin to the thread that wrote it. const t2 = ThreadSchema.ID.make((yield* threads.fork({ threadId: main, entryId: e4!.entry.id })).thread.id); @@ -422,7 +422,7 @@ describe("threads", () => { const execution = yield* RunnerExecution.Service; const { sessionId, main } = yield* createSession; yield* run(sessionId, main, "base"); - const first = (yield* threads.line(main)).find(({ entry }) => entry.type === "user"); + const [first] = yield* threads.line(main); const t2 = ThreadSchema.ID.make((yield* threads.fork({ threadId: main, entryId: first!.entry.id })).thread.id); const gates = yield* Effect.forEach([main, t2], (threadId, index) => @@ -453,14 +453,14 @@ describe("threads", () => { ); const result = { bothActive: true, main: yield* own(main), fork: yield* own(t2) }; expect(result.main).toEqual([ - "system committed", "user committed", + "system committed", "assistant committed", "user committed", "assistant committed", ]); - // The fork inherits main's system entry, so its first request declares nothing again. - expect(result.fork).toEqual(["user committed", "assistant committed"]); + // The fork cuts before main's system entry, so its first request declares its own. + expect(result.fork).toEqual(["user committed", "system committed", "assistant committed"]); // Seqs interleave between the two runs, so the artifact holds per-thread facts only. yield* artifact("thread.concurrent.json", result); }), @@ -487,7 +487,7 @@ describe("threads", () => { yield* run(sessionId, main, "after the reset"); const last = contexts.at(-1)!; // The reset drops the earlier system entry too, so the request declares the prompt again. - expect(last.roles).toEqual(["user", "system", "user"]); + expect(last.roles).toEqual(["user", "user", "system"]); expect(last.texts.includes("before the reset")).toBe(false); expect(last.texts.includes("keep the login work")).toBe(true); yield* artifact("thread.reset.json", { @@ -506,7 +506,7 @@ describe("threads", () => { const events = yield* Event.Service; const { sessionId, main } = yield* createSession; yield* run(sessionId, main, "base"); - const first = (yield* threads.line(main)).find(({ entry }) => entry.type === "user"); + const [first] = yield* threads.line(main); const t2 = ThreadSchema.ID.make((yield* threads.fork({ threadId: main, entryId: first!.entry.id })).thread.id); // What a process dying mid-turn leaves: a promoted prompt, an open draft, the run still marked. @@ -549,14 +549,14 @@ describe("threads", () => { const own = (yield* threads.line(t2)) .filter(({ entry }) => entry.threadId === t2) .map(({ entry }) => `${entry.type} ${entry.state}`); - expect(own).toEqual(["user committed", "assistant aborted", "assistant committed"]); + expect(own).toEqual(["user committed", "assistant aborted", "system committed", "assistant committed"]); const thread = Option.getOrThrow(yield* threads.get(t2)); expect(Option.getOrUndefined(thread.execution)).toBeUndefined(); expect(thread.restarts).toBe(0); // Only the thread that was cut off ran again, from its committed entries: the inherited // prompt and its own, without the aborted draft. expect(contexts.filter((one) => one.threadId === main)).toHaveLength(1); - expect(contexts.at(-1)!.roles).toEqual(["system", "user", "user"]); + expect(contexts.at(-1)!.roles).toEqual(["user", "user", "system"]); yield* artifact("thread.recovery.json", yield* snapshot(sessionId)); }), ); diff --git a/packages/harness/test/tools.toolsearch.test.ts b/packages/harness/test/tools.toolsearch.test.ts index b060246b..1ed32fcb 100644 --- a/packages/harness/test/tools.toolsearch.test.ts +++ b/packages/harness/test/tools.toolsearch.test.ts @@ -463,8 +463,8 @@ describe("toolsearch", () => { const { sessionId, main } = yield* createSession; yield* run(sessionId, main, "Load the forecast tool", [call(search("weather forecast", 1)), reply]); const line = yield* threads.line(main); - expect(line.map(({ entry }) => entry.type)).toEqual(["system", "user", "assistant", "system", "assistant"]); - const [, asked, searched] = line; + expect(line.map(({ entry }) => entry.type)).toEqual(["user", "system", "assistant", "system", "assistant"]); + const [asked, , searched] = line; const before = ThreadSchema.ID.make( (yield* threads.fork({ threadId: main, entryId: asked!.entry.id })).thread.id, diff --git a/packages/harness/test/utils/live.ts b/packages/harness/test/utils/live.ts index 7af06c76..4b6c7b31 100644 --- a/packages/harness/test/utils/live.ts +++ b/packages/harness/test/utils/live.ts @@ -7,12 +7,13 @@ import "./env.ts"; /** * A login's current access token, for its key variable. The login file itself is never used by a * test: a refresh would rotate the refresh token and sign the developer out. An expired token - * skips the provider's cases rather than refreshing. + * skips the provider's cases rather than refreshing; `expires: 0` never expires (GitHub Copilot). */ const loginToken = (provider: string): string | undefined => { try { const login = JSON.parse(readFileSync(join(homedir(), ".codework", "aikit", "auth.json"), "utf8"))[provider]; - return typeof login?.access === "string" && login.expires > Date.now() + 10 * 60_000 ? login.access : undefined; + const fresh = login?.expires === 0 || login?.expires > Date.now() + 10 * 60_000; + return typeof login?.access === "string" && fresh ? login.access : undefined; } catch { return undefined; } From 8159252a1e9c0984148c6e410f89322da55db4e2 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 31 Oct 2026 02:10:14 +0530 Subject: [PATCH 07/11] test(harness): cover GitHub Copilot GPT models in the live prompt test gpt-6-luna takes the change in place on the Responses transport; gpt-5.3-codex, without mid-conversation system messages, folds it into the leading prompt. Co-Authored-By: Claude Opus 5.5 --- ...anges.live.copilot-claude-sonnet-5.5.json} | 0 ...pt.changes.live.copilot-gpt-5.3-codex.json | 8 +++++ ...rompt.changes.live.copilot-gpt-6-luna.json | 9 ++++++ .../harness/test/prompt.changes.live.test.ts | 29 +++++++++++++++---- 4 files changed, 41 insertions(+), 5 deletions(-) rename packages/harness/test/__artifacts__/{prompt.changes.live.github-copilot.json => prompt.changes.live.copilot-claude-sonnet-5.5.json} (100%) create mode 100644 packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-5.3-codex.json create mode 100644 packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-6-luna.json diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.github-copilot.json b/packages/harness/test/__artifacts__/prompt.changes.live.copilot-claude-sonnet-5.5.json similarity index 100% rename from packages/harness/test/__artifacts__/prompt.changes.live.github-copilot.json rename to packages/harness/test/__artifacts__/prompt.changes.live.copilot-claude-sonnet-5.5.json diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-5.3-codex.json b/packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-5.3-codex.json new file mode 100644 index 00000000..45391da9 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-5.3-codex.json @@ -0,0 +1,8 @@ +{ + "answers": [ + "alpha", + "bravo" + ], + "promptChange": "folded", + "earlierSectionSent": false +} diff --git a/packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-6-luna.json b/packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-6-luna.json new file mode 100644 index 00000000..663e5327 --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.changes.live.copilot-gpt-6-luna.json @@ -0,0 +1,9 @@ +{ + "answers": [ + "alpha", + "bravo" + ], + "promptChange": "in place", + "earlierSectionSent": true, + "change": "after" +} diff --git a/packages/harness/test/prompt.changes.live.test.ts b/packages/harness/test/prompt.changes.live.test.ts index 9f1fc45e..73b76be4 100644 --- a/packages/harness/test/prompt.changes.live.test.ts +++ b/packages/harness/test/prompt.changes.live.test.ts @@ -37,17 +37,27 @@ const CASES = [ cache: true, }, { - name: "github-copilot", + name: "copilot-claude-sonnet-5.5", provider: "github-copilot", id: "claude-sonnet-5.5", env: "COPILOT_GITHUB_TOKEN", change: "after", cache: true, }, + { + name: "copilot-gpt-6-luna", + provider: "github-copilot", + id: "gpt-6-luna", + env: "COPILOT_GITHUB_TOKEN", + change: "after", + }, + // Without mid-conversation system messages: the change is folded into the leading prompt. + { name: "copilot-gpt-5.3-codex", provider: "github-copilot", id: "gpt-5.3-codex", env: "COPILOT_GITHUB_TOKEN" }, ] as const; describe.each(CASES)("prompt sections against live $name", (c) => { - const { name, provider, id, env, change } = c; + const { name, provider, id, env } = c; + const change = "change" in c ? c.change : undefined; const cache = "cache" in c && c.cache; it.skipIf(!process.env[env]?.trim())( "answers from a section changed between prompts, sent in place", @@ -117,12 +127,21 @@ describe.each(CASES)("prompt sections against live $name", (c) => { answers: replies.map((reply) => ["alpha", "bravo"].find((word) => reply.includes(word))), promptChange: second.includes('Updated system prompt section \\"codeword\\"') ? "in place" : "folded", earlierSectionSent: second.includes("The codeword is alpha."), - change: bravo < 0 ? "missing" : bravo < second.indexOf("What is the codeword now?") ? "before" : "after", + ...(change === undefined + ? {} + : { + change: + bravo < 0 + ? "missing" + : bravo < second.indexOf("What is the codeword now?") + ? "before" + : "after", + }), // The change kept the prefix: the second request reads the first one's sections from the cache. ...(cache ? { cached: cacheRead > 0 } : {}), }; - expect(artifact.promptChange).toBe("in place"); - expect(artifact.change).toBe(change); + expect(artifact.promptChange).toBe(change === undefined ? "folded" : "in place"); + if (change !== undefined) expect(artifact.change).toBe(change); if (cache) expect(artifact.cached).toBe(true); await expect(`${JSON.stringify(artifact, null, "\t")}\n`).toMatchFileSnapshot( `./__artifacts__/prompt.changes.live.${name}.json`, From b4d89166ca92a2122e02d6c420bc49be89465f6e Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 31 Oct 2026 02:17:50 +0530 Subject: [PATCH 08/11] docs: changelog for in-place prompt changes across providers Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d718ea23..312e65b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ This file is the canonical source for unreleased changes and published release n - MCP OAuth: servers sign in with discovery, client registration and PKCE, tokens stored in `~/.codework/mcpauth.json` by server name and URL and refreshed under a lock shared across processes. A server waiting for sign-in reconnects on the next message once its sign-in changes. `auth: { provider }` sends a model provider's credential instead. - The ephemeral `mcp.server.status` event reports each MCP server state change (`connecting`, `connected`, `disconnected`, `failed`, `authenticate`, `disabled`) to every thread using the server. A tool the MCP SDK rejects is skipped instead of failing its server, and listed as `skipped` on the `connected` status. - `toolsearch` turns itself on when a thread has deferred tools or an enabled MCP server with `deferred` exposure; a settings entry still turns it off or on. A search waits up to 10 seconds for such servers still connecting. -- Prompt sections reach the model as transcript system messages. A request sends the foundation as its system prompt; the first request of a thread appends a `system` entry declaring every tagged section (``, ``, ``, plugin sections), and a later request appends one only for the sections that changed, a removed section as `null`, together with any tool change. Resume and fork replay these entries, so the model sees what it saw before, and a model that takes system messages in place (OpenAI Codex, GitHub Copilot Responses) keeps its cached prefix when a section changes. +- Prompt sections reach the model as transcript system messages. A request sends the foundation as its system prompt; the first request of a thread appends a `system` entry declaring every tagged section (``, ``, ``, plugin sections), and a later request appends one only for the sections that changed, a removed section as `null`, together with any tool change. Resume and fork replay these entries, so the model sees what it saw before, and a model that takes system messages in place (OpenAI Codex, OpenAI and GitHub Copilot Responses, Anthropic and GitHub Copilot Claude) keeps its cached prefix when a section changes. #### Breaking Changes @@ -91,9 +91,9 @@ This file is the canonical source for unreleased changes and published release n - Transcript system messages change the tools mid-conversation: `Message.createSystemMessage({ toolsAdded?, toolsRemoved?, metadata? })`. `Context.tools` is the initial set. OpenAI Codex and GitHub Copilot Responses models that support it declare every tool in place, in `additional_tools` items, and announce removals as text, so the cached prompt prefix holds across tool changes; models with tool search only get later additions as a tool search result; every other model gets the current tools top-level. - `Transcript` helpers: `getCurrentTools`, `getDeclaredTools`, `getToolStateChanges`, `declarationsEqual`, `toToolDeclaration` and `resolveTranscriptTools`. -- Transcript system messages also change the system prompt: `content` adds instructions from that point on, and `sections` replace named sections, `null` removing one. `Context.systemPrompt` is the initial prompt. OpenAI Codex and GitHub Copilot Responses models with `supportsMidConvoSystemMessages` keep the leading prompt and get each change in place as a developer message (`Updated system prompt section "": …`), so the cached prompt prefix holds; every other model gets the current prompt as its system prompt. Helpers: `getCurrentSystemPrompt`, `renderSystemPrompt`, `renderSystemPromptUpdate` and `resolveTranscriptPrompt`. -- `supportsMidConvoSystemMessages` compat flag, set by `modelgen` for the OpenAI Codex and GitHub Copilot Responses models that accept developer messages mid-conversation. In-place tool declarations (`supportsAdditionalTools`) require it. -- `Transcript.getCurrentSections`: the sections a transcript declares after replaying its system messages. +- Transcript system messages also change the system prompt: `content` adds instructions from that point on, and `sections` replace named sections, `null` removing one. `Context.systemPrompt` is the initial prompt. Models with `supportsMidConvoSystemMessages` keep the leading prompt and get each change in place as a system or developer message (`Updated system prompt section "": …`), so the cached prompt prefix holds; every other model gets the current prompt as its system prompt. On Anthropic a change is sent before the next assistant message or at the end, since the Messages API takes a system message only after a user message and never between a tool call and its result; one before the first reply joins the top-level system prompt, where it caches. Helpers: `getCurrentSystemPrompt`, `renderSystemPrompt`, `renderSystemPromptUpdate` and `resolveTranscriptPrompt`. +- `supportsMidConvoSystemMessages` compat flag, set by `modelgen` for the models that accept system messages mid-conversation: OpenAI Codex, OpenAI and GitHub Copilot Responses (gpt-5.4 and later), and Anthropic and GitHub Copilot Claude (claude-opus-4.8, -5 and -5.5, claude-sonnet-5.5, claude-haiku-5.5, claude-fable-5 and -5.1, claude-mythos-5 and -5.1). The baseline catalog carries it. In-place tool declarations (`supportsAdditionalTools`) require it; Anthropic tool changes stay top-level. +- `Transcript.getCurrentSections`: the sections a transcript declares after replaying its system messages. `Transcript.diffSections(previous, current)`: the patch between two section sets, `null` removing one. - `filterGitHubCopilotModels` keeps the catalog models a Copilot login can reach. - GitHub Copilot GPT models turn reasoning off with `reasoning.effort: "none"` where OpenAI accepts it, such as gpt-5.3-codex, gpt-5.6-luna and gpt-6-luna. From 4abdec2492b3bfedd40f6c8711f7e220f3ce2366 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 31 Oct 2026 11:40:05 +0530 Subject: [PATCH 09/11] feat(aikit): change tools in place on Anthropic by reference The request declares the initial tools, then every other tool the thread can reach with defer_loading: the deferred catalog, now passed as Context.deferredTools, and every tool the transcript declared. A later system message adds or removes one as a tool reference, so loading a deferred tool keeps the cached prefix; adding one to the top-level list did not. Direct Anthropic only: Copilot rejects tool changes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- packages/aikit/src/cli/modelgen.ts | 3 +- packages/aikit/src/llm/stream.ts | 5 +- packages/aikit/src/llm/transform.ts | 76 ++++++++++++++++++- packages/aikit/src/message/message.ts | 2 + packages/aikit/src/model/baseline.json | 65 +++++++++------- packages/aikit/src/model/model.ts | 5 +- .../aikit/test/__artifacts__/modelgen.json | 7 ++ packages/aikit/test/transcript.test.ts | 62 +++++++++++++++ packages/harness/src/runner/loop.ts | 1 + packages/harness/src/tool/registry.ts | 4 + .../tools.toolsearch.live.anthropic.json | 52 +++++++++++++ .../test/tools.toolsearch.live.test.ts | 29 ++++--- 13 files changed, 271 insertions(+), 44 deletions(-) create mode 100644 packages/harness/test/__artifacts__/tools.toolsearch.live.anthropic.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 312e65b9..2240ab56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ This file is the canonical source for unreleased changes and published release n - MCP OAuth: servers sign in with discovery, client registration and PKCE, tokens stored in `~/.codework/mcpauth.json` by server name and URL and refreshed under a lock shared across processes. A server waiting for sign-in reconnects on the next message once its sign-in changes. `auth: { provider }` sends a model provider's credential instead. - The ephemeral `mcp.server.status` event reports each MCP server state change (`connecting`, `connected`, `disconnected`, `failed`, `authenticate`, `disabled`) to every thread using the server. A tool the MCP SDK rejects is skipped instead of failing its server, and listed as `skipped` on the `connected` status. - `toolsearch` turns itself on when a thread has deferred tools or an enabled MCP server with `deferred` exposure; a settings entry still turns it off or on. A search waits up to 10 seconds for such servers still connecting. +- The deferred tools of the catalog reach every request as `deferredTools`, so Anthropic declares them up front and loading one through `toolsearch` keeps the cached prefix. - Prompt sections reach the model as transcript system messages. A request sends the foundation as its system prompt; the first request of a thread appends a `system` entry declaring every tagged section (``, ``, ``, plugin sections), and a later request appends one only for the sections that changed, a removed section as `null`, together with any tool change. Resume and fork replay these entries, so the model sees what it saw before, and a model that takes system messages in place (OpenAI Codex, OpenAI and GitHub Copilot Responses, Anthropic and GitHub Copilot Claude) keeps its cached prefix when a section changes. #### Breaking Changes @@ -92,7 +93,8 @@ This file is the canonical source for unreleased changes and published release n - Transcript system messages change the tools mid-conversation: `Message.createSystemMessage({ toolsAdded?, toolsRemoved?, metadata? })`. `Context.tools` is the initial set. OpenAI Codex and GitHub Copilot Responses models that support it declare every tool in place, in `additional_tools` items, and announce removals as text, so the cached prompt prefix holds across tool changes; models with tool search only get later additions as a tool search result; every other model gets the current tools top-level. - `Transcript` helpers: `getCurrentTools`, `getDeclaredTools`, `getToolStateChanges`, `declarationsEqual`, `toToolDeclaration` and `resolveTranscriptTools`. - Transcript system messages also change the system prompt: `content` adds instructions from that point on, and `sections` replace named sections, `null` removing one. `Context.systemPrompt` is the initial prompt. Models with `supportsMidConvoSystemMessages` keep the leading prompt and get each change in place as a system or developer message (`Updated system prompt section "": …`), so the cached prompt prefix holds; every other model gets the current prompt as its system prompt. On Anthropic a change is sent before the next assistant message or at the end, since the Messages API takes a system message only after a user message and never between a tool call and its result; one before the first reply joins the top-level system prompt, where it caches. Helpers: `getCurrentSystemPrompt`, `renderSystemPrompt`, `renderSystemPromptUpdate` and `resolveTranscriptPrompt`. -- `supportsMidConvoSystemMessages` compat flag, set by `modelgen` for the models that accept system messages mid-conversation: OpenAI Codex, OpenAI and GitHub Copilot Responses (gpt-5.4 and later), and Anthropic and GitHub Copilot Claude (claude-opus-4.8, -5 and -5.5, claude-sonnet-5.5, claude-haiku-5.5, claude-fable-5 and -5.1, claude-mythos-5 and -5.1). The baseline catalog carries it. In-place tool declarations (`supportsAdditionalTools`) require it; Anthropic tool changes stay top-level. +- `supportsMidConvoSystemMessages` compat flag, set by `modelgen` for the models that accept system messages mid-conversation: OpenAI Codex, OpenAI and GitHub Copilot Responses (gpt-5.4 and later), and Anthropic and GitHub Copilot Claude (claude-opus-4.8, -5 and -5.5, claude-sonnet-5.5, claude-haiku-5.5, claude-fable-5 and -5.1, claude-mythos-5 and -5.1). The baseline catalog carries it. In-place tool declarations (`supportsAdditionalTools`) require it. +- Anthropic models take tool changes in place (`supportsAdditionalTools`, set for direct Anthropic only, as Copilot rejects them): the request declares its initial tools, then every other tool the thread can reach deferred (`defer_loading`), the catalog passed as the new `Context.deferredTools` and every tool the transcript declared, and a later system message adds or removes one by reference. Loading a deferred tool then keeps the cached prompt prefix. - `Transcript.getCurrentSections`: the sections a transcript declares after replaying its system messages. `Transcript.diffSections(previous, current)`: the patch between two section sets, `null` removing one. - `filterGitHubCopilotModels` keeps the catalog models a Copilot login can reach. - GitHub Copilot GPT models turn reasoning off with `reasoning.effort: "none"` where OpenAI accepts it, such as gpt-5.3-codex, gpt-5.6-luna and gpt-6-luna. diff --git a/packages/aikit/src/cli/modelgen.ts b/packages/aikit/src/cli/modelgen.ts index 3c645331..86475aac 100644 --- a/packages/aikit/src/cli/modelgen.ts +++ b/packages/aikit/src/cli/modelgen.ts @@ -428,8 +428,9 @@ function applyAnthropicMetadata(model: Model.Info): void { mergeThinkingLevelMap(model, { off: null }); } if (ANTHROPIC_NO_TEMPERATURE.test(id)) mergeCompat(model, { supportsTemperature: false }); + // Copilot passes these system messages through but rejects tool changes in them. if (model.provider.id === "anthropic" && ANTHROPIC_MID_CONVO_SYSTEM_MESSAGES.test(id)) { - mergeCompat(model, { supportsMidConvoSystemMessages: true }); + mergeCompat(model, { supportsMidConvoSystemMessages: true, supportsAdditionalTools: true }); } } diff --git a/packages/aikit/src/llm/stream.ts b/packages/aikit/src/llm/stream.ts index cea730dc..48d9d305 100644 --- a/packages/aikit/src/llm/stream.ts +++ b/packages/aikit/src/llm/stream.ts @@ -1,7 +1,6 @@ import { APICallError } from "@ai-sdk/provider"; import { RetryError, streamText, type TextStreamPart, type ToolSet } from "ai"; import * as Message from "../message/message.ts"; -import * as Transcript from "../message/transcript.ts"; import * as Model from "../model/model.ts"; import { AssistantMessageEventStream } from "../utils/eventstream.ts"; import { compact } from "../utils/helpers.ts"; @@ -18,6 +17,7 @@ import { convertMessages, isOwnResponses, convertTools, + requestTools, createAssistantMessage, encodeOpenAIReasoningSignature, grammarToolInputProperties, @@ -416,7 +416,8 @@ export const stream: Protocol.StreamFunction diff --git a/packages/aikit/src/llm/transform.ts b/packages/aikit/src/llm/transform.ts index bde9edf2..167b3937 100644 --- a/packages/aikit/src/llm/transform.ts +++ b/packages/aikit/src/llm/transform.ts @@ -494,6 +494,63 @@ export function supportsMidConvoSystemMessages(model: Model.Info, method: Model. return isOwnResponses(model, method) || !isChatCompletions(model, resolveMethod(model, method)); } +/** + * Anthropic with tool changes in place: every tool the thread can reach is declared up front, the + * ones beyond its initial set deferred, and later changes name them by reference. Adding a tool + * to the top-level list costs the cached prefix; referencing a deferred one does not. + */ +function anthropicToolReferences(model: Model.Info, method: Model.APIMethodEnum, restricted: boolean): boolean { + return ( + !restricted && + isAnthropicWire(model, method) && + supportsMidConvoSystemMessages(model, method) && + model.compat?.supportsAdditionalTools === true + ); +} + +/** Messages before the first reply: on Anthropic their system messages join the leading prompt and tools. */ +function beforeFirstReply(messages: readonly Message.Message[]): Message.Message[] { + const reply = messages.findIndex((message) => message.role === "assistant"); + return reply === -1 ? [...messages] : messages.slice(0, reply); +} + +/** + * The tools the request declares top-level, and which of them are deferred. Usually every tool + * callable now. With Anthropic tool references, the initial set, then the deferred catalog and + * every other tool the transcript declared, deferred, in an order that holds across requests. + */ +export function requestTools( + context: Message.Context, + model: Model.Info, + methodOverride?: Model.APIMethodEnum, + activeTools?: readonly string[], +): { tools: Message.Tool[]; deferred: ReadonlySet } { + if (!anthropicToolReferences(model, resolveMethod(model, methodOverride), activeTools !== undefined)) { + return { tools: Transcript.getCurrentTools(context), deferred: new Set() }; + } + const initial = Transcript.getCurrentTools({ ...context, messages: beforeFirstReply(context.messages) }); + const latest = new Map(Transcript.getDeclaredTools(context).map((tool) => [tool.name, tool])); + const tools = new Map(initial.map((tool) => [tool.name, tool])); + const deferred = new Set(); + for (const tool of [...(context.deferredTools ?? []), ...latest.values()]) { + if (tools.has(tool.name)) continue; + tools.set(tool.name, latest.get(tool.name) ?? tool); + deferred.add(tool.name); + } + return { tools: [...tools.values()], deferred }; +} + +/** A later system message's tool changes as Anthropic tool references; a redefinition needs no removal. */ +function toolReferences(message: Message.SystemMessage) { + const added = new Set((message.toolsAdded ?? []).map((tool) => tool.name)); + return [ + ...(message.toolsRemoved ?? []) + .filter((tool) => !added.has(tool.name)) + .map((tool) => ({ type: "tool_removal" as const, toolName: tool.name })), + ...[...added].map((name) => ({ type: "tool_addition" as const, toolName: name })), + ]; +} + /** * The prompt the AI SDK adapters take as `system`: the leading one when changes go in place, * else the current one, every change folded in. @@ -576,6 +633,7 @@ export function convertMessages( // only after a user message, and never between a tool call and its result: each is held for // the next assistant message, or the end. const anthropic = isAnthropicWire(model, method); + const references = anthropicToolReferences(model, method, activeTools !== undefined); const leading: ModelMessage[] = []; const held: ModelMessage[] = []; let replied = false; @@ -584,6 +642,16 @@ export function convertMessages( if (msg.role === "system") { const index = systemIndex++; const update = index < prompt.foldedSystemMessages ? "" : Transcript.renderSystemPromptUpdate(msg); + // The tools of one before the first reply belong to the initial set. + const toolChanges = references && replied ? toolReferences(msg) : []; + if (toolChanges.length > 0) { + held.push({ + role: "system", + content: sanitizeSurrogates(update), + providerOptions: { [Model.optionsKey(model)]: { toolChanges } }, + }); + continue; + } const folded = index < transcript.foldedSystemMessages; const change = systemChangeMessage( update, @@ -646,17 +714,23 @@ export function convertTools( tools?: Message.Tool[], model?: Model.Info, method?: Model.APIMethodEnum, + deferred: ReadonlySet = new Set(), ): ToolSet | undefined { if (!tools || tools.length === 0) return; const result: ToolSet = {}; for (const tool of tools) { const { inputSchema, strict, providerOptions } = toolWire(tool, model, method); + const key = model && deferred.has(tool.name) ? Model.optionsKey(model) : undefined; + const options = + key === undefined + ? providerOptions + : { ...providerOptions, [key]: { ...providerOptions?.[key], deferLoading: true } }; result[tool.name] = { description: tool.description, inputSchema: jsonSchema(inputSchema as any), ...(strict ? { strict } : {}), - ...(providerOptions ? { providerOptions } : {}), + ...(options ? { providerOptions: options } : {}), }; } return result; diff --git a/packages/aikit/src/message/message.ts b/packages/aikit/src/message/message.ts index 026000a1..87f99629 100644 --- a/packages/aikit/src/message/message.ts +++ b/packages/aikit/src/message/message.ts @@ -333,6 +333,8 @@ export const ContextSchema = Type.Object({ systemPrompt: Type.Optional(Type.String()), messages: Type.Array(MessageSchema), tools: Type.Optional(Type.Array(ToolSchema)), + /** Tools a later system message may add, for transports that must declare every tool up front. */ + deferredTools: Type.Optional(Type.Array(ToolSchema)), }); export type Context = Static; diff --git a/packages/aikit/src/model/baseline.json b/packages/aikit/src/model/baseline.json index 916087af..a5ca950f 100644 --- a/packages/aikit/src/model/baseline.json +++ b/packages/aikit/src/model/baseline.json @@ -9,6 +9,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true }, "contextWindow": 1000000, @@ -52,6 +53,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true }, "contextWindow": 1000000, @@ -163,6 +165,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, @@ -370,6 +373,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, @@ -413,6 +417,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, @@ -456,6 +461,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, @@ -653,6 +659,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, @@ -6739,18 +6746,18 @@ "url": "https://openrouter.ai/api/v1" }, "baseUrl": "https://openrouter.ai/api/v1", - "contextWindow": 204800, + "contextWindow": 198000, "cost": { "cacheRead": 0.03, "cacheWrite": 0, - "input": 0.3, - "output": 1.2 + "input": 0.27, + "output": 0.95 }, "id": "minimax/minimax-m2.5", "input": [ "text" ], - "maxTokens": 131072, + "maxTokens": 32768, "name": "MiniMax: MiniMax M2.5", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -6847,10 +6854,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.0075, + "cacheRead": 0.015, "cacheWrite": 0, - "input": 0.075, - "output": 0.3 + "input": 0.15, + "output": 0.6 }, "id": "minimax/minimax-m3.1-flash", "input": [ @@ -11944,15 +11951,15 @@ "cost": { "cacheRead": 0, "cacheWrite": 0, - "input": 0.32, - "output": 3.2 + "input": 0.45, + "output": 2.7 }, "id": "qwen/qwen3.6-27b", "input": [ "text", "image" ], - "maxTokens": 81920, + "maxTokens": 65536, "name": "Qwen: Qwen3.6 27B", "npm": "@openrouter/ai-sdk-provider", "protocol": "openrouter", @@ -12918,10 +12925,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 262144, "cost": { - "cacheRead": 0.020625, + "cacheRead": 0.033, "cacheWrite": 0, - "input": 0.0825, - "output": 0.33 + "input": 0.132, + "output": 0.528 }, "id": "tencent/hy3", "input": [ @@ -13002,10 +13009,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.0378, + "cacheRead": 0.042, "cacheWrite": 0, - "input": 0.7506, - "output": 2.2509 + "input": 0.834, + "output": 2.501 }, "id": "tencent/hy4-preview", "input": [ @@ -14275,10 +14282,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.076, + "cacheRead": 0.0689, "cacheWrite": 0, - "input": 0.08, - "output": 3.6 + "input": 0.0699, + "output": 1.4 }, "id": "z-ai/glm-5.3", "input": [ @@ -14320,7 +14327,7 @@ "cacheRead": 0.0197, "cacheWrite": 0, "input": 0.0197, - "output": 0.1324 + "output": 0.1238 }, "id": "z-ai/glm-5.3-flash", "input": [ @@ -14660,10 +14667,10 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.004, + "cacheRead": 0.0143, "cacheWrite": 0, - "input": 0.022, - "output": 0.192 + "input": 0.0155, + "output": 0.19 }, "id": "~deepseek/deepseek-flash-latest", "input": [ @@ -14706,7 +14713,7 @@ "cacheRead": 0.1, "cacheWrite": 0, "input": 0.1035, - "output": 4 + "output": 8 }, "id": "~deepseek/deepseek-pro-latest", "input": [ @@ -14745,9 +14752,9 @@ "baseUrl": "https://openrouter.ai/api/v1", "contextWindow": 1048576, "cost": { - "cacheRead": 0.0442, + "cacheRead": 0.0378, "cacheWrite": 0, - "input": 0.0452, + "input": 0.0388, "output": 0.3 }, "id": "~deepseek/deepseek-v4-flash-latest", @@ -14875,8 +14882,8 @@ "cost": { "cacheRead": 0.15, "cacheWrite": 0, - "input": 0.1539, - "output": 10.5539 + "input": 0.1558, + "output": 10.5478 }, "id": "~moonshotai/kimi-latest", "input": [ @@ -15256,7 +15263,7 @@ "cacheRead": 0.0197, "cacheWrite": 0, "input": 0.0197, - "output": 0.1324 + "output": 0.1238 }, "id": "~z-ai/glm-flash-latest", "input": [ diff --git a/packages/aikit/src/model/model.ts b/packages/aikit/src/model/model.ts index 879a06cd..5cc4bae7 100644 --- a/packages/aikit/src/model/model.ts +++ b/packages/aikit/src/model/model.ts @@ -160,7 +160,10 @@ export const CompatibilitySchema = Type.Object({ /** The Google protocol accepts thinking levels rather than a token budget. */ supportsThinkingLevel: Type.Optional(Type.Boolean()), supportsToolSearch: Type.Optional(Type.Boolean()), - /** Requires `supportsMidConvoSystemMessages`. */ + /** + * Tool changes go in place: Responses `additional_tools` items, Anthropic tool references. + * Requires `supportsMidConvoSystemMessages`. + */ supportsAdditionalTools: Type.Optional(Type.Boolean()), /** The model accepts developer messages after the conversation has started. */ supportsMidConvoSystemMessages: Type.Optional(Type.Boolean()), diff --git a/packages/aikit/test/__artifacts__/modelgen.json b/packages/aikit/test/__artifacts__/modelgen.json index 9deb6d93..b27e677e 100644 --- a/packages/aikit/test/__artifacts__/modelgen.json +++ b/packages/aikit/test/__artifacts__/modelgen.json @@ -9,6 +9,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true }, "contextWindow": 1000000, @@ -52,6 +53,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true }, "contextWindow": 1000000, @@ -163,6 +165,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, @@ -370,6 +373,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, @@ -413,6 +417,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, @@ -456,6 +461,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, @@ -653,6 +659,7 @@ "baseUrl": "https://api.anthropic.com/v1", "compat": { "forceAdaptiveThinking": true, + "supportsAdditionalTools": true, "supportsMidConvoSystemMessages": true, "supportsTemperature": false }, diff --git a/packages/aikit/test/transcript.test.ts b/packages/aikit/test/transcript.test.ts index d4f931cc..2a9939f2 100644 --- a/packages/aikit/test/transcript.test.ts +++ b/packages/aikit/test/transcript.test.ts @@ -421,3 +421,65 @@ describe("transcript prompt changes on the Anthropic wire", () => { }); }); }); + +describe("transcript tool changes on the Anthropic wire", () => { + type ToolsBody = { + tools?: Array<{ name: string; defer_loading?: boolean }>; + messages: Array<{ role: string; content: Array<{ type: string; text?: string; tool?: { name: string } }> }>; + }; + const claude = makeModel({ + npm: "@ai-sdk/anthropic", + compat: { supportsMidConvoSystemMessages: true, supportsAdditionalTools: true }, + }); + const capture = async (messages: Message.Message[]) => { + const server = fakeAnthropic(textReply); + await stream( + claude, + { + systemPrompt: "base prompt", + tools: [baseTool, dropTool], + deferredTools: [lateTool, tool("spare_tool")], + messages, + }, + { apiKey: "test-key", factoryOptions: { fetch: server.fetch } }, + ).result(); + const body = server.body() as ToolsBody; + return { + tools: body.tools?.map((one) => `${one.name}${one.defer_loading ? " (deferred)" : ""}`), + messages: body.messages.map((message) => [ + message.role, + message.content + .map((part) => (part.tool ? `${part.type} ${part.tool.name}` : (part.text ?? part.type))) + .join(" | "), + ]), + }; + }; + const first = [makeUserMessage("first"), system({ toolsAdded: [tool("early_tool")] })]; + const toolTurn = makeAssistantMessage(claude, { stopReason: "toolUse", parts: [makeCompletedToolCall("call-1")] }); + const later = [ + toolTurn, + system({ toolsAdded: [lateTool], toolsRemoved: [{ name: "drop_tool" }] }), + makeUserMessage("second"), + ]; + + it("declares the catalog deferred up front and references a later change", async () => { + const before = await capture(first); + const after = await capture([...first, ...later]); + + // Every tool is declared from the first request on, so a later change keeps the list as it was. + expect(after.tools).toEqual(before.tools); + expect(after.tools).toEqual([ + "base_tool", + "drop_tool", + "early_tool", + "late_tool (deferred)", + "spare_tool (deferred)", + ]); + expect(after.messages).toEqual([ + ["user", "first"], + ["assistant", "tool_use"], + ["user", "tool_result | second"], + ["system", "tool_removal drop_tool | tool_addition late_tool"], + ]); + }); +}); diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index bbf001ff..29e89f0e 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -357,6 +357,7 @@ export const layer = (options: Options = {}) => systemPrompt: snapshot.prompt.foundation, messages: [...messages], tools: [...snapshot.tools.wire], + deferredTools: [...snapshot.tools.deferred], }, provider: snapshot.provider, model: snapshot.model, diff --git a/packages/harness/src/tool/registry.ts b/packages/harness/src/tool/registry.ts index bb9c1172..e9ca3427 100644 --- a/packages/harness/src/tool/registry.ts +++ b/packages/harness/src/tool/registry.ts @@ -39,6 +39,8 @@ export interface Resolved { readonly wire: ReadonlyArray; /** The offered tools' wire view: the tool state the transcript's system messages bring `wire` to. */ readonly offered: ReadonlyArray; + /** Every deferred tool in the catalog, loaded or not: what a transcript system message may add. */ + readonly deferred: ReadonlyArray; /** * Transform one complete pending tool-call part into a complete terminal part. The **only** * execution entry point — the @@ -86,6 +88,7 @@ export const make = (tools: ReadonlyArray): R const defs = Object.freeze(effective.map((entry) => entry.tool.definition)); const names = Object.freeze(effective.map((entry) => entry.tool.definition.name)); const isDeferred = (entry: ToolRegistration) => entry.tool.definition.exposure === "deferred"; + const catalog = Object.freeze(effective.filter(isDeferred).map((entry) => toAikitTool(entry.tool.definition))); const resolve = (selection?: Selection, live?: Effect.Effect>): Resolved => { const removed = new Set(selection?.remove); @@ -113,6 +116,7 @@ export const make = (tools: ReadonlyArray): R defs: Object.freeze(offered.map((entry) => entry.tool.definition)), wire: Object.freeze(declaredSet.map((entry) => toAikitTool(entry.tool.definition))), offered: Object.freeze([...executor.wire]), + deferred: catalog, handle: executor.handle, }); }; diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.live.anthropic.json b/packages/harness/test/__artifacts__/tools.toolsearch.live.anthropic.json new file mode 100644 index 00000000..acdce09e --- /dev/null +++ b/packages/harness/test/__artifacts__/tools.toolsearch.live.anthropic.json @@ -0,0 +1,52 @@ +{ + "requests": [ + { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch" + ], + "system": [ + [] + ] + }, + { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch" + ], + "system": [ + [], + [ + "vault_read_token" + ] + ] + }, + { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch" + ], + "system": [ + [], + [ + "vault_read_token" + ] + ] + } + ], + "calls": [ + "toolsearch:completed", + "vault_read_token:completed" + ], + "found": true, + "cachedAfterLoad": true +} diff --git a/packages/harness/test/tools.toolsearch.live.test.ts b/packages/harness/test/tools.toolsearch.live.test.ts index 4ff4bacf..7a1f1a39 100644 --- a/packages/harness/test/tools.toolsearch.live.test.ts +++ b/packages/harness/test/tools.toolsearch.live.test.ts @@ -17,11 +17,16 @@ import { available, LIVE } from "./utils/live.ts"; /** * One run against a real model: the token is held by a deferred tool the model is not offered, * so a right reply means it searched, the load reached its next request, and it called the tool. + * On Anthropic the deferred catalog is declared up front and the load names the tool by reference, + * so the request after the load reads the one before it from the cache. */ const TOKEN = "vq7-lumen-4821"; -const codex = LIVE.find((one) => one.name === "openai-codex")!; -const MODEL = "gpt-6-luna"; +process.env.CODEWORK_CACHE_RETENTION ??= "short"; +const CASES = [ + { live: LIVE.find((one) => one.name === "openai-codex")!, model: "gpt-6-luna", cache: false }, + { live: LIVE.find((one) => one.name === "anthropic")!, model: "claude-sonnet-5-5", cache: true }, +]; /** A request as sent: its declared tools, and the tools its system messages add. */ type Sent = { readonly tools: ReadonlyArray; readonly system: ReadonlyArray> }; @@ -32,12 +37,12 @@ const sentOf = (context: Message.Context): Sent => ({ ), }); -describe("toolsearch against live openai-codex", () => { - it.skipIf(!available(codex))("finds a deferred tool, then calls it", { timeout: 300_000 }, () => +describe.each(CASES)("toolsearch against live $live.name", ({ live, model, cache }) => { + it.skipIf(!available(live))("finds a deferred tool, then calls it", { timeout: 300_000 }, () => withSettings(async ({ root, custom }) => { await writeFile( join(custom, "settings.jsonc"), - JSON.stringify({ model: { provider: codex.provider, id: MODEL } }), + JSON.stringify({ model: { provider: live.provider, id: model } }), ); const vault = define({ id: "acme.tool.vault", @@ -65,7 +70,7 @@ describe("toolsearch against live openai-codex", () => { return LLM.open(input, signal); }; - const { calls, reply } = await Effect.gen(function* () { + const { calls, reply, cacheReads } = await Effect.gen(function* () { const handle = yield* Session.create({ directory: root }); yield* handle.main.run( "Find the tool that reads the deployment access token from the secrets vault, call it, " + @@ -82,6 +87,10 @@ describe("toolsearch against live openai-codex", () => { reply: (last === undefined ? [] : ContextCodec.parts(last)) .flatMap((part) => (part.type === "text" ? [part.text] : [])) .join(""), + // One assistant message per request, in order. + cacheReads: messages.flatMap((message) => + message.role === "assistant" ? [message.usage.cacheRead] : [], + ), }; }).pipe( Effect.provide( @@ -108,10 +117,12 @@ describe("toolsearch against live openai-codex", () => { expect(loadedAt).toBeGreaterThan(0); expect(sent.every((one) => !one.tools.includes("vault_read_token"))).toBe(true); expect(found).toBe(true); + const cachedAfterLoad = (cacheReads[loadedAt] ?? 0) > 0; + if (cache) expect(cachedAfterLoad).toBe(true); - await expect(`${JSON.stringify({ requests: sent, calls, found }, null, "\t")}\n`).toMatchFileSnapshot( - "./__artifacts__/tools.toolsearch.live.openai-codex.json", - ); + await expect( + `${JSON.stringify({ requests: sent, calls, found, ...(cache ? { cachedAfterLoad } : {}) }, null, "\t")}\n`, + ).toMatchFileSnapshot(`./__artifacts__/tools.toolsearch.live.${live.name}.json`); }), ); }); From e819b6d264ee0077dd6af64053b5e6995208ef59 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 31 Oct 2026 12:38:37 +0530 Subject: [PATCH 10/11] feat(harness): give a thread its own prompt foundation The agent document gains foundation: set through configure, it replaces the foundation prompt plugins render for that thread only, and null goes back to it. Forks take it as of their cut and it survives a restart, as the thread's model and thinking level do. thread.configure and session.create take it over RPC. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + packages/codework/src/server/contract.ts | 3 +- packages/codework/src/server/handlers.ts | 2 + packages/harness/src/document/schema.ts | 3 + packages/harness/src/effect/session.ts | 11 ++- packages/harness/src/state/state.ts | 4 + .../test/__artifacts__/prompt.foundation.json | 66 +++++++++++++ .../harness/test/prompt.foundation.test.ts | 95 +++++++++++++++++++ 8 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 packages/harness/test/__artifacts__/prompt.foundation.json create mode 100644 packages/harness/test/prompt.foundation.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 2240ab56..0b53b798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ This file is the canonical source for unreleased changes and published release n - MCP OAuth: servers sign in with discovery, client registration and PKCE, tokens stored in `~/.codework/mcpauth.json` by server name and URL and refreshed under a lock shared across processes. A server waiting for sign-in reconnects on the next message once its sign-in changes. `auth: { provider }` sends a model provider's credential instead. - The ephemeral `mcp.server.status` event reports each MCP server state change (`connecting`, `connected`, `disconnected`, `failed`, `authenticate`, `disabled`) to every thread using the server. A tool the MCP SDK rejects is skipped instead of failing its server, and listed as `skipped` on the `connected` status. - `toolsearch` turns itself on when a thread has deferred tools or an enabled MCP server with `deferred` exposure; a settings entry still turns it off or on. A search waits up to 10 seconds for such servers still connecting. +- A thread can have its own prompt foundation: `configure({ foundation })` records it in the thread's `agent` document, in place of the one prompt plugins render, and `foundation: null` goes back to that. Other threads keep theirs; a fork starts with the foundation of its cut, and the choice survives a restart. Each change leads the thread's next request with the new foundation, a cache miss once. `Session.create` takes `foundation` for the main thread. - The deferred tools of the catalog reach every request as `deferredTools`, so Anthropic declares them up front and loading one through `toolsearch` keeps the cached prefix. - Prompt sections reach the model as transcript system messages. A request sends the foundation as its system prompt; the first request of a thread appends a `system` entry declaring every tagged section (``, ``, ``, plugin sections), and a later request appends one only for the sections that changed, a removed section as `null`, together with any tool change. Resume and fork replay these entries, so the model sees what it saw before, and a model that takes system messages in place (OpenAI Codex, OpenAI and GitHub Copilot Responses, Anthropic and GitHub Copilot Claude) keeps its cached prefix when a section changes. @@ -63,6 +64,7 @@ This file is the canonical source for unreleased changes and published release n - Added the `thread.update` RPC (`{ sessionId, threadId?, messageId, delivery?, text? }`), which moves a queued prompt to the other lane or rewrites its text, keeping its place. `delivery: "steer"` also wakes the thread. `updated` is `false` when the prompt was already taken in. - `codework serve` resumes the sessions a crash or a shutdown cut off when it starts. Thread info carries `parked: true` for a thread that restarted too often to resume on its own; its next prompt runs it again. +- `thread.configure` and `session.create` take `runtime.foundation`, a thread's own prompt foundation; `null` on `thread.configure` goes back to the rendered one. - `codework run --thread ` continues one thread of `--session`. - `codework thread list ` lists a session's threads with their model, usage and title. `codework thread fork --at ` or `--before ` starts a new thread and prints its id, then for `--before` the text of the prompt it left out. `codework session relink --thread ` moves one thread to another checkout. All three take `--json`. - `ThreadInfo.usage` reports a thread's own cost and tokens. `event.subscribe` and `session.log` take `threadId` to keep only that thread's events. diff --git a/packages/codework/src/server/contract.ts b/packages/codework/src/server/contract.ts index 30fbea21..f815e4e6 100644 --- a/packages/codework/src/server/contract.ts +++ b/packages/codework/src/server/contract.ts @@ -146,10 +146,11 @@ export const LogSynced = Schema.Struct({ }); export type LogSynced = typeof LogSynced.Type; -/** A thread's model and thinking choices: a given field replaces the stored one, `null` clears it. */ +/** A thread's model, thinking and foundation choices: a given field replaces the stored one, `null` clears it. */ export const RuntimeConfig = Schema.Struct({ model: optional(Schema.NullOr(Schema.Struct({ provider: Schema.String, id: Schema.String }))), thinkingLevel: optional(Schema.NullOr(Schema.Enum(Model.ThinkingLevelEnum))), + foundation: optional(Schema.NullOr(Schema.NonEmptyString)), }); export type RuntimeConfig = typeof RuntimeConfig.Type; diff --git a/packages/codework/src/server/handlers.ts b/packages/codework/src/server/handlers.ts index 85c47814..b907582f 100644 --- a/packages/codework/src/server/handlers.ts +++ b/packages/codework/src/server/handlers.ts @@ -74,6 +74,7 @@ const toThreadInfo = (info: Session.ThreadInfo): ThreadInfo => ({ const runtimeInput = (runtime: RuntimeConfig | undefined): Session.RuntimeInput => ({ ...(runtime?.model == null ? {} : { model: runtime.model }), ...(runtime?.thinkingLevel == null ? {} : { thinkingLevel: runtime.thinkingLevel }), + ...(runtime?.foundation == null ? {} : { foundation: runtime.foundation }), }); const threadInfo = (threadId: ThreadSchema.ID) => Session.threadInfo(threadId).pipe(Effect.map(toThreadInfo)); @@ -239,6 +240,7 @@ export const layer = Contract.Api.toLayer( const change = { ...(runtime.model === undefined ? {} : { model: runtime.model }), ...(runtime.thinkingLevel === undefined ? {} : { thinkingLevel: runtime.thinkingLevel }), + ...(runtime.foundation === undefined ? {} : { foundation: runtime.foundation }), }; if (Object.keys(change).length > 0) { yield* events.publish(EventList.ThreadConfigured, { diff --git a/packages/harness/src/document/schema.ts b/packages/harness/src/document/schema.ts index 90725581..1e6abbec 100644 --- a/packages/harness/src/document/schema.ts +++ b/packages/harness/src/document/schema.ts @@ -48,6 +48,8 @@ export const Agent = Schema.Struct({ modelTools: optional(ModelTools), /** Physical boundary override; absent = the session's. */ placement: optional(Placement), + /** The prompt foundation for this thread, in place of the one prompt plugins render. */ + foundation: optional(Schema.NonEmptyString), }); export type Agent = typeof Agent.Type; @@ -57,6 +59,7 @@ export const AgentChange = Schema.Struct({ thinkingLevel: optional(Schema.NullOr(ThinkingLevel)), modelTools: optional(Schema.NullOr(ModelTools)), placement: optional(Schema.NullOr(Placement)), + foundation: optional(Schema.NullOr(Schema.NonEmptyString)), }); export type AgentChange = typeof AgentChange.Type; diff --git a/packages/harness/src/effect/session.ts b/packages/harness/src/effect/session.ts index 971de97a..5dad0bc9 100644 --- a/packages/harness/src/effect/session.ts +++ b/packages/harness/src/effect/session.ts @@ -16,7 +16,7 @@ import { PromptSchema } from "../session/prompt/schema.ts"; import * as SessionRuntime from "../session/runtime.ts"; import type { ThreadRow } from "../db/schema.sql.ts"; import { Document } from "../document/document.ts"; -import type { DocumentSchema } from "../document/schema.ts"; +import { DocumentSchema } from "../document/schema.ts"; import { SessionSchema } from "../session/schema.ts"; import { Session as SessionStore } from "../session/session.ts"; import { ThreadSchema } from "../thread/schema.ts"; @@ -43,6 +43,8 @@ export interface SystemPromptConfig { export interface RuntimeInput { readonly model?: ModelConfig; readonly thinkingLevel?: Model.ThinkingLevel; + /** The thread's own prompt foundation, in place of the rendered one; `null` goes back to it. */ + readonly foundation?: string | null; readonly tools?: ToolsConfig; readonly systemPrompt?: SystemPromptConfig; } @@ -158,7 +160,7 @@ export interface ThreadHandle { readonly resume: () => ReturnType; /** Stops active work and waits for its cleanup; idle is a no-op returning false. */ readonly interrupt: () => ReturnType; - /** Records model and thinking choices in the thread's `agent` document. */ + /** Records model, thinking and foundation choices in the thread's `agent` document. */ readonly configure: (input: RuntimeInput) => Effect.Effect; /** A new thread cut at a committed entry of this one's line; `text` refills the editor for `before`. */ readonly fork: ( @@ -196,9 +198,10 @@ const runtimeBindings = (input: RuntimeInput): SessionRuntime.Bindings => ({ }); /** The `agent` fields this input names. */ -const chosen = (input: RuntimeInput): DocumentSchema.Agent => ({ +const chosen = (input: RuntimeInput): DocumentSchema.AgentChange => ({ ...(input.model === undefined ? {} : { model: { provider: input.model.provider, id: input.model.id } }), ...(input.thinkingLevel === undefined ? {} : { thinkingLevel: input.thinkingLevel }), + ...(input.foundation === undefined ? {} : { foundation: input.foundation }), }); /** Records a choice as one `agent` revision; nothing when the input names none. */ @@ -405,7 +408,7 @@ export const create = Effect.fn("Session.create")(function* (input: CreateInput directory: location.directory, ...(input.title === undefined ? {} : { title: input.title }), ...(input.hostDir === undefined ? {} : { hostDir: declaredHostDir(input.hostDir) }), - agent: chosen(input), + agent: DocumentSchema.applyAgentChange({}, chosen(input)), }); yield* runtime.set(id, runtimeBindings(input)); return yield* makeHandle(id).pipe(Effect.orDie); diff --git a/packages/harness/src/state/state.ts b/packages/harness/src/state/state.ts index 07890514..9281db02 100644 --- a/packages/harness/src/state/state.ts +++ b/packages/harness/src/state/state.ts @@ -524,6 +524,10 @@ export const layer = ( sandbox, location, ...contributions, + prompt: + agent.foundation === undefined + ? contributions.prompt + : { ...contributions.prompt, foundation: agent.foundation }, provider, model, resolvedModel, diff --git a/packages/harness/test/__artifacts__/prompt.foundation.json b/packages/harness/test/__artifacts__/prompt.foundation.json new file mode 100644 index 00000000..bda9ec5c --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.foundation.json @@ -0,0 +1,66 @@ +[ + { + "step": "before any choice", + "thread": "main", + "request": { + "foundation": "default", + "declarations": 1 + } + }, + { + "step": "after configuring main", + "thread": "main", + "request": { + "foundation": "reviewer", + "declarations": 1 + } + }, + { + "step": "fork cut before the choice", + "thread": "early fork", + "request": { + "foundation": "default", + "declarations": 1 + } + }, + { + "step": "fork cut after the choice", + "thread": "late fork", + "request": { + "foundation": "reviewer", + "declarations": 1 + } + }, + { + "step": "after configuring the fork", + "thread": "late fork", + "request": { + "foundation": "scout", + "declarations": 1 + } + }, + { + "step": "main after the fork's choice", + "thread": "main", + "request": { + "foundation": "reviewer", + "declarations": 1 + } + }, + { + "step": "main after a restart", + "thread": "main", + "request": { + "foundation": "reviewer", + "declarations": 1 + } + }, + { + "step": "main after clearing", + "thread": "main", + "request": { + "foundation": "default", + "declarations": 1 + } + } +] diff --git a/packages/harness/test/prompt.foundation.test.ts b/packages/harness/test/prompt.foundation.test.ts new file mode 100644 index 00000000..7c526fc4 --- /dev/null +++ b/packages/harness/test/prompt.foundation.test.ts @@ -0,0 +1,95 @@ +import type { Message } from "@codeworksh/aikit"; +import "./utils/env.ts"; +import { Effect } from "effect"; +import { join } from "node:path"; +import { describe, expect, it } from "vite-plus/test"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import { defaultPromptPlugin } from "../src/plugin/builtin/prompt/default.ts"; +import { immediateOpen } from "./fixtures/llm.ts"; +import { withSettings } from "./fixtures/settings.ts"; + +const REVIEWER = "You are a code reviewer."; +const SCOUT = "You are a scout."; + +/** The foundation a request led with, by the persona it names, and how many section declarations it carried. */ +const request = (context: Message.Context) => ({ + foundation: context.systemPrompt === REVIEWER ? "reviewer" : context.systemPrompt === SCOUT ? "scout" : "default", + declarations: context.messages.filter((message) => message.role === "system").length, +}); + +describe("a thread's own prompt foundation", () => { + it("leads only that thread's requests, follows forks as of the cut, survives a restart and clears", () => + withSettings(async ({ root }) => { + const contexts: Message.Context[] = []; + // One open for both runtimes: its reply ids count on across them. + const open = immediateOpen(contexts); + const runtime = () => + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: join(root, "sessions.db"), + llm: open, + plugins: [defaultPromptPlugin], + }); + const steps: Array<{ step: string; thread: string; request: ReturnType }> = []; + const ran = (step: string, thread: string) => steps.push({ step, thread, request: request(contexts.at(-1)!) }); + + const sessionId = await Effect.runPromise( + Effect.gen(function* () { + const { id, main } = yield* Session.create({ directory: root }); + yield* main.run("one"); + ran("before any choice", "main"); + const firstReply = (yield* main.history()).find(({ entry }) => entry.type === "assistant")!; + + yield* main.configure({ foundation: REVIEWER }); + yield* main.run("two"); + ran("after configuring main", "main"); + + // Cut before the choice: the fork has the foundation of that point. + const { thread: early } = yield* main.fork({ entryId: firstReply.entry.id }); + yield* early.run("three"); + ran("fork cut before the choice", "early fork"); + + // Cut after it: the fork starts with main's choice, then makes its own. + const latest = (yield* main.history()).findLast(({ entry }) => entry.type === "assistant")!; + const { thread: late } = yield* main.fork({ entryId: latest.entry.id }); + yield* late.run("four"); + ran("fork cut after the choice", "late fork"); + yield* late.configure({ foundation: SCOUT }); + yield* late.run("five"); + ran("after configuring the fork", "late fork"); + yield* main.run("six"); + ran("main after the fork's choice", "main"); + return id; + }).pipe(Effect.provide(runtime()), Effect.scoped), + ); + + await Effect.runPromise( + Effect.gen(function* () { + const main = yield* Session.attach({ sessionId }); + yield* main.run("seven"); + ran("main after a restart", "main"); + yield* main.configure({ foundation: null }); + yield* main.run("eight"); + ran("main after clearing", "main"); + }).pipe(Effect.provide(runtime()), Effect.scoped), + ); + + expect(steps.map(({ request }) => request.foundation)).toEqual([ + "default", + "reviewer", + "default", + "reviewer", + "scout", + "reviewer", + "reviewer", + "default", + ]); + // A foundation choice declares no section again: each thread's declarations stay at one. + expect(steps.every(({ request }) => request.declarations === 1)).toBe(true); + await expect(`${JSON.stringify(steps, null, "\t")}\n`).toMatchFileSnapshot( + "./__artifacts__/prompt.foundation.json", + ); + })); +}); From bb24cbce93a2d47465bc8362a974616e604eec8e Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 31 Oct 2026 13:03:17 +0530 Subject: [PATCH 11/11] feat: report a thread's foundation in its info, and test it live ThreadInfo carries foundation, in the harness and over RPC, so a client reads back what it set. A live test runs a session on a buggy function on Anthropic, OpenAI, Codex and Copilot Claude: main explains it with the default foundation, a reviewer fork and a tester fork each answer in their own role, and main keeps its own after them. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- packages/codework/src/server/contract.ts | 2 + packages/codework/src/server/handlers.ts | 1 + packages/harness/src/effect/session.ts | 3 + .../test/__artifacts__/prompt.foundation.json | 24 ++- .../prompt.foundation.live.anthropic.json | 19 +++ ...dation.live.copilot-claude-sonnet-5.5.json | 19 +++ .../prompt.foundation.live.openai-codex.json | 19 +++ .../prompt.foundation.live.openai.json | 19 +++ .../test/prompt.foundation.live.test.ts | 137 ++++++++++++++++++ .../harness/test/prompt.foundation.test.ts | 39 +++-- 11 files changed, 264 insertions(+), 22 deletions(-) create mode 100644 packages/harness/test/__artifacts__/prompt.foundation.live.anthropic.json create mode 100644 packages/harness/test/__artifacts__/prompt.foundation.live.copilot-claude-sonnet-5.5.json create mode 100644 packages/harness/test/__artifacts__/prompt.foundation.live.openai-codex.json create mode 100644 packages/harness/test/__artifacts__/prompt.foundation.live.openai.json create mode 100644 packages/harness/test/prompt.foundation.live.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b53b798..44b8865d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ This file is the canonical source for unreleased changes and published release n - MCP OAuth: servers sign in with discovery, client registration and PKCE, tokens stored in `~/.codework/mcpauth.json` by server name and URL and refreshed under a lock shared across processes. A server waiting for sign-in reconnects on the next message once its sign-in changes. `auth: { provider }` sends a model provider's credential instead. - The ephemeral `mcp.server.status` event reports each MCP server state change (`connecting`, `connected`, `disconnected`, `failed`, `authenticate`, `disabled`) to every thread using the server. A tool the MCP SDK rejects is skipped instead of failing its server, and listed as `skipped` on the `connected` status. - `toolsearch` turns itself on when a thread has deferred tools or an enabled MCP server with `deferred` exposure; a settings entry still turns it off or on. A search waits up to 10 seconds for such servers still connecting. -- A thread can have its own prompt foundation: `configure({ foundation })` records it in the thread's `agent` document, in place of the one prompt plugins render, and `foundation: null` goes back to that. Other threads keep theirs; a fork starts with the foundation of its cut, and the choice survives a restart. Each change leads the thread's next request with the new foundation, a cache miss once. `Session.create` takes `foundation` for the main thread. +- A thread can have its own prompt foundation: `configure({ foundation })` records it in the thread's `agent` document, in place of the one prompt plugins render, and `foundation: null` goes back to that. Other threads keep theirs; a fork starts with the foundation of its cut, and the choice survives a restart. Each change leads the thread's next request with the new foundation, a cache miss once. `Session.create` takes `foundation` for the main thread, and `Session.ThreadInfo.foundation` reports a thread's choice. - The deferred tools of the catalog reach every request as `deferredTools`, so Anthropic declares them up front and loading one through `toolsearch` keeps the cached prefix. - Prompt sections reach the model as transcript system messages. A request sends the foundation as its system prompt; the first request of a thread appends a `system` entry declaring every tagged section (``, ``, ``, plugin sections), and a later request appends one only for the sections that changed, a removed section as `null`, together with any tool change. Resume and fork replay these entries, so the model sees what it saw before, and a model that takes system messages in place (OpenAI Codex, OpenAI and GitHub Copilot Responses, Anthropic and GitHub Copilot Claude) keeps its cached prefix when a section changes. @@ -64,7 +64,7 @@ This file is the canonical source for unreleased changes and published release n - Added the `thread.update` RPC (`{ sessionId, threadId?, messageId, delivery?, text? }`), which moves a queued prompt to the other lane or rewrites its text, keeping its place. `delivery: "steer"` also wakes the thread. `updated` is `false` when the prompt was already taken in. - `codework serve` resumes the sessions a crash or a shutdown cut off when it starts. Thread info carries `parked: true` for a thread that restarted too often to resume on its own; its next prompt runs it again. -- `thread.configure` and `session.create` take `runtime.foundation`, a thread's own prompt foundation; `null` on `thread.configure` goes back to the rendered one. +- `thread.configure` and `session.create` take `runtime.foundation`, a thread's own prompt foundation; `null` on `thread.configure` goes back to the rendered one. `ThreadInfo.foundation` reports it. - `codework run --thread ` continues one thread of `--session`. - `codework thread list ` lists a session's threads with their model, usage and title. `codework thread fork --at ` or `--before ` starts a new thread and prints its id, then for `--before` the text of the prompt it left out. `codework session relink --thread ` moves one thread to another checkout. All three take `--json`. - `ThreadInfo.usage` reports a thread's own cost and tokens. `event.subscribe` and `session.log` take `threadId` to keep only that thread's events. diff --git a/packages/codework/src/server/contract.ts b/packages/codework/src/server/contract.ts index f815e4e6..d2a2295e 100644 --- a/packages/codework/src/server/contract.ts +++ b/packages/codework/src/server/contract.ts @@ -89,6 +89,8 @@ export const ThreadInfo = Schema.Struct({ model: optional(Schema.Struct({ provider: Schema.String, id: Schema.String })), /** The thinking level chosen for this thread; absent when settings decide. */ thinkingLevel: optional(Schema.Enum(Model.ThinkingLevelEnum)), + /** The prompt foundation chosen for this thread; absent when prompt plugins render it. */ + foundation: optional(Schema.String), /** Where it runs instead of the session's space and directory. */ placement: optional(Schema.Struct({ spaceId: Schema.String, directory: Session.AbsolutePath })), /** Its unfinished run was cut off by restarts too often to resume on its own; the next prompt runs it. */ diff --git a/packages/codework/src/server/handlers.ts b/packages/codework/src/server/handlers.ts index b907582f..0a6e17c2 100644 --- a/packages/codework/src/server/handlers.ts +++ b/packages/codework/src/server/handlers.ts @@ -63,6 +63,7 @@ const toThreadInfo = (info: Session.ThreadInfo): ThreadInfo => ({ updatedAt: info.updatedAt, ...(info.model === undefined ? {} : { model: info.model }), ...(info.thinkingLevel === undefined ? {} : { thinkingLevel: info.thinkingLevel }), + ...(info.foundation === undefined ? {} : { foundation: info.foundation }), ...(info.placement === undefined ? {} : { placement: info.placement }), ...(info.parked ? { parked: true } : {}), usage: info.usage, diff --git a/packages/harness/src/effect/session.ts b/packages/harness/src/effect/session.ts index 5dad0bc9..bdd8c7e1 100644 --- a/packages/harness/src/effect/session.ts +++ b/packages/harness/src/effect/session.ts @@ -124,6 +124,8 @@ export interface ThreadInfo { readonly model?: { readonly provider: string; readonly id: string }; /** The thinking level chosen for this thread over settings; absent when settings decide. */ readonly thinkingLevel?: Model.ThinkingLevel; + /** The prompt foundation chosen for this thread; absent when prompt plugins render it. */ + readonly foundation?: string; /** Where it runs instead of the session's space and directory. */ readonly placement?: DocumentSchema.Placement; /** @@ -293,6 +295,7 @@ export const threadInfo = Effect.fn("Session.threadInfo")(function* (id: ThreadS updatedAt: DateTime.toEpochMillis(row.updatedAt), ...(agent.model === undefined ? {} : { model: agent.model }), ...(agent.thinkingLevel === undefined ? {} : { thinkingLevel: agent.thinkingLevel }), + ...(agent.foundation === undefined ? {} : { foundation: agent.foundation }), ...(agent.placement === undefined ? {} : { placement: agent.placement }), parked, usage: { diff --git a/packages/harness/test/__artifacts__/prompt.foundation.json b/packages/harness/test/__artifacts__/prompt.foundation.json index bda9ec5c..886be706 100644 --- a/packages/harness/test/__artifacts__/prompt.foundation.json +++ b/packages/harness/test/__artifacts__/prompt.foundation.json @@ -5,7 +5,8 @@ "request": { "foundation": "default", "declarations": 1 - } + }, + "info": "default" }, { "step": "after configuring main", @@ -13,7 +14,8 @@ "request": { "foundation": "reviewer", "declarations": 1 - } + }, + "info": "reviewer" }, { "step": "fork cut before the choice", @@ -21,7 +23,8 @@ "request": { "foundation": "default", "declarations": 1 - } + }, + "info": "default" }, { "step": "fork cut after the choice", @@ -29,7 +32,8 @@ "request": { "foundation": "reviewer", "declarations": 1 - } + }, + "info": "reviewer" }, { "step": "after configuring the fork", @@ -37,7 +41,8 @@ "request": { "foundation": "scout", "declarations": 1 - } + }, + "info": "scout" }, { "step": "main after the fork's choice", @@ -45,7 +50,8 @@ "request": { "foundation": "reviewer", "declarations": 1 - } + }, + "info": "reviewer" }, { "step": "main after a restart", @@ -53,7 +59,8 @@ "request": { "foundation": "reviewer", "declarations": 1 - } + }, + "info": "reviewer" }, { "step": "main after clearing", @@ -61,6 +68,7 @@ "request": { "foundation": "default", "declarations": 1 - } + }, + "info": "default" } ] diff --git a/packages/harness/test/__artifacts__/prompt.foundation.live.anthropic.json b/packages/harness/test/__artifacts__/prompt.foundation.live.anthropic.json new file mode 100644 index 00000000..3c0e48dd --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.foundation.live.anthropic.json @@ -0,0 +1,19 @@ +{ + "replies": { + "main": "none", + "reviewer": "REVIEW:", + "tester": "TESTS:", + "mainAfterForks": "none" + }, + "requestFoundations": [ + "default", + "reviewer", + "tester", + "default" + ], + "info": { + "main": "default", + "reviewer": "reviewer", + "tester": "tester" + } +} diff --git a/packages/harness/test/__artifacts__/prompt.foundation.live.copilot-claude-sonnet-5.5.json b/packages/harness/test/__artifacts__/prompt.foundation.live.copilot-claude-sonnet-5.5.json new file mode 100644 index 00000000..3c0e48dd --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.foundation.live.copilot-claude-sonnet-5.5.json @@ -0,0 +1,19 @@ +{ + "replies": { + "main": "none", + "reviewer": "REVIEW:", + "tester": "TESTS:", + "mainAfterForks": "none" + }, + "requestFoundations": [ + "default", + "reviewer", + "tester", + "default" + ], + "info": { + "main": "default", + "reviewer": "reviewer", + "tester": "tester" + } +} diff --git a/packages/harness/test/__artifacts__/prompt.foundation.live.openai-codex.json b/packages/harness/test/__artifacts__/prompt.foundation.live.openai-codex.json new file mode 100644 index 00000000..3c0e48dd --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.foundation.live.openai-codex.json @@ -0,0 +1,19 @@ +{ + "replies": { + "main": "none", + "reviewer": "REVIEW:", + "tester": "TESTS:", + "mainAfterForks": "none" + }, + "requestFoundations": [ + "default", + "reviewer", + "tester", + "default" + ], + "info": { + "main": "default", + "reviewer": "reviewer", + "tester": "tester" + } +} diff --git a/packages/harness/test/__artifacts__/prompt.foundation.live.openai.json b/packages/harness/test/__artifacts__/prompt.foundation.live.openai.json new file mode 100644 index 00000000..3c0e48dd --- /dev/null +++ b/packages/harness/test/__artifacts__/prompt.foundation.live.openai.json @@ -0,0 +1,19 @@ +{ + "replies": { + "main": "none", + "reviewer": "REVIEW:", + "tester": "TESTS:", + "mainAfterForks": "none" + }, + "requestFoundations": [ + "default", + "reviewer", + "tester", + "default" + ], + "info": { + "main": "default", + "reviewer": "reviewer", + "tester": "tester" + } +} diff --git a/packages/harness/test/prompt.foundation.live.test.ts b/packages/harness/test/prompt.foundation.live.test.ts new file mode 100644 index 00000000..afd52f24 --- /dev/null +++ b/packages/harness/test/prompt.foundation.live.test.ts @@ -0,0 +1,137 @@ +import { Effect } from "effect"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vite-plus/test"; +import { ContextCodec } from "../src/context/codec.ts"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import { SessionRuntime } from "../src/session/runtime.ts"; +import type { State } from "../src/state/state.ts"; +import { withSettings } from "./fixtures/settings.ts"; +import "./utils/live.ts"; + +/** + * One session on a buggy function, three threads: main explains it with the default foundation, + * a reviewer fork and a tester fork each lead with their own. Each reply opens with its thread's + * marker, every request carries its own thread's foundation, and the thread info reports it. + */ + +const REVIEWER = + "You are a strict code reviewer. Begin every reply with the line `REVIEW:`, then list the concrete defects you find."; +const TESTER = + "You write unit tests. Begin every reply with the line `TESTS:`, then give the test code and nothing else."; +const CODE = "function average(xs) { return xs.reduce((a, b) => a + b) / xs.length; }"; + +const CASES = [ + { name: "anthropic", provider: "anthropic", id: "claude-sonnet-5-5", env: "ANTHROPIC_API_KEY" }, + { name: "openai", provider: "openai", id: "gpt-6-luna", env: "OPENAI_API_KEY" }, + { name: "openai-codex", provider: "openai-codex", id: "gpt-6-luna", env: "OPENAI_CODEX_API_KEY" }, + { + name: "copilot-claude-sonnet-5.5", + provider: "github-copilot", + id: "claude-sonnet-5.5", + env: "COPILOT_GITHUB_TOKEN", + }, +] as const; + +describe.each(CASES)("thread foundations against live $name", ({ name, provider, id, env }) => { + it.skipIf(!process.env[env]?.trim())("leads each forked thread with its own foundation", { timeout: 300_000 }, () => + withSettings(async ({ root, custom }) => { + await writeFile(join(custom, "settings.jsonc"), JSON.stringify({ model: { provider, id } })); + const bodies: string[] = []; + const onPayload: NonNullable = async (payload) => { + bodies.push(JSON.stringify(payload)); + return payload; + }; + /** The text of a thread's last reply. */ + const lastReply = (thread: Session.ThreadHandle) => + Effect.gen(function* () { + const messages = yield* Effect.forEach(yield* thread.history(), ContextCodec.decodeMessage); + const reply = messages.findLast((message) => message.role === "assistant")!; + return ContextCodec.parts(reply) + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join("") + .trim(); + }); + + const result = await Effect.gen(function* () { + const handle = yield* Session.create({ directory: root }); + const runtime = yield* SessionRuntime.Service; + yield* runtime.update(handle.id, { onPayload }); + const main = handle.main; + yield* main.run(`Here is my function:\n\n${CODE}\n\nIn one sentence, what does it do?`); + const explained = (yield* main.history()).findLast(({ entry }) => entry.type === "assistant")!; + const explanation = yield* lastReply(main); + + const { thread: reviewer } = yield* main.fork({ entryId: explained.entry.id }); + yield* reviewer.configure({ foundation: REVIEWER }); + yield* reviewer.run("Review the function."); + const review = yield* lastReply(reviewer); + + const { thread: tester } = yield* main.fork({ entryId: explained.entry.id }); + yield* tester.configure({ foundation: TESTER }); + yield* tester.run("Write one test for the input that breaks it."); + const tests = yield* lastReply(tester); + + yield* main.run("Thanks. Is there anything to watch out for when calling it?"); + const followUp = yield* lastReply(main); + + const info = { + main: (yield* main.info).foundation, + reviewer: (yield* reviewer.info).foundation, + tester: (yield* tester.info).foundation, + }; + return { explanation, review, tests, followUp, info }; + }).pipe( + Effect.provide( + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: ":memory:", + userConfigDir: custom, + plugins: [], + }), + ), + Effect.scoped, + Effect.timeout("240 seconds"), + Effect.runPromise, + ); + + const opensWith = (text: string) => + text.startsWith("REVIEW:") ? "REVIEW:" : text.startsWith("TESTS:") ? "TESTS:" : "none"; + // Requests in order: main, reviewer, tester, main. + const foundations = bodies.map((body) => + body.includes("strict code reviewer") + ? "reviewer" + : body.includes("You write unit tests") + ? "tester" + : "default", + ); + const artifact = { + replies: { + main: opensWith(result.explanation), + reviewer: opensWith(result.review), + tester: opensWith(result.tests), + mainAfterForks: opensWith(result.followUp), + }, + requestFoundations: foundations, + info: { + main: result.info.main === undefined ? "default" : "own", + reviewer: result.info.reviewer === REVIEWER ? "reviewer" : "other", + tester: result.info.tester === TESTER ? "tester" : "other", + }, + }; + expect(artifact.replies).toEqual({ + main: "none", + reviewer: "REVIEW:", + tester: "TESTS:", + mainAfterForks: "none", + }); + expect(foundations).toEqual(["default", "reviewer", "tester", "default"]); + expect(artifact.info).toEqual({ main: "default", reviewer: "reviewer", tester: "tester" }); + await expect(`${JSON.stringify(artifact, null, "\t")}\n`).toMatchFileSnapshot( + `./__artifacts__/prompt.foundation.live.${name}.json`, + ); + }), + ); +}); diff --git a/packages/harness/test/prompt.foundation.test.ts b/packages/harness/test/prompt.foundation.test.ts index 7c526fc4..4b0839f6 100644 --- a/packages/harness/test/prompt.foundation.test.ts +++ b/packages/harness/test/prompt.foundation.test.ts @@ -12,9 +12,13 @@ import { withSettings } from "./fixtures/settings.ts"; const REVIEWER = "You are a code reviewer."; const SCOUT = "You are a scout."; -/** The foundation a request led with, by the persona it names, and how many section declarations it carried. */ +/** A foundation by the persona it names; plugins render the default. */ +const persona = (foundation: string | undefined) => + foundation === REVIEWER ? "reviewer" : foundation === SCOUT ? "scout" : "default"; + +/** The foundation a request led with, and how many section declarations it carried. */ const request = (context: Message.Context) => ({ - foundation: context.systemPrompt === REVIEWER ? "reviewer" : context.systemPrompt === SCOUT ? "scout" : "default", + foundation: persona(context.systemPrompt), declarations: context.messages.filter((message) => message.role === "system").length, }); @@ -32,35 +36,44 @@ describe("a thread's own prompt foundation", () => { llm: open, plugins: [defaultPromptPlugin], }); - const steps: Array<{ step: string; thread: string; request: ReturnType }> = []; - const ran = (step: string, thread: string) => steps.push({ step, thread, request: request(contexts.at(-1)!) }); + const steps: Array<{ + step: string; + thread: string; + request: ReturnType; + info: ReturnType; + }> = []; + /** What the thread's last request led with, and what its info reports. */ + const ran = (step: string, thread: string, handle: Session.ThreadHandle) => + Effect.map(handle.info, (info) => + steps.push({ step, thread, request: request(contexts.at(-1)!), info: persona(info.foundation) }), + ); const sessionId = await Effect.runPromise( Effect.gen(function* () { const { id, main } = yield* Session.create({ directory: root }); yield* main.run("one"); - ran("before any choice", "main"); + yield* ran("before any choice", "main", main); const firstReply = (yield* main.history()).find(({ entry }) => entry.type === "assistant")!; yield* main.configure({ foundation: REVIEWER }); yield* main.run("two"); - ran("after configuring main", "main"); + yield* ran("after configuring main", "main", main); // Cut before the choice: the fork has the foundation of that point. const { thread: early } = yield* main.fork({ entryId: firstReply.entry.id }); yield* early.run("three"); - ran("fork cut before the choice", "early fork"); + yield* ran("fork cut before the choice", "early fork", early); // Cut after it: the fork starts with main's choice, then makes its own. const latest = (yield* main.history()).findLast(({ entry }) => entry.type === "assistant")!; const { thread: late } = yield* main.fork({ entryId: latest.entry.id }); yield* late.run("four"); - ran("fork cut after the choice", "late fork"); + yield* ran("fork cut after the choice", "late fork", late); yield* late.configure({ foundation: SCOUT }); yield* late.run("five"); - ran("after configuring the fork", "late fork"); + yield* ran("after configuring the fork", "late fork", late); yield* main.run("six"); - ran("main after the fork's choice", "main"); + yield* ran("main after the fork's choice", "main", main); return id; }).pipe(Effect.provide(runtime()), Effect.scoped), ); @@ -69,10 +82,10 @@ describe("a thread's own prompt foundation", () => { Effect.gen(function* () { const main = yield* Session.attach({ sessionId }); yield* main.run("seven"); - ran("main after a restart", "main"); + yield* ran("main after a restart", "main", main); yield* main.configure({ foundation: null }); yield* main.run("eight"); - ran("main after clearing", "main"); + yield* ran("main after clearing", "main", main); }).pipe(Effect.provide(runtime()), Effect.scoped), ); @@ -86,6 +99,8 @@ describe("a thread's own prompt foundation", () => { "reviewer", "default", ]); + // Each thread's info reports the foundation its request led with. + expect(steps.every(({ request, info }) => request.foundation === info)).toBe(true); // A foundation choice declares no section again: each thread's declarations stay at one. expect(steps.every(({ request }) => request.declarations === 1)).toBe(true); await expect(`${JSON.stringify(steps, null, "\t")}\n`).toMatchFileSnapshot(