From f5436463dfe367807d152940442b46c1f373326d Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 31 Oct 2026 15:33:12 +0530 Subject: [PATCH] feat(harness): list deferred MCP servers in an mcpservers prompt section Servers whose tools are not declared to the model are listed in a new harness-rendered `mcpservers` section, each with how its tools are reached (`toolsearch`) and a one-line summary: the configured description, or the first line of the server's instructions once it connected. The section is worked out after the startup wait for servers with direct tools and held for the rest of a run, so a server that connects mid-run shows at the next prompt, as a system entry carrying only that section. It is left out while toolsearch is not declared, and a server disconnected on request is dropped from it. Servers whose tools are all codemode are not listed, since nothing reaches them yet. Co-Authored-By: Claude Opus 5.5 --- packages/harness/src/mcp/mcp.ts | 24 +++- packages/harness/src/mcp/section.ts | 57 +++++++++ .../src/plugin/builtin/tool/toolsearch.ts | 2 +- packages/harness/src/plugin/host.ts | 4 +- packages/harness/src/plugin/prompt/render.ts | 7 ++ packages/harness/src/plugin/registry.ts | 14 ++- packages/harness/src/runner/loop.ts | 16 ++- packages/harness/src/tool/search.ts | 3 + .../test/__artifacts__/mcp.section.json | 48 +++++++ .../mcp.section.live.openai-codex.json | 27 ++++ .../test/__artifacts__/mcp.thread.json | 3 +- .../test/__artifacts__/mcp.thread.loaded.json | 21 +++- .../__artifacts__/mcp.thread.resumed.json | 3 +- .../mcp.toolsearch.connecting.json | 3 +- .../harness/test/fixtures/mcp/session.mjs | 7 +- .../harness/test/mcp.section.live.test.ts | 117 ++++++++++++++++++ packages/harness/test/mcp.section.test.ts | 99 +++++++++++++++ packages/harness/test/mcp.thread.test.ts | 2 +- packages/plugin/src/plugin/section.ts | 2 +- 19 files changed, 434 insertions(+), 25 deletions(-) create mode 100644 packages/harness/src/mcp/section.ts create mode 100644 packages/harness/test/__artifacts__/mcp.section.json create mode 100644 packages/harness/test/__artifacts__/mcp.section.live.openai-codex.json create mode 100644 packages/harness/test/mcp.section.live.test.ts create mode 100644 packages/harness/test/mcp.section.test.ts diff --git a/packages/harness/src/mcp/mcp.ts b/packages/harness/src/mcp/mcp.ts index 9111f2dc..8d32dbce 100644 --- a/packages/harness/src/mcp/mcp.ts +++ b/packages/harness/src/mcp/mcp.ts @@ -70,6 +70,7 @@ import { McpLog } from "./log.ts"; import { McpOAuth } from "./oauth.ts"; import { McpOutput } from "./output.ts"; import { McpResources } from "./resources.ts"; +import { McpSection } from "./section.ts"; import { McpTool } from "./tool.ts"; import { McpValue } from "./value.ts"; import { hostPath } from "../host.ts"; @@ -135,6 +136,8 @@ export interface Reconciled { * Waits first, within {@link STARTUP_WAIT}, for those servers' connect attempts in progress. */ readonly deferred: Effect.Effect>; + /** The `mcpservers` prompt section, as the servers stand once the ones with declared tools were waited for. */ + readonly section: string | undefined; } export interface Interface { @@ -186,6 +189,8 @@ interface Server { skipped: EventList.McpSkipped | undefined; /** Whether the server offered resources when it last connected. */ resources: boolean; + /** The instructions it sent when it last connected; kept while it is not connected. */ + instructions: string | undefined; /** The credential it was refused with, when it needs sign-in: a new one is worth a retry. */ refused: string | undefined; live: Live | undefined; @@ -605,6 +610,7 @@ export const layer = (options: { server.live = live; yield* listed(server, exit.value.listing); server.resources = exit.value.connection.capabilities?.resources !== undefined; + server.instructions = exit.value.connection.instructions; yield* setState(server, "connected"); watch(server, live); yield* Effect.logInfo("mcp connected", { server: server.name, tools: server.tools.length }); @@ -898,11 +904,12 @@ export const layer = (options: { return tools; }; - /** The thread's enabled servers configured with `deferred` exposure. */ - const deferring = (thread: Thread) => - [...thread.servers.values()].flatMap(({ config, server }) => - server !== undefined && hasTools(config, "deferred") ? [server] : [], + /** The thread's enabled servers configured with `deferred` exposure, by name. */ + const deferringEntries = (thread: Thread) => + [...thread.servers].flatMap(([name, { config, server }]) => + server !== undefined && hasTools(config, "deferred") ? [{ name, config, server }] : [], ); + const deferring = (thread: Thread) => deferringEntries(thread).map(({ server }) => server); /** Settles when the server's connect attempt in progress, if any, does. */ const attempted = (server: Server) => @@ -1004,6 +1011,7 @@ export const layer = (options: { tools: [], skipped: undefined, resources: false, + instructions: undefined, refused: undefined, live: undefined, attempt: undefined, @@ -1076,6 +1084,14 @@ export const layer = (options: { tools: toolsOf(thread), defers: deferring(thread).length > 0, deferred: deferred(thread), + // A server disconnected on request offers no tools to search. + section: McpSection.render( + deferringEntries(thread).flatMap(({ name, config, server }) => + server.stopped + ? [] + : [{ name, description: config.description, instructions: server.instructions }], + ), + ), } satisfies Reconciled; }); diff --git a/packages/harness/src/mcp/section.ts b/packages/harness/src/mcp/section.ts new file mode 100644 index 00000000..e6b602db --- /dev/null +++ b/packages/harness/src/mcp/section.ts @@ -0,0 +1,57 @@ +/** + * The `mcpservers` prompt section: the servers whose tools are not declared to the model, since + * the tool search does not list them. Only `deferred` tools have a way in: a server whose tools are + * all `codemode` is left out until codemode exists, and the section is left out while the tool search + * is not declared. + */ + +import { ToolSearch } from "../tool/search.ts"; +import { namespace } from "./config.ts"; + +export const name = "mcpservers"; + +/** Characters of a server description, as Codex allows for deferred namespaces. */ +const MAX_DESCRIPTION_CHARS = 250; +/** Descriptions shrink to fit; when the server lines alone do not, the last servers are counted instead. */ +const MAX_SECTION_CHARS = 4096; + +const INTRO = `MCP servers whose tools are not declared to you. Load the tools of \`${ToolSearch.name}\` servers with \`${ToolSearch.name}\`.`; + +/** An enabled server with deferred tools. */ +export interface Listing { + readonly name: string; + readonly description: string | undefined; + /** The server's instructions, once it connected. */ + readonly instructions: string | undefined; +} + +const truncate = (text: string, max: number) => + text.length <= max ? text : max <= 1 ? "" : `${text.slice(0, max - 1).trimEnd()}…`; + +/** First line of the configured description, or of the server instructions once connected. */ +const summary = (server: Listing) => { + const text = server.description?.trim() || server.instructions || ""; + return (text.split("\n", 1)[0] ?? "").trim(); +}; + +/** Undefined when there are no servers. */ +export const render = (servers: ReadonlyArray): string | undefined => { + const listed = [...servers].sort((a, b) => a.name.localeCompare(b.name)); + if (listed.length === 0) return undefined; + const heads = listed.map((server) => `- ${namespace(server.name)} (${ToolSearch.name})`); + const omitted = (count: number) => + count > 0 ? [`- … ${count} more server${count === 1 ? "" : "s"}; find their tools with ${ToolSearch.name}`] : []; + const size = (kept: number) => [INTRO, ...heads.slice(0, kept), ...omitted(listed.length - kept)].join("\n").length; + let kept = listed.length; + while (kept > 0 && size(kept) > MAX_SECTION_CHARS) kept--; + // Each description also takes a ": " separator. + const each = + kept === 0 ? 0 : Math.min(MAX_DESCRIPTION_CHARS, Math.floor((MAX_SECTION_CHARS - size(kept)) / kept) - 2); + const lines = listed.slice(0, kept).map((server, index) => { + const text = each > 0 ? truncate(summary(server), each) : ""; + return text ? `${heads[index]}: ${text}` : heads[index]; + }); + return [INTRO, ...lines, ...omitted(listed.length - kept)].join("\n"); +}; + +export * as McpSection from "./section.ts"; diff --git a/packages/harness/src/plugin/builtin/tool/toolsearch.ts b/packages/harness/src/plugin/builtin/tool/toolsearch.ts index b20fa19e..d9382438 100644 --- a/packages/harness/src/plugin/builtin/tool/toolsearch.ts +++ b/packages/harness/src/plugin/builtin/tool/toolsearch.ts @@ -10,7 +10,7 @@ import type { Builtin } from "../../plugin.ts"; * `modelTools.add`, so loaded tools stay offered on that line. */ -export const TOOL_SEARCH_NAME = "toolsearch"; +export const TOOL_SEARCH_NAME = ToolSearch.name; export const DEFAULT_LIMIT = 8; // Fixed, so it does not change as deferred tools are registered, for example as MCP servers connect. diff --git a/packages/harness/src/plugin/host.ts b/packages/harness/src/plugin/host.ts index e1e6b0db..08792fd5 100644 --- a/packages/harness/src/plugin/host.ts +++ b/packages/harness/src/plugin/host.ts @@ -92,9 +92,9 @@ export const run = Effect.fn("PluginHost.run")(function* ( yield* apply(prepared); } yield* flush(Infinity); - const { deferred } = yield* settle; + const { deferred, section } = yield* settle; return yield* Effect.try({ - try: () => buckets.freeze(input.location.directory, tools, deferred), + try: () => buckets.freeze(input.location.directory, tools, deferred, section), catch: (cause) => new SetupError({ message: `plugin snapshot freeze failed: ${cause instanceof Error ? cause.message : String(cause)}`, diff --git a/packages/harness/src/plugin/prompt/render.ts b/packages/harness/src/plugin/prompt/render.ts index 65e7af75..5f7975f8 100644 --- a/packages/harness/src/plugin/prompt/render.ts +++ b/packages/harness/src/plugin/prompt/render.ts @@ -1,6 +1,8 @@ // Bodies are never escaped: a plugin may nest its own tags inside an entry. import * as Section from "@codeworksh/plugin/plugin/section"; +import { McpSection } from "../../mcp/section.ts"; +import { ToolSearch } from "../../tool/search.ts"; import type { AnyToolDef } from "../../tool/tool.ts"; import { fallback, type Snapshot, type Written } from "./registry.ts"; @@ -8,6 +10,8 @@ interface Input { readonly prompt: Snapshot; readonly tools: ReadonlyArray; readonly directory: string; + /** The `mcpservers` section's body, rendered by the `mcp` domain. */ + readonly mcpservers: string | undefined; } const tag = (name: string, body: string): string => `<${name}>\n${body}\n`; @@ -64,6 +68,9 @@ export const render = (input: Input): Rendered => { for (const section of Section.builtins) push(section.name); sections.cwd = tag("cwd", input.directory); + // The servers are listed only while the tool that reaches them is declared. + if (input.mcpservers !== undefined && input.tools.some((tool) => tool.name === ToolSearch.name)) + sections[McpSection.name] = tag(McpSection.name, input.mcpservers); for (const name of Array.from(written.keys())) push(name); // A blank foundation would leave the Responses transport without instructions, and it would diff --git a/packages/harness/src/plugin/registry.ts b/packages/harness/src/plugin/registry.ts index 0fceeb02..27d051e9 100644 --- a/packages/harness/src/plugin/registry.ts +++ b/packages/harness/src/plugin/registry.ts @@ -27,14 +27,22 @@ export const make = (servers: ReadonlyArray) => { /** Close the MCP bucket and return the servers the `mcp` domain settled on. */ settle: mcp.settle, close, - /** `live` is the MCP servers' deferred tools as they stand when searched; see `Registry.resolve`. */ - freeze: (directory: string, selection?: Selection, live?: Effect.Effect>) => { + /** + * `live` is the MCP servers' deferred tools as they stand when searched; see `Registry.resolve`. + * `mcpservers` is the body of that prompt section, when there is one. + */ + freeze: ( + directory: string, + selection?: Selection, + live?: Effect.Effect>, + mcpservers?: string, + ) => { close(); const resolved = makeCatalog(tools.entries()).resolve(selection, live); return { tools: resolved, // Only declared tools: loading a deferred one must not change the prompt. - prompt: render({ prompt: prompt.snapshot(), tools: resolved.defs.filter(declared), directory }), + prompt: render({ prompt: prompt.snapshot(), tools: resolved.defs.filter(declared), directory, mcpservers }), }; }, }; diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index 29e89f0e..76f8de26 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -12,6 +12,7 @@ import { Context } from "../context/context.ts"; import { Event } from "../event/event.ts"; import { EventList } from "../event/list.ts"; import { Inbox } from "../inbox/inbox.ts"; +import { McpSection } from "../mcp/section.ts"; import { SessionMessageSchema } from "../session/message/schema.ts"; import type { ThreadSchema } from "../thread/schema.ts"; import { Thread } from "../thread/thread.ts"; @@ -86,6 +87,16 @@ const observeTool = ( return Telemetry.record(Telemetry.toolDuration, seconds, labels); }; +/** The `mcpservers` section is the one the run started with: servers that connect mid-run show at the next run. */ +const holdServers = (start: State.Snapshot, next: State.Snapshot): State.Snapshot => { + const { [McpSection.name]: _current, ...sections } = next.prompt.sections; + const held = start.prompt.sections[McpSection.name]; + return { + ...next, + prompt: { ...next.prompt, sections: held === undefined ? sections : { ...sections, [McpSection.name]: held } }, + }; +}; + export const layer = (options: Options = {}) => Layer.effect( Runner.Service, @@ -520,7 +531,8 @@ export const layer = (options: Options = {}) => }); const drain = Effect.gen(function* () { - let snapshot = yield* state.snapshot(input.threadId); + const start = yield* state.snapshot(input.threadId); + let snapshot = start; let continuing = input.force; // Tool results and steers continue first; a follow-up only once both are exhausted. while (true) { @@ -544,7 +556,7 @@ export const layer = (options: Options = {}) => if (!continuing && !steer && !followUp) return; promotion = followUp ? "followUp" : "steer"; - const next = yield* state.snapshot(input.threadId); + const next = holdServers(start, yield* state.snapshot(input.threadId)); snapshot = completed === undefined ? next : yield* hooks.prepareNextTurn(completed, next); } }); diff --git a/packages/harness/src/tool/search.ts b/packages/harness/src/tool/search.ts index 4a4c3de8..80e2fc76 100644 --- a/packages/harness/src/tool/search.ts +++ b/packages/harness/src/tool/search.ts @@ -2,6 +2,9 @@ import type { Message } from "@codeworksh/aikit"; /** BM25 ranking over tool metadata: what `toolsearch` loads deferred tools by. */ +/** The tool that loads deferred tools. */ +export const name = "toolsearch"; + export interface Document { readonly name: string; readonly text: string; diff --git a/packages/harness/test/__artifacts__/mcp.section.json b/packages/harness/test/__artifacts__/mcp.section.json new file mode 100644 index 00000000..cb5ef350 --- /dev/null +++ b/packages/harness/test/__artifacts__/mcp.section.json @@ -0,0 +1,48 @@ +{ + "systems": [ + 1, + 2, + 3, + 4 + ], + "changes": [ + { + "sections": [ + "tools", + "rules", + "cwd", + "mcpservers" + ], + "added": [], + "removed": [] + }, + { + "sections": [], + "added": [ + "mcp__atlas__echo" + ], + "removed": [] + }, + { + "sections": [ + "mcpservers" + ], + "added": [], + "removed": [] + }, + { + "sections": [ + "mcpservers" + ], + "added": [], + "removed": [ + "mcp__atlas__echo" + ] + } + ], + "mcpservers": [ + "\nMCP servers whose tools are not declared to you. Load the tools of `toolsearch` servers with `toolsearch`.\n- mcp__atlas (toolsearch)\n", + "\nMCP servers whose tools are not declared to you. Load the tools of `toolsearch` servers with `toolsearch`.\n- mcp__atlas (toolsearch): Tide tables for coastal towns.\n", + null + ] +} diff --git a/packages/harness/test/__artifacts__/mcp.section.live.openai-codex.json b/packages/harness/test/__artifacts__/mcp.section.live.openai-codex.json new file mode 100644 index 00000000..dd0b186c --- /dev/null +++ b/packages/harness/test/__artifacts__/mcp.section.live.openai-codex.json @@ -0,0 +1,27 @@ +{ + "declared": [ + { + "sections": [ + "tools", + "rules", + "cwd", + "mcpservers" + ], + "mcpservers": "\nMCP servers whose tools are not declared to you. Load the tools of `toolsearch` servers with `toolsearch`.\n- mcp__atlas (toolsearch): Capital cities of the world.\n" + }, + { + "sections": [ + "mcpservers" + ], + "mcpservers": "\nMCP servers whose tools are not declared to you. Load the tools of `toolsearch` servers with `toolsearch`.\n- mcp__atlas (toolsearch): Tide tables for coastal towns.\n" + } + ], + "order": [ + "user", + "system", + "assistant", + "user", + "system" + ], + "answer": "mcp__atlas" +} diff --git a/packages/harness/test/__artifacts__/mcp.thread.json b/packages/harness/test/__artifacts__/mcp.thread.json index 79f41a6b..7c06bec3 100644 --- a/packages/harness/test/__artifacts__/mcp.thread.json +++ b/packages/harness/test/__artifacts__/mcp.thread.json @@ -492,7 +492,8 @@ "sections": [ "tools", "rules", - "cwd" + "cwd", + "mcpservers" ], "added": [], "removed": [] diff --git a/packages/harness/test/__artifacts__/mcp.thread.loaded.json b/packages/harness/test/__artifacts__/mcp.thread.loaded.json index 000cc486..c466908b 100644 --- a/packages/harness/test/__artifacts__/mcp.thread.loaded.json +++ b/packages/harness/test/__artifacts__/mcp.thread.loaded.json @@ -5,7 +5,8 @@ "sections": [ "tools", "rules", - "cwd" + "cwd", + "mcpservers" ], "added": [], "removed": [] @@ -24,7 +25,8 @@ "sections": [ "tools", "rules", - "cwd" + "cwd", + "mcpservers" ], "added": [], "removed": [] @@ -38,7 +40,9 @@ "removed": [] }, { - "sections": [], + "sections": [ + "mcpservers" + ], "added": [], "removed": [ "mcp__slow__echo", @@ -57,7 +61,8 @@ "sections": [ "tools", "rules", - "cwd" + "cwd", + "mcpservers" ], "added": [], "removed": [] @@ -71,7 +76,9 @@ "removed": [] }, { - "sections": [], + "sections": [ + "mcpservers" + ], "added": [], "removed": [ "mcp__slow__echo", @@ -79,7 +86,9 @@ ] }, { - "sections": [], + "sections": [ + "mcpservers" + ], "added": [ "mcp__slow__echo", "mcp__slow__big" diff --git a/packages/harness/test/__artifacts__/mcp.thread.resumed.json b/packages/harness/test/__artifacts__/mcp.thread.resumed.json index 6f1e1b20..a2505a0a 100644 --- a/packages/harness/test/__artifacts__/mcp.thread.resumed.json +++ b/packages/harness/test/__artifacts__/mcp.thread.resumed.json @@ -17,7 +17,8 @@ "sections": [ "tools", "rules", - "cwd" + "cwd", + "mcpservers" ], "added": [], "removed": [] diff --git a/packages/harness/test/__artifacts__/mcp.toolsearch.connecting.json b/packages/harness/test/__artifacts__/mcp.toolsearch.connecting.json index 0ff8ab5b..16d4f626 100644 --- a/packages/harness/test/__artifacts__/mcp.toolsearch.connecting.json +++ b/packages/harness/test/__artifacts__/mcp.toolsearch.connecting.json @@ -12,7 +12,8 @@ "sections": [ "tools", "rules", - "cwd" + "cwd", + "mcpservers" ], "added": [], "removed": [] diff --git a/packages/harness/test/fixtures/mcp/session.mjs b/packages/harness/test/fixtures/mcp/session.mjs index 3ad9e5b9..838a56de 100644 --- a/packages/harness/test/fixtures/mcp/session.mjs +++ b/packages/harness/test/fixtures/mcp/session.mjs @@ -5,7 +5,7 @@ import { appendFileSync, existsSync } from "node:fs"; // E2E_STARTS: a file that gets each start's pid on a line. E2E_HOLD: while this file exists, wait before // answering the handshake. E2E_GATE: if this file exists once the hold is over, exit. E2E_DELAY: -// milliseconds to wait before answering the handshake. +// milliseconds to wait before answering the handshake. E2E_INSTRUCTIONS: the instructions it sends. if (process.env.E2E_STARTS) appendFileSync(process.env.E2E_STARTS, `${process.pid}\n`); while (process.env.E2E_HOLD && existsSync(process.env.E2E_HOLD)) await new Promise((resolve) => setTimeout(resolve, 20)); @@ -20,7 +20,10 @@ const text = (value) => ({ content: [{ type: "text", text: String(value) }] }); const server = new Server( { name: "session-fixture", version: "1.0.0" }, - { capabilities: { tools: {}, resources: {} } }, + { + capabilities: { tools: {}, resources: {} }, + ...(process.env.E2E_INSTRUCTIONS ? { instructions: process.env.E2E_INSTRUCTIONS } : {}), + }, ); server.setRequestHandler("tools/list", () => ({ tools: [ diff --git a/packages/harness/test/mcp.section.live.test.ts b/packages/harness/test/mcp.section.live.test.ts new file mode 100644 index 00000000..62c3c92d --- /dev/null +++ b/packages/harness/test/mcp.section.live.test.ts @@ -0,0 +1,117 @@ +import type { Message } from "@codeworksh/aikit"; +import { Effect } from "effect"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vite-plus/test"; +import { ContextCodec } from "../src/context/codec.ts"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import { Mcp } from "../src/mcp/mcp.ts"; +import { builtins } from "../src/plugin/builtin.ts"; +import { LLM } from "../src/runner/llm.ts"; +import { sectionNames } from "./fixtures/llm.ts"; +import { stdio, waitFor, write } from "./fixtures/mcp/turns.ts"; +import { withSettings } from "./fixtures/settings.ts"; +import { available, LIVE } from "./utils/live.ts"; + +/** + * A real model reads the `mcpservers` section. A deferred server is listed with its configured + * description; once that is dropped, its summary is the first line of the instructions it sent on + * connecting, and the next prompt carries only that section, after the prompt it came with. + */ + +const live = LIVE.find((one) => one.name === "openai-codex")!; +const MODEL = "gpt-6-luna"; +const ask = (topic: string) => + `Which MCP server would you load tools from with toolsearch for ${topic}? Answer from your instructions, ` + + "without calling any tool. Reply with only its namespace, or `none`."; + +it.skipIf(!available(live))( + "mcp: the mcpservers section lists a deferred server, and a changed summary is sent alone", + { + timeout: 300_000, + }, + () => + withSettings(async ({ root, custom }) => { + await writeFile( + join(custom, "settings.jsonc"), + JSON.stringify({ model: { provider: live.provider, id: MODEL } }), + ); + const home = join(root, "home"); + const server = (description?: string) => + write(join(custom, "mcp.json"), { + mcpServers: { + atlas: stdio( + { exposure: "deferred", ...(description === undefined ? {} : { description }) }, + { E2E_INSTRUCTIONS: "Tide tables for coastal towns.\nCall the echo tool to test." }, + ), + }, + }); + await server("Capital cities of the world."); + + const sent: Message.Context[] = []; + const llm: LLM.Open = (input, signal) => { + sent.push(input.context); + return LLM.open(input, signal); + }; + + const replies = await Effect.gen(function* () { + const handle = yield* Session.create({ directory: root }); + yield* handle.main.run(ask("capital cities")); + yield* handle.main.wait(); + // Same connection: description is not part of what an instance is started from. + yield* Effect.promise(() => server()); + const mcp = yield* Mcp.Service; + yield* waitFor(mcp.status(handle.main.id), (all) => all.some((one) => one.state === "connected")); + yield* handle.main.run(ask("tide tables")); + yield* handle.main.wait(); + const messages = yield* Effect.forEach(yield* handle.main.history(), ContextCodec.decodeMessage); + return messages.flatMap((message) => + message.role === "assistant" + ? [ + ContextCodec.parts(message) + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join("") + .trim(), + ] + : [], + ); + }).pipe( + Effect.provide( + Harness.layer({ + home, + hostCwd: root, + database: ":memory:", + userConfigDir: custom, + plugins: builtins, + llm, + }), + ), + Effect.scoped, + Effect.timeout("240 seconds"), + Effect.runPromise, + ); + + const last = sent.at(-1)!; + const declared = last.messages.flatMap((message) => + message.role === "system" + ? [{ sections: sectionNames(message.sections), mcpservers: message.sections?.mcpservers }] + : [], + ); + const order = last.messages.map((message) => message.role); + const artifact = { + declared, + // The change follows the second prompt: everything before it is the first request's prefix. + order, + // The model reads the change: tide tables come only from the second declaration. + answer: replies.at(-1)?.replace(/`/g, ""), + }; + expect(declared).toHaveLength(2); + expect(declared[1]?.sections).toEqual(["mcpservers"]); + expect(replies).toHaveLength(2); + expect(artifact.answer).toBe("mcp__atlas"); + await expect(`${JSON.stringify(artifact, null, "\t")}\n`).toMatchFileSnapshot( + `./__artifacts__/mcp.section.live.${live.name}.json`, + ); + }), +); diff --git a/packages/harness/test/mcp.section.test.ts b/packages/harness/test/mcp.section.test.ts new file mode 100644 index 00000000..ebff5f2e --- /dev/null +++ b/packages/harness/test/mcp.section.test.ts @@ -0,0 +1,99 @@ +import "./utils/env.ts"; +import { Effect } from "effect"; +import fs from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vite-plus/test"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import { Mcp } from "../src/mcp/mcp.ts"; +import { defaultPromptPlugin } from "../src/plugin/builtin/prompt/default.ts"; +import type { LLM } from "../src/runner/llm.ts"; +import { changes, scripted, stdio, toolSearch, waitFor, write } from "./fixtures/mcp/turns.ts"; +import { tmpdir } from "./fixtures/tempdir.ts"; +import { pendingCall } from "./tools.fixture.ts"; + +it("mcp: a server that connects mid-run gets its summary in the mcpservers section at the next prompt, and leaves it once disconnected", async () => { + await using dir = await tmpdir(); + const home = join(dir.path, "home"); + const work = join(dir.path, "work"); + await fs.mkdir(work, { recursive: true }); + // The server answers its handshake only once `hold` is gone, so the first request finds it connecting. + const hold = join(dir.path, "hold"); + await fs.writeFile(hold, "held"); + await write(join(home, "mcp.json"), { + mcpServers: { + atlas: stdio( + { exposure: "deferred" }, + { E2E_HOLD: hold, E2E_INSTRUCTIONS: "Tide tables for coastal towns.\nCall the echo tool to test." }, + ), + }, + }); + + const model = scripted(); + let connected: Effect.Effect = Effect.void; + // The first request lets the server connect, and goes out once it has. + const open: LLM.Open = (input, signal) => + Effect.andThen( + model.requests.length === 0 + ? Effect.andThen( + Effect.promise(() => fs.rm(hold)), + connected, + ) + : Effect.void, + model.open(input, signal), + ); + + const result = await Effect.gen(function* () { + const session = yield* Session.create({ directory: work, model: { provider: "openai", id: "gpt-5.6-luna" } }); + const main = session.main; + const mcp = yield* Mcp.Service; + connected = waitFor(mcp.status(main.id), (all) => all.some((one) => one.state === "connected")); + // The run's second request follows a search, with the server connected by then. + model.turn(main.id, [pendingCall("toolsearch", { query: "echo" }, "search")]); + yield* main.run("one"); + yield* main.wait(); + model.turn(main.id); + yield* main.run("two"); + yield* main.wait(); + // Disconnected on request, it has nothing to search: the next prompt removes the section. + yield* mcp.disconnect(main.id, "atlas"); + model.turn(main.id); + yield* main.run("three"); + yield* main.wait(); + const last = model.requests.at(-1)!.context.messages; + return { + // Per request: how many system messages it carries. + systems: model.requests.map( + (request) => request.context.messages.filter((message) => message.role === "system").length, + ), + changes: changes(last), + mcpservers: last.flatMap((message) => + message.role === "system" && message.sections?.mcpservers !== undefined + ? [message.sections.mcpservers] + : [], + ), + }; + }).pipe( + Effect.provide( + Harness.layer({ + home, + hostCwd: dir.path, + database: ":memory:", + plugins: [defaultPromptPlugin, ...toolSearch], + llm: open, + }), + ), + Effect.scoped, + Effect.runPromise, + ); + + expect(result.systems).toEqual([1, 2, 3, 4]); + expect(result.changes.map((change) => change.sections)).toEqual([ + ["tools", "rules", "cwd", "mcpservers"], + [], + ["mcpservers"], + ["mcpservers"], + ]); + expect(result.mcpservers.at(-1)).toBeNull(); + await expect(`${JSON.stringify(result, null, "\t")}\n`).toMatchFileSnapshot("./__artifacts__/mcp.section.json"); +}); diff --git a/packages/harness/test/mcp.thread.test.ts b/packages/harness/test/mcp.thread.test.ts index 8c6cee15..d17e09d1 100644 --- a/packages/harness/test/mcp.thread.test.ts +++ b/packages/harness/test/mcp.thread.test.ts @@ -437,7 +437,7 @@ it("mcp: a thread resumed in a new process awaits the server of a tool it loaded expect(first.loaded?.add).toContain("mcp__docs__echo"); // The prompt and the loaded tools stay declared by the first run's system entries; nothing is declared again. expect(second.changes).toEqual([ - { sections: ["tools", "rules", "cwd"], added: [], removed: [] }, + { sections: ["tools", "rules", "cwd", "mcpservers"], added: [], removed: [] }, { sections: [], added: ["mcp__docs__echo", "mcp__docs__big"], removed: [] }, ]); expect(second.calls.again).toBe("hello: again"); diff --git a/packages/plugin/src/plugin/section.ts b/packages/plugin/src/plugin/section.ts index bfc9735b..a72fde9d 100644 --- a/packages/plugin/src/plugin/section.ts +++ b/packages/plugin/src/plugin/section.ts @@ -18,7 +18,7 @@ export const Skills: Section<"skills"> = Object.freeze({ name: "skills", format: export const builtins: ReadonlyArray
= Object.freeze([Rules, Addendum, ProjectContext, Skills]); /** Rendered by the harness, never by a plugin. */ -export const reserved: ReadonlySet = new Set(["tools", "cwd", "foundation"]); +export const reserved: ReadonlySet = new Set(["tools", "cwd", "mcpservers", "foundation"]); const pattern = /^[a-z][a-z0-9_-]*$/;