diff --git a/packages/harness/src/context/assemble.ts b/packages/harness/src/context/assemble.ts index ac29c868..148c126e 100644 --- a/packages/harness/src/context/assemble.ts +++ b/packages/harness/src/context/assemble.ts @@ -3,13 +3,13 @@ import { DateTime, Effect, Schema } from "effect"; import { optional } from "../schema.ts"; import { SessionSchema } from "../session/schema.ts"; import type { Thread } from "../thread/thread.ts"; -import { decodeMessage, decodeSyntheticMessage, type ContextMessage } from "./codec.ts"; +import { decodeMessage, decodeSyntheticMessage } from "./codec.ts"; import { ContextDecodeError } from "./errors.ts"; export interface Snapshot { readonly threadId: string; readonly leafEntryId: string | null; - readonly messages: ReadonlyArray; + readonly messages: ReadonlyArray; readonly lastAssistant?: { readonly entryId: string; readonly message: Message.AssistantMessage; @@ -64,7 +64,7 @@ const wrapperMessage = (entry: Thread.HydratedEntry, text: string): Message.User /** * Builds a thread's LLM context from its line (own entries, then inherited ones, in seq order). * Only committed entries count, and the context starts at the newest `reset` or `compaction`. - * Entries that are not messages (system, custom) stay in the transcript, out of the context. + * `system` entries become transcript system messages; `custom` entries stay out of the context. */ export const assembleLine = Effect.fn("Context.assembleLine")(function* ( threadId: string, @@ -78,7 +78,7 @@ export const assembleLine = Effect.fn("Context.assembleLine")(function* ( | { readonly index: number; readonly handoff: string | undefined } | undefined; - const messagesByEntry = new Map(); + const messagesByEntry = new Map(); const compactions = new Map(); const handoffs = new Map(); @@ -126,8 +126,10 @@ export const assembleLine = Effect.fn("Context.assembleLine")(function* ( latestStart = { index, handoff: data.handoff }; break; } - case "system": + case "system": { + messagesByEntry.set(entry.id, yield* decodeMessage(hydrated)); break; + } case "custom": { yield* decodeCustomData(entry.data).pipe(Effect.mapError(decodeFailure(hydrated))); break; @@ -156,7 +158,7 @@ export const assembleLine = Effect.fn("Context.assembleLine")(function* ( effectivePath = [committedPath[index]!, ...kept, ...committedPath.slice(index + 1)]; } - const messages: ContextMessage[] = []; + const messages: Message.Message[] = []; for (const hydrated of effectivePath) { const { entry } = hydrated; const message = messagesByEntry.get(entry.id); diff --git a/packages/harness/src/context/codec.ts b/packages/harness/src/context/codec.ts index bd513e41..48ca0a52 100644 --- a/packages/harness/src/context/codec.ts +++ b/packages/harness/src/context/codec.ts @@ -17,7 +17,7 @@ export interface EncodedMessage { type AikitPart = Message.AssistantMessage["parts"][number] | Message.UserMessage["parts"][number]; -/** A message the session stores as an entry. Transcript system messages are not stored yet. */ +/** A message the session stores as a user or assistant entry. System messages have `system` entries. */ export type ContextMessage = Message.UserMessage | Message.AssistantMessage; const reasonOf = (cause: unknown): string => (cause instanceof Error ? cause.message : String(cause)); @@ -199,15 +199,61 @@ export const decodeSyntheticMessage = Effect.fn("Context.decodeSyntheticMessage" ); }); +/** The transcript system message of a tool change; empty sides are left out. */ +export const systemMessage = ( + messageId: string, + created: number, + changes: { + readonly toolsAdded: ReadonlyArray; + readonly toolsRemoved: ReadonlyArray; + }, +) => ({ + messageId, + role: "system" as const, + time: { created }, + ...(changes.toolsAdded.length === 0 ? {} : { toolsAdded: [...changes.toolsAdded] }), + ...(changes.toolsRemoved.length === 0 ? {} : { toolsRemoved: changes.toolsRemoved.map(({ name }) => ({ name })) }), +}); + +const decodeSystemData = Schema.decodeUnknownEffect(SessionSchema.SystemData); + +export const decodeSystemMessage = Effect.fn("Context.decodeSystemMessage")(function* ( + hydrated: Thread.HydratedEntry, +): Effect.fn.Return { + const { entry } = hydrated; + const data = yield* decodeSystemData(entry.data).pipe( + Effect.mapError( + (cause) => new ContextDecodeError({ entryId: entry.id, type: entry.type, reason: cause.message }), + ), + ); + const message = yield* Effect.try({ + try: () => + validateAikitMessage( + systemMessage(entry.id, DateTime.toEpochMillis(entry.createdAt), data), + `context entry ${entry.id}`, + ), + catch: (cause) => new ContextDecodeError({ entryId: entry.id, type: entry.type, reason: reasonOf(cause) }), + }); + if (message.role !== "system") { + return yield* new ContextDecodeError({ entryId: entry.id, type: entry.type, reason: "not a system message" }); + } + return message; +}); + +/** A message's parts; a system message has none. */ +export const parts = (message: Message.Message) => (message.role === "system" ? [] : message.parts); + +/** The aikit message a `user`, `assistant` or `system` entry stores. */ export const decodeMessage = Effect.fn("Context.decodeMessage")(function* ( hydrated: Thread.HydratedEntry, -): Effect.fn.Return { +): Effect.fn.Return { const { entry } = hydrated; + if (entry.type === "system") return yield* decodeSystemMessage(hydrated); if (entry.type !== "user" && entry.type !== "assistant") { return yield* new ContextDecodeError({ entryId: entry.id, type: entry.type, - reason: "only user and assistant entries contain complete aikit messages", + reason: "only user, assistant and system entries contain complete aikit messages", }); } diff --git a/packages/harness/src/event/list.ts b/packages/harness/src/event/list.ts index 37e82e79..a96a7f46 100644 --- a/packages/harness/src/event/list.ts +++ b/packages/harness/src/event/list.ts @@ -112,6 +112,17 @@ export const ThreadConfigured = EventSchema.define({ }); export type ThreadConfigured = typeof ThreadConfigured.Type; +/** + * The tools offered to a thread's next request differ from those its transcript declares, such as + * after `toolsearch` loaded some. Appends a `system` entry carrying the change. + */ +export const ToolsChanged = EventSchema.define({ + type: "session.thread.tools.changed", + ...durableOptions, + schema: { ...baseOptions, messageId: SessionMessageSchema.ID, ...SessionSchema.ToolChanges.fields }, +}); +export type ToolsChanged = typeof ToolsChanged.Type; + const LLMFields = { ...baseOptions, messageId: SessionMessageSchema.ID, @@ -344,6 +355,7 @@ export const Definitions = EventSchema.inventory( SessionCreated, ThreadForked, ThreadConfigured, + ToolsChanged, TurnStarted, TurnEnded, TurnAborted, @@ -379,6 +391,7 @@ export const DurableDefinitions = EventSchema.inventory( SessionCreated, ThreadForked, ThreadConfigured, + ToolsChanged, TurnEnded, LLMStarted, LLMEnded, diff --git a/packages/harness/src/plugin/builtin.ts b/packages/harness/src/plugin/builtin.ts index e7246954..c7d02748 100644 --- a/packages/harness/src/plugin/builtin.ts +++ b/packages/harness/src/plugin/builtin.ts @@ -7,13 +7,14 @@ import { findPlugin } from "./builtin/tool/find.ts"; import { grepPlugin } from "./builtin/tool/grep.ts"; import { lsPlugin } from "./builtin/tool/ls.ts"; import { readPlugin } from "./builtin/tool/read.ts"; +import { toolSearchPlugin } from "./builtin/tool/toolsearch.ts"; import { writePlugin } from "./builtin/tool/write.ts"; /** * Every plugin the harness ships, and the selection a caller who passes no `plugins` gets. Tool * contributors come first; within the prompt domain, declaration order is run order. The search * tools (grep, find, ls) are `optIn` and ship off: settings turn each on with - * `{ "plugin": "codework.tool.grep", "enabled": true }`. + * `{ "plugin": "codework.tool.grep", "enabled": true }`. So is toolsearch, which loads deferred tools. */ export const builtins = Object.freeze([ bashPlugin, @@ -23,6 +24,7 @@ export const builtins = Object.freeze([ grepPlugin, findPlugin, lsPlugin, + toolSearchPlugin, defaultPromptPlugin, instructionPlugin, skillPlugin, diff --git a/packages/harness/src/plugin/builtin/tool/toolsearch.ts b/packages/harness/src/plugin/builtin/tool/toolsearch.ts new file mode 100644 index 00000000..6dd1a454 --- /dev/null +++ b/packages/harness/src/plugin/builtin/tool/toolsearch.ts @@ -0,0 +1,86 @@ +import { Effect, Schema } from "effect"; +import { ToolOffer } from "../../../tool/offer.ts"; +import { ToolSearch } from "../../../tool/search.ts"; +import * as Tool from "../../../tool/tool.ts"; +import { define } from "../../plugin.ts"; + +/** + * The built-in toolsearch plugin: the model searches the deferred tools it is not offered yet, + * and the matches are loaded for its next request. Loading writes the thread's `agent` + * `modelTools.add`, so loaded tools stay offered on that line. + */ + +export const TOOL_SEARCH_NAME = "toolsearch"; +export const DEFAULT_LIMIT = 8; + +// Fixed, so it does not change as deferred tools are registered, for example as MCP servers connect. +const DESCRIPTION = `# Tool discovery\n\nSearches over deferred tool metadata with BM25 and exposes matching tools for the next model call.\n\nSome of the tools, such as tools of MCP servers, may not have been provided to you upfront, and you should use this tool (\`${TOOL_SEARCH_NAME}\`) to search for the required tools. For MCP tool discovery, always use \`${TOOL_SEARCH_NAME}\`.`; + +const ToolSearchParams = Schema.Struct({ + query: Schema.String.annotate({ description: "Search query for deferred tools." }), + limit: Schema.optional( + Schema.Finite.annotate({ description: `Maximum number of tools to return. Defaults to ${DEFAULT_LIMIT}.` }), + ), +}); + +/** The `details` of a completed call: the tools it loaded, best match first. */ +export const ToolSearchDetails = Schema.Struct({ loaded: Schema.Array(Schema.String) }); +export type ToolSearchDetails = typeof ToolSearchDetails.Type; + +const ToolSearchSuccess = Schema.Struct({ text: Schema.String, loaded: Schema.Array(Schema.String) }); + +export class ToolSearchFailed extends Schema.TaggedError()("ToolSearchFailed", { + message: Schema.String, +}) {} + +export const toolSearchDef = Tool.define({ + name: TOOL_SEARCH_NAME, + label: "toolsearch", + description: DESCRIPTION, + // Searching changes what the model is offered, so only the model calls it. + exposure: "model", + parameters: ToolSearchParams, + success: ToolSearchSuccess, + failure: ToolSearchFailed, + encodeContent: (success) => [{ type: "text", text: success.text }], + encodeFailureContent: (failure) => [{ type: "text", text: failure.message }], + encodeDetails: (success): ToolSearchDetails => ({ loaded: success.loaded }), +}); + +const fail = (message: string) => Effect.fail(new ToolSearchFailed({ message })); + +export const toolSearchHandler: Tool.Handler< + typeof ToolSearchParams, + typeof ToolSearchSuccess, + typeof ToolSearchFailed +> = Effect.fn("ToolSearch.handler")(function* ({ query, limit }) { + if (query.trim() === "") return yield* fail("query must not be empty"); + const max = limit ?? DEFAULT_LIMIT; + if (!Number.isInteger(max) || max <= 0) return yield* fail("limit must be a positive integer"); + const { deferred } = yield* ToolOffer; + const loaded = ToolSearch.rank( + query, + deferred.map((tool) => ({ name: tool.name, text: ToolSearch.text(tool) })), + max, + ); + const listed = loaded.map((name) => { + const description = deferred.find((tool) => tool.name === name)?.description ?? ""; + return `- ${name}: ${description.trim().split(/\r?\n/)[0]}`; + }); + const text = + loaded.length === 0 + ? "No matching tools found." + : `Loaded ${loaded.length} tool${loaded.length === 1 ? "" : "s"}. They are available from your next call:\n${listed.join("\n")}`; + return { text, loaded }; +}); + +export const toolSearchTool = Tool.implement(toolSearchDef, toolSearchHandler); + +export const toolSearchPlugin = define({ + id: "codework.tool.toolsearch", + kind: "tool", + optIn: true, + setup: (ctx) => { + ctx.plugin.tools.add(Tool.register(toolSearchTool)); + }, +}); diff --git a/packages/harness/src/plugin/host.ts b/packages/harness/src/plugin/host.ts index f57a0153..7c6436de 100644 --- a/packages/harness/src/plugin/host.ts +++ b/packages/harness/src/plugin/host.ts @@ -1,6 +1,7 @@ import { Cause, Effect, Schema } from "effect"; import type { Prepared } from "./catalog.ts"; import type { SharedPluginContext } from "./context.ts"; +import type { Selection } from "../tool/registry.ts"; import { make } from "./registry.ts"; export class SetupError extends Schema.TaggedError()("Plugin.SetupError", { @@ -12,6 +13,7 @@ export class SetupError extends Schema.TaggedError()("Plugin.SetupEr export const run = Effect.fn("PluginHost.run")(function* ( selection: ReadonlyArray, input: Omit, + tools?: Selection, ) { const buckets = make(); const ctx = Object.freeze({ ...input, plugin: buckets.registry }); @@ -57,7 +59,7 @@ export const run = Effect.fn("PluginHost.run")(function* ( ); } return yield* Effect.try({ - try: () => buckets.freeze(input.location.directory), + try: () => buckets.freeze(input.location.directory, tools), catch: (cause) => new SetupError({ message: `plugin snapshot freeze failed: ${cause instanceof Error ? cause.message : String(cause)}`, diff --git a/packages/harness/src/plugin/registry.ts b/packages/harness/src/plugin/registry.ts index e7bc56da..1f5abb94 100644 --- a/packages/harness/src/plugin/registry.ts +++ b/packages/harness/src/plugin/registry.ts @@ -1,5 +1,6 @@ import type { PluginRegistry } from "@codeworksh/plugin/plugin"; -import { make as makeCatalog } from "../tool/registry.ts"; +import { make as makeCatalog, type Selection } from "../tool/registry.ts"; +import { declared } from "../tool/tool.ts"; import { make as makePrompt } from "./prompt/registry.ts"; import { render } from "./prompt/render.ts"; import { make as makeTools } from "./tool/registry.ts"; @@ -17,12 +18,13 @@ export const make = () => { return { registry: Object.freeze({ tools: tools.registry, prompt: prompt.registry }), close, - freeze: (directory: string) => { + freeze: (directory: string, selection?: Selection) => { close(); - const resolved = makeCatalog(tools.entries()).resolve(); + const resolved = makeCatalog(tools.entries()).resolve(selection); return { tools: resolved, - systemPrompt: render({ prompt: prompt.snapshot(), tools: resolved.defs, directory }), + // Only declared tools: loading a deferred one must not change the prompt. + systemPrompt: render({ prompt: prompt.snapshot(), tools: resolved.defs.filter(declared), directory }), }; }, }; diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index b495a4ca..5f3fe98f 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -5,8 +5,8 @@ * event. LLMStarted owns the speculative draft; TurnEnded is the only commit. */ -import { type Message } from "@codeworksh/aikit"; -import { Cause, DateTime, Duration, Effect, Exit, Layer, Option } from "effect"; +import { type Message, Transcript } from "@codeworksh/aikit"; +import { Cause, DateTime, Duration, Effect, Exit, Layer, Option, Schema } from "effect"; import { ContextCodec } from "../context/codec.ts"; import { Context } from "../context/context.ts"; import { Event } from "../event/event.ts"; @@ -18,6 +18,7 @@ import { Thread } from "../thread/thread.ts"; import { State } from "../state/state.ts"; import { Telemetry } from "../telemetry/telemetry.ts"; import { errorMessage } from "../util/error.ts"; +import { Hash } from "../util/hash.ts"; import { LLMEventPublisher } from "./event.ts"; import { Hooks } from "./hooks.ts"; import { LLM } from "./llm.ts"; @@ -39,6 +40,9 @@ const terminalResult = (text: string) => ({ isError: true as const, }); +const encodeJson = (value: unknown) => + Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))(value).pipe(Effect.orDie); + const endTime = (call: Message.ToolCallPendingPart, now: number) => Math.max(call.time.end, now); const skippedPart = (call: Message.ToolCallPendingPart, message: string, now: number): Message.ToolCallSkippedPart => ({ @@ -270,6 +274,37 @@ export const layer = (options: Options = {}) => return interruptedCause; }); + /** + * The request keeps the declared tools as `tools`; what the thread offers beyond them (or + * withholds) is declared by a `system` entry, published when the transcript's tool state + * differs from the offered one. Returns its message, for the request at hand. + */ + const declareToolChanges = Effect.fn("Loop.declareToolChanges")(function* ( + snapshot: State.Snapshot, + messages: ReadonlyArray, + ) { + const previous = Transcript.resolveTranscriptTools( + { messages: [...messages], tools: [...snapshot.tools.wire] }, + "none", + ).requestTools; + const changes = Transcript.getToolStateChanges(previous, snapshot.tools.offered); + if (changes.toolsAdded.length === 0 && changes.toolsRemoved.length === 0) return []; + const messageId = SessionMessageSchema.ID.create(); + const timestamp = yield* DateTime.now; + const hash = Hash.sha256(yield* encodeJson(snapshot.tools.offered.map(Transcript.toToolDeclaration))); + yield* events.publish(EventList.ToolsChanged, { + timestamp, + sessionId: snapshot.sessionId, + threadId: snapshot.threadId, + messageId, + ...changes, + hash, + }); + return [ + ContextCodec.systemMessage(messageId, DateTime.toEpochMillis(timestamp), changes), + ] satisfies ReadonlyArray; + }); + const runTurnAttempt = Effect.fn("Loop.runTurnAttempt", { level: "Info" })(function* ( promotion: Promotion, snapshot: State.Snapshot, @@ -291,6 +326,7 @@ export const layer = (options: Options = {}) => if (leaf !== undefined && (leaf.stopReason === "stop" || leaf.stopReason === "length")) { return undefined; } + const messages = [...assembled.messages, ...(yield* declareToolChanges(snapshot, assembled.messages))]; // The latest attempt's publisher; a retried turn has one per provider request. let publisher: LLMEventPublisher.Publisher | undefined; @@ -316,7 +352,7 @@ export const layer = (options: Options = {}) => { context: { systemPrompt: snapshot.systemPrompt, - messages: [...assembled.messages], + messages: [...messages], tools: [...snapshot.tools.wire], }, provider: snapshot.provider, diff --git a/packages/harness/src/session/projector.ts b/packages/harness/src/session/projector.ts index 4b091e84..bbdb9f5a 100644 --- a/packages/harness/src/session/projector.ts +++ b/packages/harness/src/session/projector.ts @@ -28,18 +28,23 @@ */ import { Message } from "@codeworksh/aikit"; -import { DateTime, Effect, Layer } from "effect"; +import { DateTime, Effect, Layer, Option, Schema } from "effect"; import { ContextCodec } from "../context/codec.ts"; import { Event } from "../event/event.ts"; import { EventList } from "../event/list.ts"; import { Document } from "../document/document.ts"; import { Inbox } from "../inbox/inbox.ts"; +import { TOOL_SEARCH_NAME, ToolSearchDetails } from "../plugin/builtin/tool/toolsearch.ts"; import { ThreadSchema } from "../thread/schema.ts"; import { Thread } from "../thread/thread.ts"; import { PromptSchema } from "./prompt/schema.ts"; import type { SessionMessageSchema } from "./message/schema.ts"; +import { SessionSchema } from "./schema.ts"; import { Session } from "./session.ts"; +const decodeLoaded = Schema.decodeUnknownOption(ToolSearchDetails); +const encodeSystemData = Schema.encodeEffect(SessionSchema.SystemData); + export const layer = Layer.effectDiscard( Effect.gen(function* () { const events = yield* Event.Service; @@ -193,15 +198,48 @@ export const layer = Layer.effectDiscard( }), ); + yield* events.project(EventList.ToolsChanged, (event) => + Effect.gen(function* () { + const { toolsAdded, toolsRemoved, hash } = event.data; + yield* threads + .append({ + id: event.data.messageId, + threadId: event.data.threadId, + seq: yield* seqOf(event), + type: "system", + data: yield* encodeSystemData({ toolsAdded, toolsRemoved, hash }).pipe(Effect.orDie), + }) + .pipe(Effect.orDie); + }), + ); + yield* events.project(EventList.ToolSettled, (event) => Effect.gen(function* () { - const encoded = yield* ContextCodec.encodePart(event.data.part).pipe(Effect.orDie); + const { part, threadId } = event.data; + const encoded = yield* ContextCodec.encodePart(part).pipe(Effect.orDie); yield* threads.settleToolCall({ entryId: event.data.messageId, callId: event.data.callID, - status: event.data.part.status, + status: part.status, data: encoded.data, }); + // Written here, not by a second event: a load is a fact of the settled call, so its `agent` + // revision lands at the settle's own seq and can never be split from it. + const loaded = + part.name === TOOL_SEARCH_NAME && part.status === "completed" + ? decodeLoaded(part.result.details) + : Option.none(); + if (Option.isNone(loaded) || loaded.value.loaded.length === 0) return; + const names = loaded.value.loaded; + const current = (yield* documents.agent(threadId)).modelTools; + const remove = (current?.remove ?? []).filter((name) => !names.includes(name)); + const add = [...(current?.add ?? [])]; + for (const name of names) if (!add.includes(name)) add.push(name); + yield* documents.configure( + threadId, + { modelTools: { ...(remove.length === 0 ? {} : { remove }), add } }, + yield* seqOf(event), + ); }), ); diff --git a/packages/harness/src/session/schema.ts b/packages/harness/src/session/schema.ts index d12bd439..5995e9b3 100644 --- a/packages/harness/src/session/schema.ts +++ b/packages/harness/src/session/schema.ts @@ -1,5 +1,5 @@ import { Schema } from "effect"; -import { NonNegativeCost, NonNegativeInt } from "../schema.ts"; +import { NonNegativeCost, NonNegativeInt, optional } from "../schema.ts"; // Session identity lives in `@codeworksh/plugin`: it is on every plugin context and every // tool call, so the brand a plugin holds has to be the one the harness mints. @@ -36,6 +36,25 @@ export const CompactionData = Schema.fromJsonString( ); export type CompactionData = typeof CompactionData.Type; +/** A tool as declared to the model (aikit `Tool`). Validated in full where it becomes a system message. */ +export const ToolDeclaration = Schema.Struct({ + name: Schema.String, + description: Schema.String, + parameters: Schema.Unknown, + constrainedSampling: optional(Schema.Unknown), +}); + +/** A change to the tools offered from one point of the transcript on, as a `system` entry stores it. */ +export const ToolChanges = Schema.Struct({ + toolsAdded: Schema.Array(ToolDeclaration), + toolsRemoved: Schema.Array(Schema.Struct({ name: Schema.String })), + /** sha256 of the offered tool declarations after the change. */ + hash: Schema.String, +}); +export type ToolChanges = typeof ToolChanges.Type; + +export const SystemData = Schema.fromJsonString(ToolChanges); + // Harness-owned shape of the billed usage inside a persisted assistant envelope. // This is data-structure integrity only — business validation // belongs to the Context Manager, a layer above. Stricter than the wire: diff --git a/packages/harness/src/state/state.ts b/packages/harness/src/state/state.ts index 4ada18e0..833497ed 100644 --- a/packages/harness/src/state/state.ts +++ b/packages/harness/src/state/state.ts @@ -378,6 +378,7 @@ export const layer = ( sessionId, hostDir, loadedSettings, + agent: chosen, configured: { ...composed, resolvedModel, thinkingLevel }, }; }); @@ -414,7 +415,7 @@ export const layer = ( } satisfies Configuration; }), snapshot: Effect.fn("State.snapshot")(function* (threadId: ThreadSchema.ID) { - const { sessionId, hostDir, loadedSettings, configured } = yield* configure(threadId); + const { sessionId, hostDir, loadedSettings, agent, configured } = yield* configure(threadId); const owner = { sessionId, threadId }; const { promptCustom, @@ -477,20 +478,26 @@ export const layer = ( }); } - const contributions = yield* setup(chosen.selection, { - sessionId, - threadId, - sandbox, - paths: global, - location, - settings: loadedSettings, - model: resolvedModel, - events, - config: { - ...(promptCustom === undefined ? {} : { promptCustom: resolver(promptCustom) }), - ...(promptSystemAppend === undefined ? {} : { promptSystemAppend: resolver(promptSystemAppend) }), + const contributions = yield* setup( + chosen.selection, + { + sessionId, + threadId, + sandbox, + paths: global, + location, + settings: loadedSettings, + model: resolvedModel, + events, + config: { + ...(promptCustom === undefined ? {} : { promptCustom: resolver(promptCustom) }), + ...(promptSystemAppend === undefined + ? {} + : { promptSystemAppend: resolver(promptSystemAppend) }), + }, }, - }).pipe(Effect.mapError((cause) => new SnapshotError({ threadId, reason: cause.message, cause }))); + agent.modelTools, + ).pipe(Effect.mapError((cause) => new SnapshotError({ threadId, reason: cause.message, cause }))); return { sessionId, diff --git a/packages/harness/src/tool/error.ts b/packages/harness/src/tool/error.ts index 281320e0..915d98fe 100644 --- a/packages/harness/src/tool/error.ts +++ b/packages/harness/src/tool/error.ts @@ -1 +1,12 @@ +import { Schema } from "effect"; + export { ToolExecutionError } from "@codeworksh/plugin/tool/error"; + +/** A call to a registered tool this request does not offer, such as a deferred tool not loaded yet. */ +export class ToolUnavailable extends Schema.TaggedError()("ToolUnavailable", { + name: Schema.String, +}) { + override get message(): string { + return `Tool "${this.name}" is not available in this thread.`; + } +} diff --git a/packages/harness/src/tool/executor.ts b/packages/harness/src/tool/executor.ts index a191811e..d45f49c9 100644 --- a/packages/harness/src/tool/executor.ts +++ b/packages/harness/src/tool/executor.ts @@ -23,7 +23,8 @@ import { } from "effect"; import { isAikitToolCallTerminalPart } from "../schema.ts"; import { errorMessage } from "../util/error.ts"; -import { ToolExecutionError } from "./error.ts"; +import { ToolExecutionError, ToolUnavailable } from "./error.ts"; +import { type Offer, ToolOffer } from "./offer.ts"; import { ToolProgress, type ToolProgressPartial } from "./progress.ts"; import { type AnyToolDef, type ModelContent, type RegisteredTool, toAikitTool, type ToolCallContext } from "./tool.ts"; @@ -262,12 +263,21 @@ const patchOutcome = (terminal: ToolOutcome, patch: ToolAfterResult | void): Too const ABORT_HOOK_GRACE = Duration.seconds(1); +const encodeUnavailable = Schema.encodeEffect(ToolUnavailable); + +export interface MakeOptions extends Partial { + /** Every registered tool name. A call to one of them that `tools` lacks is unavailable, not unknown. */ + readonly registered?: ReadonlyArray; +} + /** * Build an executor over a set of {@link RegisteredTool}s — tools whose capability `R` was * already discharged at registration (`Tool.provide`). The executor therefore needs no tool * `R`; only a progress sink's `RProgress` (if any) surfaces from `handle`. */ -export const make = (tools: ReadonlyArray): Executor => { +export const make = (tools: ReadonlyArray, options?: MakeOptions): Executor => { + const registered = new Set(options?.registered); + const offer: Offer = { deferred: options?.deferred ?? [] }; const impls = new Map(); for (const item of tools) { const entry = "tool" in item ? item : { tool: item, hooks: {} }; @@ -306,8 +316,13 @@ export const make = (tools: ReadonlyArray): E const entry = impls.get(call.name); if (entry === undefined) { const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - return errored(call, [text(`Unknown tool: ${call.name}`)], now, { - error: "unknown_tool", + if (registered.has(call.name)) { + const unavailable = new ToolUnavailable({ name: call.name }); + const details = yield* encodeUnavailable(unavailable).pipe(Effect.orDie); + return errored(call, [text(unavailable.message)], now, details); + } + return errored(call, [text(`Unknown Tool: ${call.name}`)], now, { + error: "unknown-tool", name: call.name, }); } @@ -323,16 +338,16 @@ export const make = (tools: ReadonlyArray): E ); if (Result.isFailure(prepared)) { const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - return errored(call, [text(`Invalid arguments for ${call.name}: ${prepared.failure}`)], now, { - error: "invalid_arguments", + return errored(call, [text(`Invalid Arguments for ${call.name}: ${prepared.failure}`)], now, { + error: "invalid-arguments", name: call.name, }); } const decoded = yield* Effect.result(decoders.get(call.name)!(prepared.success)); if (Result.isFailure(decoded)) { const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - return errored(call, [text(`Invalid arguments for ${call.name}: ${decoded.failure.message}`)], now, { - error: "invalid_arguments", + return errored(call, [text(`Invalid Arguments for ${call.name}: ${decoded.failure.message}`)], now, { + error: "invalid-arguments", name: call.name, }); } @@ -444,6 +459,7 @@ export const make = (tools: ReadonlyArray): E const exit = yield* Effect.suspend(() => impl.handler(decoded.success, ctx)).pipe( Effect.scoped, Effect.provideService(ToolProgress, progress), + Effect.provideService(ToolOffer, offer), Effect.exit, ); diff --git a/packages/harness/src/tool/offer.ts b/packages/harness/src/tool/offer.ts new file mode 100644 index 00000000..3cd34554 --- /dev/null +++ b/packages/harness/src/tool/offer.ts @@ -0,0 +1,15 @@ +import type { Message } from "@codeworksh/aikit"; +import { Context } from "effect"; + +export interface Offer { + /** Deferred tools the model may call and is not offered yet, in registry order. */ + readonly deferred: ReadonlyArray; +} + +/** + * What the current request offers, provided by the executor to every handler. A reference, so a + * handler reading it keeps `ToolProgress` as its only requirement and registers like any other. + */ +export const ToolOffer = Context.Reference("@codeworksh/harness/tool/offer/ToolOffer", { + defaultValue: () => ({ deferred: [] }), +}); diff --git a/packages/harness/src/tool/registry.ts b/packages/harness/src/tool/registry.ts index 3e785910..e6228aa9 100644 --- a/packages/harness/src/tool/registry.ts +++ b/packages/harness/src/tool/registry.ts @@ -2,7 +2,7 @@ import type { ToolRegistration } from "../plugin/tool/schema.ts"; import type { Message } from "@codeworksh/aikit"; import { Context, Layer } from "effect"; import * as Executor from "./executor.ts"; -import { type AnyToolDef, declared, type RegisteredTool } from "./tool.ts"; +import { type AnyToolDef, declared, type RegisteredTool, toAikitTool } from "./tool.ts"; /** * `ToolRegistry` — the loop-facing entry point to the tool catalog @@ -19,12 +19,28 @@ import { type AnyToolDef, declared, type RegisteredTool } from "./tool.ts"; * third-party/plugin) compose without a shared capability union. */ +/** + * A thread's choice of offered tools (its `agent` document's `modelTools`): `add` loads deferred + * tools, in load order; `remove` withholds tools, declared or loaded. + */ +export interface Selection { + readonly add?: ReadonlyArray | undefined; + readonly remove?: ReadonlyArray | undefined; +} + /** A frozen effective tool set: model-facing defs + the single execution entry point. */ export interface Resolved { - /** Pure definition data (prompt index, debugging, future codemode). */ + /** Definitions of the offered tools: the declared ones, then loaded deferred ones in load order. */ readonly defs: ReadonlyArray; - /** The aikit wire view handed to the LLM request as `tools`. */ + /** + * The declared tools' wire view, handed to the LLM request as `tools`. Loaded tools never join + * it: they reach the model as transcript system messages, so loading one keeps the request prefix. + */ readonly wire: ReadonlyArray; + /** The offered tools' wire view: the tool state the transcript's system messages bring `wire` to. */ + readonly offered: ReadonlyArray; + /** Deferred tools not offered, in registry order: what `toolsearch` searches. */ + readonly deferred: ReadonlyArray; /** * Transform one complete pending tool-call part into a complete terminal part. The **only** * execution entry point — the @@ -42,8 +58,8 @@ export interface Registry { readonly names: ReadonlyArray; /** Metadata lookup only — never exposes the executable impl. */ readonly getDef: (name: string) => AnyToolDef | undefined; - /** Resolve a frozen snapshot of the tools declared to the model. */ - readonly resolve: () => Resolved; + /** Resolve a frozen snapshot of the tools offered to the model under `selection`. */ + readonly resolve: (selection?: Selection) => Resolved; } /** @@ -62,28 +78,43 @@ export const make = (tools: ReadonlyArray): R byName.set(entry.tool.definition.name, entry); } - // Capture the effective set ONCE; the snapshot closes over these frozen arrays, so it + // Capture the effective set ONCE; every snapshot closes over these frozen arrays, so it // keeps its winning implementations and membership regardless of any later change. const effective = Object.freeze([...byName.values()]); const defs = Object.freeze(effective.map((entry) => entry.tool.definition)); const names = Object.freeze(effective.map((entry) => entry.tool.definition.name)); + const isDeferred = (entry: ToolRegistration) => entry.tool.definition.exposure === "deferred"; - // Only declared tools run: a call to any other gets the executor's `Unknown tool` result. - const declaredSet = effective.filter((entry) => declared(entry.tool.definition)); - // The effective set is unique, so the executor's own duplicate guard never trips. - const executor = Executor.make(declaredSet); - - const resolved: Resolved = Object.freeze({ - defs: Object.freeze(declaredSet.map((entry) => entry.tool.definition)), - wire: Object.freeze([...executor.wire]), - handle: executor.handle, - }); + const resolve = (selection?: Selection): Resolved => { + const removed = new Set(selection?.remove); + const declaredSet = effective.filter((entry) => declared(entry.tool.definition)); + const kept = declaredSet.filter((entry) => !removed.has(entry.tool.definition.name)); + const loaded = [...new Set(selection?.add)].flatMap((name) => { + const entry = byName.get(name); + return entry !== undefined && isDeferred(entry) && !removed.has(name) ? [entry] : []; + }); + const offered = [...kept, ...loaded]; + const deferred = Object.freeze( + effective + .filter((entry) => isDeferred(entry) && !offered.includes(entry)) + .map((entry) => toAikitTool(entry.tool.definition)), + ); + // Only offered tools run: a call to another registered one gets `ToolUnavailable`. + const executor = Executor.make(offered, { registered: names, deferred }); + return Object.freeze({ + defs: Object.freeze(offered.map((entry) => entry.tool.definition)), + wire: Object.freeze(declaredSet.map((entry) => toAikitTool(entry.tool.definition))), + offered: Object.freeze([...executor.wire]), + deferred, + handle: executor.handle, + }); + }; return Object.freeze({ defs, names, getDef: (name: string) => byName.get(name)?.tool.definition, - resolve: () => resolved, + resolve, }); }; diff --git a/packages/harness/src/tool/search.ts b/packages/harness/src/tool/search.ts new file mode 100644 index 00000000..4a4c3de8 --- /dev/null +++ b/packages/harness/src/tool/search.ts @@ -0,0 +1,115 @@ +import type { Message } from "@codeworksh/aikit"; + +/** BM25 ranking over tool metadata: what `toolsearch` loads deferred tools by. */ + +export interface Document { + readonly name: string; + readonly text: string; +} + +const STOP_WORDS: ReadonlySet = new Set([ + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "by", + "for", + "from", + "in", + "is", + "it", + "of", + "on", + "or", + "that", + "the", + "this", + "to", + "with", +]); + +/** Naive singular form, so `issues` matches `issue` and `searches` matches `search`. */ +const stem = (term: string): string => { + if (term.length > 4 && term.endsWith("ies")) return `${term.slice(0, -3)}y`; + if (term.length > 4 && /(ches|shes|sses|xes|zes)$/.test(term)) return term.slice(0, -2); + if (term.length > 3 && term.endsWith("s") && !term.endsWith("ss")) return term.slice(0, -1); + return term; +}; + +/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */ +export const tokenize = (text: string): string[] => + text + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((term) => term.length > 0 && !STOP_WORDS.has(term)) + .map(stem); + +/** Okapi BM25 (k1 1.2, b 0.75): the names of the best matches, best first; ties keep document order. */ +export const rank = (query: string, documents: ReadonlyArray, limit: number): string[] => { + const k1 = 1.2; + const b = 0.75; + const queryTerms = [...new Set(tokenize(query))]; + if (queryTerms.length === 0 || documents.length === 0 || limit <= 0) return []; + const termCounts = documents.map((document) => { + const counts = new Map(); + for (const term of tokenize(document.text)) counts.set(term, (counts.get(term) ?? 0) + 1); + return counts; + }); + const lengths = termCounts.map((counts) => [...counts.values()].reduce((sum, count) => sum + count, 0)); + const averageLength = lengths.reduce((sum, length) => sum + length, 0) / documents.length || 1; + const idf = new Map( + queryTerms.map((term) => { + const frequency = termCounts.filter((counts) => counts.has(term)).length; + return [term, Math.log(1 + (documents.length - frequency + 0.5) / (frequency + 0.5))] as const; + }), + ); + const matches: { name: string; score: number }[] = []; + documents.forEach((document, index) => { + let score = 0; + for (const term of queryTerms) { + const count = termCounts[index]!.get(term); + if (!count) continue; + const norm = k1 * (1 - b + (b * lengths[index]!) / averageLength); + score += (idf.get(term) ?? 0) * ((count * (k1 + 1)) / (count + norm)); + } + if (score > 0) matches.push({ name: document.name, score }); + }); + return matches + .sort((left, right) => right.score - left.score) + .slice(0, limit) + .map((match) => match.name); +}; + +const isObject = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +/** Schema descriptions and property names, recursively. */ +const schemaText = (schema: unknown, parts: string[]): void => { + if (!isObject(schema)) return; + if (typeof schema.description === "string") parts.push(schema.description); + if (isObject(schema.properties)) { + for (const [name, property] of Object.entries(schema.properties)) { + parts.push(name); + schemaText(property, parts); + } + } + schemaText(schema.items, parts); + for (const key of ["anyOf", "oneOf", "allOf"]) { + const variants = schema[key]; + if (Array.isArray(variants)) for (const variant of variants) schemaText(variant, parts); + } +}; + +/** Search text of a tool: its name, the name with `_` as spaces, its description, and its parameters' text. */ +export const text = (tool: Message.Tool): string => { + const parts = [tool.name, tool.name.replaceAll("_", " "), tool.description]; + schemaText(tool.parameters, parts); + return parts.filter((part) => part.trim() !== "").join(" "); +}; + +export * as ToolSearch from "./search.ts"; diff --git a/packages/harness/test/__artifacts__/tools.edit.json b/packages/harness/test/__artifacts__/tools.edit.json index f7e52e00..b82c2209 100644 --- a/packages/harness/test/__artifacts__/tools.edit.json +++ b/packages/harness/test/__artifacts__/tools.edit.json @@ -560,11 +560,11 @@ "content": [ { "type": "text", - "text": "Invalid arguments for edit: Expected array\n at [\"edits\"]" + "text": "Invalid Arguments for edit: Expected array\n at [\"edits\"]" } ], "details": { - "error": "invalid_arguments", + "error": "invalid-arguments", "name": "edit" } } diff --git a/packages/harness/test/__artifacts__/tools.exposure.json b/packages/harness/test/__artifacts__/tools.exposure.json index 8b39530a..dc065a2e 100644 --- a/packages/harness/test/__artifacts__/tools.exposure.json +++ b/packages/harness/test/__artifacts__/tools.exposure.json @@ -33,11 +33,11 @@ "content": [ { "type": "text", - "text": "Unknown tool: x_deferred" + "text": "Tool \"x_deferred\" is not available in this thread." } ], "details": { - "error": "unknown_tool", + "_tag": "ToolUnavailable", "name": "x_deferred" }, "isError": true diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.fork.json b/packages/harness/test/__artifacts__/tools.toolsearch.fork.json new file mode 100644 index 00000000..2aa7e65f --- /dev/null +++ b/packages/harness/test/__artifacts__/tools.toolsearch.fork.json @@ -0,0 +1,128 @@ +{ + "before": { + "agent": {}, + "firstRequest": { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch", + "notes_read" + ], + "system": [] + }, + "results": [ + { + "name": "weather_forecast", + "arguments": { + "city": "Porto" + }, + "status": "error", + "text": "Tool \"weather_forecast\" is not available in this thread.", + "details": { + "_tag": "ToolUnavailable", + "name": "weather_forecast" + } + } + ] + }, + "after": { + "agent": { + "modelTools": { + "add": [ + "weather_forecast" + ] + } + }, + "firstRequest": { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch", + "notes_read" + ], + "system": [ + { + "added": [ + "weather_forecast" + ], + "removed": [] + } + ] + }, + "results": [ + { + "name": "weather_forecast", + "arguments": { + "city": "Porto" + }, + "status": "completed", + "text": "weather_forecast ran", + "details": "weather_forecast ran" + } + ], + "entries": [ + { + "id": "E1", + "seq": 24, + "type": "user", + "state": "committed" + }, + { + "id": "E2", + "seq": 25, + "type": "system", + "state": "committed", + "data": { + "toolsAdded": [ + { + "name": "weather_forecast", + "description": "Get the weather forecast for a city.\nReturns daily highs and lows.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City to get the weather forecast for" + }, + "days": { + "anyOf": [ + { + "type": "number", + "description": "Number of forecast days" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "city" + ], + "additionalProperties": true + } + } + ], + "toolsRemoved": [], + "hash": "ec3094bdf14770d4650a88a2307556654cabf8623185fefb11702ba2e9a9c6e8" + } + }, + { + "id": "E3", + "seq": 26, + "type": "assistant", + "state": "committed" + }, + { + "id": "E4", + "seq": 30, + "type": "assistant", + "state": "committed" + } + ] + } +} diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.limit.json b/packages/harness/test/__artifacts__/tools.toolsearch.limit.json new file mode 100644 index 00000000..47d5e1a9 --- /dev/null +++ b/packages/harness/test/__artifacts__/tools.toolsearch.limit.json @@ -0,0 +1,74 @@ +{ + "results": [ + { + "name": "toolsearch", + "arguments": { + "query": "github issue", + "limit": 2 + }, + "status": "completed", + "text": "Loaded 2 tools. They are available from your next call:\n- github_create_issue: Create an issue in a GitHub repository.\n- jira_search_issues: Search Jira issues with a JQL query.", + "details": { + "loaded": [ + "github_create_issue", + "jira_search_issues" + ] + } + }, + { + "name": "toolsearch", + "arguments": { + "query": "create send search list get query translate run weather issue message quote event" + }, + "status": "completed", + "text": "Loaded 8 tools. They are available from your next call:\n- jira_search_issues: Search Jira issues with a JQL query.\n- calendar_create_event: Create an event in the user's calendar.\n- stock_quote: Get the latest stock quote for a ticker symbol.\n- database_query: Run a read-only SQL query against the analytics database.\n- slack_send_message: Send a message to a Slack channel.\n- weather_forecast: Get the weather forecast for a city.\n- github_create_issue: Create an issue in a GitHub repository.\n- email_send: Send an email message to a recipient.", + "details": { + "loaded": [ + "jira_search_issues", + "calendar_create_event", + "stock_quote", + "database_query", + "slack_send_message", + "weather_forecast", + "github_create_issue", + "email_send" + ] + } + } + ], + "agentAdd": [ + "github_create_issue", + "jira_search_issues", + "calendar_create_event", + "stock_quote", + "database_query", + "slack_send_message", + "weather_forecast", + "email_send" + ], + "nextRequest": { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch", + "notes_read" + ], + "system": [ + { + "added": [ + "github_create_issue", + "jira_search_issues", + "calendar_create_event", + "stock_quote", + "database_query", + "slack_send_message", + "weather_forecast", + "email_send" + ], + "removed": [] + } + ] + } +} diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.live.openai-codex.json b/packages/harness/test/__artifacts__/tools.toolsearch.live.openai-codex.json new file mode 100644 index 00000000..ed9cba22 --- /dev/null +++ b/packages/harness/test/__artifacts__/tools.toolsearch.live.openai-codex.json @@ -0,0 +1,47 @@ +{ + "requests": [ + { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch" + ], + "system": [] + }, + { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch" + ], + "system": [ + [ + "vault_read_token" + ] + ] + }, + { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch" + ], + "system": [ + [ + "vault_read_token" + ] + ] + } + ], + "calls": [ + "toolsearch:completed", + "vault_read_token:completed" + ], + "found": true +} diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.load.json b/packages/harness/test/__artifacts__/tools.toolsearch.load.json new file mode 100644 index 00000000..f622611c --- /dev/null +++ b/packages/harness/test/__artifacts__/tools.toolsearch.load.json @@ -0,0 +1,220 @@ +{ + "requests": [ + { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch", + "notes_read" + ], + "system": [] + }, + { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch", + "notes_read" + ], + "system": [ + { + "added": [ + "weather_forecast" + ], + "removed": [] + } + ] + }, + { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch", + "notes_read" + ], + "system": [ + { + "added": [ + "weather_forecast" + ], + "removed": [] + } + ] + } + ], + "results": [ + { + "name": "toolsearch", + "arguments": { + "query": "weather forecast", + "limit": 1 + }, + "status": "completed", + "text": "Loaded 1 tool. They are available from your next call:\n- weather_forecast: Get the weather forecast for a city.", + "details": { + "loaded": [ + "weather_forecast" + ] + } + }, + { + "name": "weather_forecast", + "arguments": { + "city": "Lisbon" + }, + "status": "completed", + "text": "weather_forecast ran", + "details": "weather_forecast ran" + } + ], + "events": [ + { + "seq": 1, + "type": "session.created.1" + }, + { + "seq": 2, + "type": "session.prompt.admitted.1" + }, + { + "seq": 3, + "type": "session.prompt.promoted.1" + }, + { + "seq": 4, + "type": "session.llm.started.1" + }, + { + "seq": 5, + "type": "session.llm.ended.1" + }, + { + "seq": 6, + "type": "session.tool.settled.1" + }, + { + "seq": 7, + "type": "session.turn.ended.1" + }, + { + "seq": 8, + "type": "session.thread.tools.changed.1" + }, + { + "seq": 9, + "type": "session.llm.started.1" + }, + { + "seq": 10, + "type": "session.llm.ended.1" + }, + { + "seq": 11, + "type": "session.tool.settled.1" + }, + { + "seq": 12, + "type": "session.turn.ended.1" + }, + { + "seq": 13, + "type": "session.llm.started.1" + }, + { + "seq": 14, + "type": "session.llm.ended.1" + }, + { + "seq": 15, + "type": "session.turn.ended.1" + } + ], + "entries": [ + { + "id": "E1", + "seq": 3, + "type": "user", + "state": "committed" + }, + { + "id": "E2", + "seq": 4, + "type": "assistant", + "state": "committed" + }, + { + "id": "E3", + "seq": 8, + "type": "system", + "state": "committed", + "data": { + "toolsAdded": [ + { + "name": "weather_forecast", + "description": "Get the weather forecast for a city.\nReturns daily highs and lows.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City to get the weather forecast for" + }, + "days": { + "anyOf": [ + { + "type": "number", + "description": "Number of forecast days" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "city" + ], + "additionalProperties": true + } + } + ], + "toolsRemoved": [], + "hash": "ec3094bdf14770d4650a88a2307556654cabf8623185fefb11702ba2e9a9c6e8" + } + }, + { + "id": "E4", + "seq": 9, + "type": "assistant", + "state": "committed" + }, + { + "id": "E5", + "seq": 13, + "type": "assistant", + "state": "committed" + } + ], + "agent": [ + { + "seq": 1, + "value": {} + }, + { + "seq": 6, + "value": { + "modelTools": { + "add": [ + "weather_forecast" + ] + } + } + } + ] +} diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.refused.json b/packages/harness/test/__artifacts__/tools.toolsearch.refused.json new file mode 100644 index 00000000..67f4f5d8 --- /dev/null +++ b/packages/harness/test/__artifacts__/tools.toolsearch.refused.json @@ -0,0 +1,59 @@ +{ + "results": [ + { + "name": "toolsearch", + "arguments": { + "query": " " + }, + "status": "error", + "text": "query must not be empty", + "details": { + "_tag": "ToolSearchFailed", + "message": "query must not be empty" + } + }, + { + "name": "toolsearch", + "arguments": { + "query": "weather", + "limit": 0 + }, + "status": "error", + "text": "limit must be a positive integer", + "details": { + "_tag": "ToolSearchFailed", + "message": "limit must be a positive integer" + } + }, + { + "name": "toolsearch", + "arguments": { + "query": "weather", + "limit": 1.5 + }, + "status": "error", + "text": "limit must be a positive integer", + "details": { + "_tag": "ToolSearchFailed", + "message": "limit must be a positive integer" + } + }, + { + "name": "toolsearch", + "arguments": { + "query": "zebra xylophone" + }, + "status": "completed", + "text": "No matching tools found.", + "details": { + "loaded": [] + } + } + ], + "agent": [ + { + "seq": 1, + "value": {} + } + ] +} diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json b/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json new file mode 100644 index 00000000..9dd4139e --- /dev/null +++ b/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json @@ -0,0 +1,89 @@ +{ + "firstRequest": { + "tools": [ + "bash", + "read", + "write", + "edit", + "toolsearch", + "notes_read" + ], + "system": [ + { + "added": [], + "removed": [ + "notes_read" + ] + } + ] + }, + "results": [ + { + "name": "notes_read", + "arguments": {}, + "status": "error", + "text": "Tool \"notes_read\" is not available in this thread.", + "details": { + "_tag": "ToolUnavailable", + "name": "notes_read" + } + }, + { + "name": "stock_quote", + "arguments": { + "symbol": "ACME" + }, + "status": "error", + "text": "Tool \"stock_quote\" is not available in this thread.", + "details": { + "_tag": "ToolUnavailable", + "name": "stock_quote" + } + }, + { + "name": "no_such_tool", + "arguments": {}, + "status": "error", + "text": "Unknown Tool: no_such_tool", + "details": { + "error": "unknown-tool", + "name": "no_such_tool" + } + } + ], + "entries": [ + { + "id": "E1", + "seq": 4, + "type": "user", + "state": "committed" + }, + { + "id": "E2", + "seq": 5, + "type": "system", + "state": "committed", + "data": { + "toolsAdded": [], + "toolsRemoved": [ + { + "name": "notes_read" + } + ], + "hash": "d6d66014ddda5da53e6e8e6520f0e6e1851a1a9c246d23802463da98342397a1" + } + }, + { + "id": "E3", + "seq": 6, + "type": "assistant", + "state": "committed" + }, + { + "id": "E4", + "seq": 12, + "type": "assistant", + "state": "committed" + } + ] +} diff --git a/packages/harness/test/__artifacts__/tools.write.json b/packages/harness/test/__artifacts__/tools.write.json index 0ac2a8b2..ce015788 100644 --- a/packages/harness/test/__artifacts__/tools.write.json +++ b/packages/harness/test/__artifacts__/tools.write.json @@ -94,11 +94,11 @@ "content": [ { "type": "text", - "text": "Invalid arguments for write: Missing key\n at [\"content\"]" + "text": "Invalid Arguments for write: Missing key\n at [\"content\"]" } ], "details": { - "error": "invalid_arguments", + "error": "invalid-arguments", "name": "write" } } diff --git a/packages/harness/test/context.test.ts b/packages/harness/test/context.test.ts index 028c7d88..dd05633b 100644 --- a/packages/harness/test/context.test.ts +++ b/packages/harness/test/context.test.ts @@ -170,11 +170,15 @@ describe("context assembly", () => { expect(snapshot.leafEntryId).toBe("a2"); expect(snapshot.lastAssistant).toMatchObject({ entryId: "a2", message: { messageId: "a2" } }); expect(snapshot.messages.map((message) => message.messageId)).toEqual(["compact1", "u2", "synthetic1", "a2"]); - expect(snapshot.messages[0]?.parts[0]).toEqual({ + const firstPart = (index: number) => { + const message = snapshot.messages[index]; + return message?.role === "system" ? undefined : message?.parts[0]; + }; + expect(firstPart(0)).toEqual({ type: "text", text: "The conversation history before this point was compacted into the following summary:\n\n\nEarlier work was summarized.\n", }); - expect(snapshot.messages[2]?.parts[0]).toEqual({ type: "text", text: "file contents" }); + expect(firstPart(2)).toEqual({ type: "text", text: "file contents" }); const line = yield* threads.line(thread.id); const compacted = line.find((item) => item.entry.id === "compact1"); diff --git a/packages/harness/test/control.prompt.test.ts b/packages/harness/test/control.prompt.test.ts index 104ba6d3..ea1923cb 100644 --- a/packages/harness/test/control.prompt.test.ts +++ b/packages/harness/test/control.prompt.test.ts @@ -131,7 +131,7 @@ describe("Control.prompt", () => { const line = yield* threads.line(threadId); expect(line.map((h) => h.entry.id)).toEqual([messageId]); expect(line[0]!.entry.seq).toBe(promoted.promotedSeq); - expect((yield* ContextCodec.decodeMessage(line[0]!)).parts[0]).toMatchObject({ + expect(ContextCodec.parts(yield* ContextCodec.decodeMessage(line[0]!))[0]).toMatchObject({ type: "text", text: "Promote once", }); @@ -319,7 +319,9 @@ describe("Control.cancel", () => { expect(yield* inputs.promoteFollowUp(threadId)).toBe(true); const line = yield* threads.line(threadId); expect(line.map((h) => h.entry.id)).toEqual([messageId]); - expect((yield* ContextCodec.decodeMessage(line[0]!)).parts[0]).toMatchObject({ text: "Fix it another way" }); + expect(ContextCodec.parts(yield* ContextCodec.decodeMessage(line[0]!))[0]).toMatchObject({ + text: "Fix it another way", + }); })); // The promotion the runner had already read for a row that is then cancelled: it must lose, @@ -455,7 +457,7 @@ describe("Control.update", () => { yield* inputs.promoteSteers(threadId, Number.MAX_SAFE_INTEGER); const line = yield* threads.line(threadId); expect(line.map((h) => h.entry.id)).toEqual([first.id, second.id]); - expect((yield* ContextCodec.decodeMessage(line[0]!)).parts[0]).toMatchObject({ + expect(ContextCodec.parts(yield* ContextCodec.decodeMessage(line[0]!))[0]).toMatchObject({ text: "Fix the flaky test only", }); })); @@ -510,7 +512,7 @@ describe("Control.update", () => { expect(line).toEqual([]); expect(PromptSchema.Prompt.text((yield* control.pending(threadId))[0]!.prompt)).toBe("Rewritten"); } else { - expect((yield* ContextCodec.decodeMessage(line[0]!)).parts[0]).toMatchObject({ + expect(ContextCodec.parts(yield* ContextCodec.decodeMessage(line[0]!))[0]).toMatchObject({ text: updated ? "Rewritten" : "Original", }); } @@ -574,7 +576,9 @@ describe("Control queue races", () => { expect(yield* inputs.promoteSteers(threadId, Number.MAX_SAFE_INTEGER)).toBe(1); const line = yield* threads.line(threadId); expect(line.map((h) => h.entry.id)).toEqual([first.id, second.id]); - expect((yield* ContextCodec.decodeMessage(line[1]!)).parts[0]).toMatchObject({ text: "Second, rewritten" }); + expect(ContextCodec.parts(yield* ContextCodec.decodeMessage(line[1]!))[0]).toMatchObject({ + text: "Second, rewritten", + }); })); // The prompt was cancelled and sent again, unchanged, after the promotion selected it: the stale diff --git a/packages/harness/test/fixtures/bash.spec.ts b/packages/harness/test/fixtures/bash.spec.ts index 92977935..dd38a637 100644 --- a/packages/harness/test/fixtures/bash.spec.ts +++ b/packages/harness/test/fixtures/bash.spec.ts @@ -46,7 +46,9 @@ export const bashPluginSpec = (options: { const path = yield* session.history(); const messages = yield* Effect.forEach(path, ContextCodec.decodeMessage); expect(path.every((entry) => entry.entry.state === "committed")).toBe(true); - const calls = messages.flatMap((message) => message.parts.filter((part) => part.type === "toolCall")); + const calls = messages.flatMap((message) => + ContextCodec.parts(message).filter((part) => part.type === "toolCall"), + ); expect(calls).toHaveLength(1); const call = calls[0]; if (call === undefined || call.type !== "toolCall" || (call.status !== "completed" && call.status !== "error")) @@ -213,7 +215,9 @@ export const bashPluginSpec = (options: { const final = messages.at(-1); expect(final?.role).toBe("assistant"); expect( - final?.parts.flatMap((part) => (part.type === "text" ? [part.text] : [])).join(""), + (final === undefined ? [] : ContextCodec.parts(final)) + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join(""), ).toContain(marker); }), 180_000, diff --git a/packages/harness/test/plugin.external.test.ts b/packages/harness/test/plugin.external.test.ts index c3865f87..76d21576 100644 --- a/packages/harness/test/plugin.external.test.ts +++ b/packages/harness/test/plugin.external.test.ts @@ -342,7 +342,7 @@ describe("third-party plugins", () => { expect(prompts[0]).toContain("\n(none)\n"); const settled = JSON.parse(path[1]?.parts[0]?.data ?? "{}"); expect(settled).toMatchObject({ status: "error", result: { isError: true } }); - expect(settled.result.content[0].text).toContain("Unknown tool: bash"); + expect(settled.result.content[0].text).toContain("Unknown Tool: bash"); })); }); diff --git a/packages/harness/test/runner.turns.live.test.ts b/packages/harness/test/runner.turns.live.test.ts index 336c9317..33dee212 100644 --- a/packages/harness/test/runner.turns.live.test.ts +++ b/packages/harness/test/runner.turns.live.test.ts @@ -101,12 +101,16 @@ describe.each(LIVE)("turn snapshots against live $name", (live) => { "the tools returned, separated by one space, and nothing else.", ); const messages = yield* Effect.forEach(yield* handle.main.history(), ContextCodec.decodeMessage); - const last = messages.findLast((message) => message.parts.some((part) => part.type === "text")); + const last = messages.findLast((message) => + ContextCodec.parts(message).some((part) => part.type === "text"), + ); return { calls: messages - .flatMap((message) => message.parts) + .flatMap((message) => ContextCodec.parts(message)) .flatMap((part) => (part.type === "toolCall" ? [`${part.name}:${part.status}`] : [])), - reply: (last?.parts ?? []).flatMap((part) => (part.type === "text" ? [part.text] : [])).join(""), + reply: (last === undefined ? [] : ContextCodec.parts(last)) + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join(""), }; }).pipe( Effect.provide( diff --git a/packages/harness/test/tools.edit.test.ts b/packages/harness/test/tools.edit.test.ts index 5ad48bb6..eec5f545 100644 --- a/packages/harness/test/tools.edit.test.ts +++ b/packages/harness/test/tools.edit.test.ts @@ -249,7 +249,7 @@ const editAll = (root: string, custom: string) => const path = yield* session.history(); expect(path.every((entry) => entry.entry.state === "committed")).toBe(true); const messages = yield* Effect.forEach(path, ContextCodec.decodeMessage); - return messages.flatMap((message) => message.parts.filter((part) => part.type === "toolCall")); + return messages.flatMap((message) => ContextCodec.parts(message).filter((part) => part.type === "toolCall")); }).pipe( Effect.provide( Harness.layer({ diff --git a/packages/harness/test/tools.empty.live.test.ts b/packages/harness/test/tools.empty.live.test.ts index e00a0a5c..7fcece75 100644 --- a/packages/harness/test/tools.empty.live.test.ts +++ b/packages/harness/test/tools.empty.live.test.ts @@ -57,11 +57,15 @@ describe.each(direct)("parameterless tool against live $name", (live) => { yield* session.run("Call the finite tool, then reply with exactly the token it returned and nothing else."); const path = yield* session.history(); const messages = yield* Effect.forEach(path, ContextCodec.decodeMessage); - const parts = messages.flatMap((message) => message.parts); - const last = messages.findLast((message) => message.parts.some((part) => part.type === "text")); + const parts = messages.flatMap((message) => ContextCodec.parts(message)); + const last = messages.findLast((message) => + ContextCodec.parts(message).some((part) => part.type === "text"), + ); return { calls: parts.flatMap((part) => (part.type === "toolCall" ? [part] : [])), - reply: (last?.parts ?? []).flatMap((part) => (part.type === "text" ? [part.text] : [])).join(""), + reply: (last === undefined ? [] : ContextCodec.parts(last)) + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join(""), }; }).pipe( Effect.provide( diff --git a/packages/harness/test/tools.read.test.ts b/packages/harness/test/tools.read.test.ts index 79090e26..83856513 100644 --- a/packages/harness/test/tools.read.test.ts +++ b/packages/harness/test/tools.read.test.ts @@ -143,7 +143,7 @@ const readAll = (root: string, custom: string) => const path = yield* session.history(); expect(path.every((entry) => entry.entry.state === "committed")).toBe(true); const messages = yield* Effect.forEach(path, ContextCodec.decodeMessage); - return messages.flatMap((message) => message.parts.filter((part) => part.type === "toolCall")); + return messages.flatMap((message) => ContextCodec.parts(message).filter((part) => part.type === "toolCall")); }).pipe( Effect.provide( Harness.layer({ diff --git a/packages/harness/test/tools.registry.test.ts b/packages/harness/test/tools.registry.test.ts index d77be9d5..e451c170 100644 --- a/packages/harness/test/tools.registry.test.ts +++ b/packages/harness/test/tools.registry.test.ts @@ -45,7 +45,7 @@ describe("ToolRegistry — execution through the snapshot", () => { const outcome = await Effect.runPromise(resolved.handle(pendingCall("ghost"))); expect(outcome.status).toBe("error"); - expect(outcome.result.details).toMatchObject({ error: "unknown_tool", name: "ghost" }); + expect(outcome.result.details).toMatchObject({ error: "unknown-tool", name: "ghost" }); }); it("maps an interrupted handler to an aborted outcome carrying the last reported partial", async () => { diff --git a/packages/harness/test/tools.search.live.test.ts b/packages/harness/test/tools.search.live.test.ts index 57b92833..953acc3f 100644 --- a/packages/harness/test/tools.search.live.test.ts +++ b/packages/harness/test/tools.search.live.test.ts @@ -60,11 +60,15 @@ describe.each(direct)("search tools against live $name", (live) => { yield* session.run(PROMPT); const path = yield* session.history(); const messages = yield* Effect.forEach(path, ContextCodec.decodeMessage); - const parts = messages.flatMap((message) => message.parts); - const last = messages.findLast((message) => message.parts.some((part) => part.type === "text")); + const parts = messages.flatMap((message) => ContextCodec.parts(message)); + const last = messages.findLast((message) => + ContextCodec.parts(message).some((part) => part.type === "text"), + ); return { calls: parts.flatMap((part) => (part.type === "toolCall" ? [part] : [])), - reply: (last?.parts ?? []).flatMap((part) => (part.type === "text" ? [part.text] : [])).join(""), + reply: (last === undefined ? [] : ContextCodec.parts(last)) + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join(""), }; }).pipe( Effect.provide( diff --git a/packages/harness/test/tools.toolsearch.live.test.ts b/packages/harness/test/tools.toolsearch.live.test.ts new file mode 100644 index 00000000..4ff4bacf --- /dev/null +++ b/packages/harness/test/tools.toolsearch.live.test.ts @@ -0,0 +1,117 @@ +import type { Message } from "@codeworksh/aikit"; +import { Effect, Schema } from "effect"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vite-plus/test"; +import { ContextCodec } from "../src/context/codec.ts"; +import { Harness } from "../src/effect/harness.ts"; +import { Session } from "../src/effect/session.ts"; +import { builtins } from "../src/plugin/builtin.ts"; +import { TOOL_SEARCH_NAME, toolSearchPlugin } from "../src/plugin/builtin/tool/toolsearch.ts"; +import { define } from "../src/plugin/plugin.ts"; +import { LLM } from "../src/runner/llm.ts"; +import * as Tool from "../src/tool/tool.ts"; +import { withSettings } from "./fixtures/settings.ts"; +import { available, LIVE } from "./utils/live.ts"; + +/** + * One run against a real model: the token is held by a deferred tool the model is not offered, + * so a right reply means it searched, the load reached its next request, and it called the tool. + */ + +const TOKEN = "vq7-lumen-4821"; +const codex = LIVE.find((one) => one.name === "openai-codex")!; +const MODEL = "gpt-6-luna"; + +/** A request as sent: its declared tools, and the tools its system messages add. */ +type Sent = { readonly tools: ReadonlyArray; readonly system: ReadonlyArray> }; +const sentOf = (context: Message.Context): Sent => ({ + tools: (context.tools ?? []).map((tool) => tool.name), + system: context.messages.flatMap((message) => + message.role === "system" ? [(message.toolsAdded ?? []).map((tool) => tool.name)] : [], + ), +}); + +describe("toolsearch against live openai-codex", () => { + it.skipIf(!available(codex))("finds a deferred tool, then calls it", { timeout: 300_000 }, () => + withSettings(async ({ root, custom }) => { + await writeFile( + join(custom, "settings.jsonc"), + JSON.stringify({ model: { provider: codex.provider, id: MODEL } }), + ); + const vault = define({ + id: "acme.tool.vault", + kind: "tool", + setup(ctx) { + ctx.plugin.tools.add( + Tool.register( + Tool.make({ + name: "vault_read_token", + description: "Read the deployment access token stored in the secrets vault.", + exposure: "deferred", + parameters: Schema.Struct({}), + success: Schema.String, + encodeContent: (value: string) => [{ type: "text", text: value }], + handler: () => Effect.succeed(TOKEN), + }), + ), + ); + }, + }); + + const sent: Sent[] = []; + const llm: LLM.Open = (input, signal) => { + sent.push(sentOf(input.context)); + return LLM.open(input, signal); + }; + + const { calls, reply } = await Effect.gen(function* () { + const handle = yield* Session.create({ directory: root }); + yield* handle.main.run( + "Find the tool that reads the deployment access token from the secrets vault, call it, " + + "and reply with exactly the token it returns and nothing else.", + ); + const messages = yield* Effect.forEach(yield* handle.main.history(), ContextCodec.decodeMessage); + const last = messages.findLast((message) => + ContextCodec.parts(message).some((part) => part.type === "text"), + ); + return { + calls: messages + .flatMap((message) => ContextCodec.parts(message)) + .flatMap((part) => (part.type === "toolCall" ? [`${part.name}:${part.status}`] : [])), + reply: (last === undefined ? [] : ContextCodec.parts(last)) + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join(""), + }; + }).pipe( + Effect.provide( + Harness.layer({ + home: join(root, "home"), + hostCwd: root, + database: ":memory:", + userConfigDir: custom, + // toolsearch ships off (`optIn`); this entry turns it on. + plugins: [...builtins, vault, { plugin: toolSearchPlugin.id, enabled: true }], + llm, + }), + ), + Effect.scoped, + Effect.timeout("240 seconds"), + Effect.runPromise, + ); + + const found = reply.includes(TOKEN); + // The request that called the vault tool is the first one whose system messages add it. + const loadedAt = sent.findIndex((one) => one.system.some((added) => added.includes("vault_read_token"))); + expect(calls).toContain(`${TOOL_SEARCH_NAME}:completed`); + expect(calls).toContain("vault_read_token:completed"); + expect(loadedAt).toBeGreaterThan(0); + expect(sent.every((one) => !one.tools.includes("vault_read_token"))).toBe(true); + expect(found).toBe(true); + + await expect(`${JSON.stringify({ requests: sent, calls, found }, null, "\t")}\n`).toMatchFileSnapshot( + "./__artifacts__/tools.toolsearch.live.openai-codex.json", + ); + }), + ); +}); diff --git a/packages/harness/test/tools.toolsearch.test.ts b/packages/harness/test/tools.toolsearch.test.ts new file mode 100644 index 00000000..3467a397 --- /dev/null +++ b/packages/harness/test/tools.toolsearch.test.ts @@ -0,0 +1,539 @@ +import "./utils/env.ts"; +import { createAssistantMessageEventStream, type Message } from "@codeworksh/aikit"; +import { DateTime, Effect, Layer, Schema } from "effect"; +import { SqlClient } from "effect/sql"; +import { describe, expect } from "vite-plus/test"; +import { ContextCodec } from "../src/context/codec.ts"; +import { Context } from "../src/context/context.ts"; +import { Control } from "../src/control.ts"; +import { Database } from "../src/db/db.ts"; +import { Document } from "../src/document/document.ts"; +import { Event } from "../src/event/event.ts"; +import { EventList } from "../src/event/list.ts"; +import { EventRegistry } from "../src/event/registry.ts"; +import { Global } from "../src/global.ts"; +import { builtins } from "../src/plugin/builtin.ts"; +import { DEFAULT_LIMIT, TOOL_SEARCH_NAME, toolSearchPlugin } from "../src/plugin/builtin/tool/toolsearch.ts"; +import type { Plugin } from "../src/plugin/plugin.ts"; +import { RunnerExecute } from "../src/runner/execute.ts"; +import { Hooks } from "../src/runner/hooks.ts"; +import { LLM } from "../src/runner/llm.ts"; +import { Loop } from "../src/runner/loop.ts"; +import { SandboxController } from "../src/sandbox/control.ts"; +import { SandboxDriverRegistry } from "../src/sandbox/registry.ts"; +import { PromptSchema } from "../src/session/prompt/schema.ts"; +import { SessionProjector } from "../src/session/projector.ts"; +import { SessionRuntime } from "../src/session/runtime.ts"; +import { SessionSchema } from "../src/session/schema.ts"; +import { Session } from "../src/session/session.ts"; +import { Settings } from "../src/settings/settings.ts"; +import { State } from "../src/state/state.ts"; +import { ThreadSchema } from "../src/thread/schema.ts"; +import { Thread } from "../src/thread/thread.ts"; +import * as Tool from "../src/tool/tool.ts"; +import { assistant } from "./fixtures/llm.ts"; +import { pooled } from "./fixtures/pool.ts"; +import { seedSpace } from "./fixtures/space.ts"; +import { testEffect } from "./utils/effect.ts"; + +/** + * `toolsearch` end to end, against a scripted model: what a search loads, how the next request + * learns of it (a transcript `system` message, never a new `tools` list), what a fork keeps, and + * what calling a tool the thread does not offer returns. + */ + +interface Call { + readonly name: string; + readonly arguments: Record; +} +/** What the model does on a thread's next request: call tools, or reply and stop. */ +type Step = { readonly _tag: "call"; readonly calls: ReadonlyArray } | { readonly _tag: "reply" }; + +const call = (...calls: ReadonlyArray): Step => ({ _tag: "call", calls }); +const reply: Step = { _tag: "reply" }; +const search = (query: string, limit?: number): Call => ({ + name: TOOL_SEARCH_NAME, + arguments: limit === undefined ? { query } : { query, limit }, +}); + +const script = new Map>(); +const requests: Array<{ readonly threadId: string; readonly context: Message.Context }> = []; +let responses = 0; + +const open: LLM.Open = (input) => + Effect.sync(() => { + const step = script.get(input.threadId)?.shift() ?? reply; + requests.push({ threadId: input.threadId, context: input.context }); + responses += 1; + const message = assistant( + input, + responses, + step._tag === "call" + ? { + stopReason: "toolUse", + parts: step.calls.map((one, index) => ({ + type: "toolCall" as const, + callID: `call_${responses}_${index + 1}`, + name: one.name, + arguments: one.arguments, + status: "pending" as const, + time: { start: 1, end: 1 }, + })), + } + : {}, + ); + const stream = createAssistantMessageEventStream(); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: step._tag === "call" ? "toolUse" : "stop", message }); + return stream; + }); + +const text = (description: string) => Schema.String.annotate({ description }); + +const deferred = (name: string, description: string, parameters: Schema.Struct.Fields) => + Tool.register( + Tool.make({ + name, + description, + exposure: "deferred", + parameters: Schema.Struct(parameters), + success: Schema.String, + encodeContent: (value: string) => [{ type: "text", text: value }], + handler: () => Effect.succeed(`${name} ran`), + }), + ); + +/** Ten deferred tools, one direct one, and two (codemode, hidden) a search must never load. */ +const catalog: Plugin = { + id: "test.tool.catalog", + kind: "tool", + setup: (ctx) => { + const tools = [ + deferred("weather_forecast", "Get the weather forecast for a city.\nReturns daily highs and lows.", { + city: text("City to get the weather forecast for"), + days: Schema.optional(Schema.Finite.annotate({ description: "Number of forecast days" })), + }), + deferred("calendar_create_event", "Create an event in the user's calendar.", { + title: text("Event title"), + start: text("Start time as an ISO date"), + }), + deferred("github_create_issue", "Create an issue in a GitHub repository.", { + repository: text("Repository as owner/name"), + title: text("Issue title"), + }), + deferred("github_list_pull_requests", "List open pull requests of a GitHub repository.", { + repository: text("Repository as owner/name"), + }), + deferred("slack_send_message", "Send a message to a Slack channel.", { + channel: text("Channel to send the message to"), + message: text("Message text"), + }), + deferred("jira_search_issues", "Search Jira issues with a JQL query.", { + query: text("JQL query to search issues with"), + }), + deferred("email_send", "Send an email message to a recipient.", { + to: text("Recipient email address"), + body: text("Message body"), + }), + deferred("translate_text", "Translate text between languages.", { + text: text("Text to translate"), + target: text("Target language code"), + }), + deferred("stock_quote", "Get the latest stock quote for a ticker symbol.", { + symbol: text("Ticker symbol to get a quote for"), + }), + deferred("database_query", "Run a read-only SQL query against the analytics database.", { + sql: text("SQL query to run"), + }), + ]; + for (const tool of tools) ctx.plugin.tools.add(tool); + ctx.plugin.tools.add( + Tool.register( + Tool.make({ + name: "notes_read", + description: "Read the user's notes.", + parameters: Schema.Struct({}), + success: Schema.String, + encodeContent: (value: string) => [{ type: "text", text: value }], + handler: () => Effect.succeed("notes"), + }), + ), + ); + for (const exposure of ["codemode", "hidden"] as const) { + ctx.plugin.tools.add( + Tool.register( + Tool.make({ + name: `weather_${exposure}`, + description: "Get the weather forecast for a city, issue, message, query, quote or event.", + exposure, + parameters: Schema.Struct({ city: text("City to get the weather forecast for") }), + success: Schema.String, + handler: () => Effect.succeed("never"), + }), + ), + ); + } + }, +}; + +const layer = () => { + const database = Database.layer(":memory:"); + const sandbox = SandboxController.layer({ hostCwd: "/" }).pipe( + Layer.provideMerge(SandboxDriverRegistry.layer()), + Layer.provideMerge(database), + ); + // toolsearch ships off (`optIn`); this selection turns it on. + const seeded = pooled([ + ...builtins.filter((plugin) => plugin.id !== toolSearchPlugin.id), + { ...toolSearchPlugin, optIn: false }, + catalog, + ]); + return Control.layer.pipe( + Layer.provideMerge( + RunnerExecute.layer.pipe( + Layer.provide(Loop.layer({ request: LLM.make(open) }).pipe(Layer.provide(Hooks.layer))), + ), + ), + Layer.provideMerge(State.layer({}, seeded.ref, seeded.references, seeded.follow, seeded.rebuild)), + Layer.provideMerge(SessionRuntime.layer), + Layer.provideMerge(Layer.succeed(Settings.Service, { load: () => Effect.succeed(Settings.defaults) })), + Layer.provideMerge(sandbox), + Layer.provideMerge(Context.layer), + Layer.provideMerge(SessionProjector.layer), + Layer.provideMerge(Session.storeLayer), + Layer.provideMerge(Event.layer), + Layer.provideMerge(EventRegistry.layer()), + Layer.provideMerge(database), + Layer.provideMerge(Layer.succeed(Global.Service, Global.make({ home: "/unused/home" }))), + ); +}; + +const { effect: it } = testEffect(layer()); + +const reset = Effect.sync(() => { + script.clear(); + requests.length = 0; + responses = 0; +}); + +const createSession = Effect.gen(function* () { + const sessions = yield* Session.Service; + const { spaceId, location } = yield* seedSpace({ location: process.cwd(), projectId: "p" }); + const { session, thread } = yield* sessions.create({ spaceId, directory: location }); + return { sessionId: SessionSchema.ID.make(session.id), main: ThreadSchema.ID.make(thread.id) }; +}); + +const run = (sessionId: SessionSchema.ID, threadId: ThreadSchema.ID, prompt: string, steps: ReadonlyArray) => + Effect.gen(function* () { + script.set(threadId, [...steps]); + const control = yield* Control.Service; + yield* control.run({ sessionId, threadId, prompt: PromptSchema.Prompt.fromText(prompt), delivery: "followUp" }); + }); + +/** A request as the model saw it: its declared tools, and the tool changes its system messages carry. */ +const sent = (context: Message.Context) => ({ + tools: (context.tools ?? []).map((tool) => tool.name), + system: context.messages.flatMap((message) => + message.role === "system" + ? [ + { + added: (message.toolsAdded ?? []).map((tool) => tool.name), + removed: (message.toolsRemoved ?? []).map((tool) => tool.name), + }, + ] + : [], + ), +}); +const requestsOf = (threadId: string) => requests.filter((one) => one.threadId === threadId).map((one) => one.context); + +/** Every settled tool call on a thread's line, as the model reads it back. */ +const results = Effect.fnUntraced(function* (threadId: ThreadSchema.ID) { + const threads = yield* Thread.Service; + const line = yield* threads.line(threadId); + const messages = yield* Effect.forEach( + line.filter(({ entry }) => entry.type === "assistant" && entry.threadId === threadId), + ContextCodec.decodeMessage, + ); + return messages.flatMap((message) => + ContextCodec.parts(message).flatMap((part) => + part.type === "toolCall" && part.status !== "pending" && part.status !== "running" + ? [ + { + name: part.name, + arguments: part.arguments, + status: part.status, + text: part.result.content.flatMap((one) => (one.type === "text" ? [one.text] : [])).join(""), + details: part.result.details, + }, + ] + : [], + ), + ); +}); + +/** The thread's events, entries and `agent` revisions, by seq; ids aliased in order seen. */ +const ledger = Effect.fnUntraced(function* (sessionId: SessionSchema.ID, threadId: ThreadSchema.ID) { + const sql = yield* SqlClient.SqlClient; + const aliases = new Map(); + const alias = (id: string) => { + const known = aliases.get(id); + if (known !== undefined) return known; + const name = `E${aliases.size + 1}`; + aliases.set(id, name); + return name; + }; + const events = yield* sql<{ seq: number; type: string; data: string }>` + SELECT seq, type, data FROM event WHERE aggregate_id = ${sessionId} ORDER BY seq + `; + const entries = yield* sql<{ id: string; seq: number; type: string; state: string; data: string }>` + SELECT id, seq, type, state, data FROM entry WHERE thread_id = ${threadId} ORDER BY seq + `; + const revisions = yield* sql<{ seq: number; value: string }>` + SELECT revision.seq, revision.value FROM revision JOIN document ON document.id = revision.document_id + WHERE document.thread_id = ${threadId} AND document.kind = 'agent' ORDER BY revision.seq + `; + return { + events: events + .filter((event) => (JSON.parse(event.data) as { readonly threadId?: string }).threadId === threadId) + .map((event) => ({ seq: event.seq, type: event.type })), + entries: entries.map((entry) => ({ + id: alias(entry.id), + seq: entry.seq, + type: entry.type, + state: entry.state, + ...(entry.type === "system" ? { data: JSON.parse(entry.data) as unknown } : {}), + })), + agent: revisions.map((revision) => ({ seq: revision.seq, value: JSON.parse(revision.value) as unknown })), + }; +}); +type Ledger = Effect.Success>; + +const systemEntries = (book: Ledger) => book.entries.filter((entry) => entry.type === "system"); +const settledSeqs = (book: Ledger) => + book.events.filter((event) => event.type.startsWith(`${EventList.ToolSettled.type}.`)).map((event) => event.seq); + +const artifact = (name: string, value: unknown) => + Effect.promise(() => + expect(`${JSON.stringify(value, null, "\t")}\n`).toMatchFileSnapshot(`./__artifacts__/${name}`), + ); + +describe("toolsearch", () => { + it( + "loads a tool: the agent revision lands at the settle, the next request declares it by a system message", + Effect.gen(function* () { + yield* reset; + const { sessionId, main } = yield* createSession; + yield* run(sessionId, main, "What is the weather in Lisbon?", [ + call(search("weather forecast", 1)), + call({ name: "weather_forecast", arguments: { city: "Lisbon" } }), + reply, + ]); + + const [first, second, third] = requestsOf(main).map(sent); + const outcomes = yield* results(main); + const book = yield* ledger(sessionId, main); + + expect(outcomes[0]).toMatchObject({ + name: TOOL_SEARCH_NAME, + status: "completed", + text: "Loaded 1 tool. They are available from your next call:\n- weather_forecast: Get the weather forecast for a city.", + details: { loaded: ["weather_forecast"] }, + }); + // The load is one `agent` revision, written by the settle event itself. + const loads = book.agent.filter((revision) => JSON.stringify(revision.value).includes("weather_forecast")); + expect(loads).toHaveLength(1); + expect(settledSeqs(book)).toContain(loads[0]!.seq); + expect(loads[0]!.value).toEqual({ modelTools: { add: ["weather_forecast"] } }); + + // The request's `tools` never change; the loaded tool arrives as a system message. + expect(first!.tools).toContain(TOOL_SEARCH_NAME); + expect(first!.tools).not.toContain("weather_forecast"); + expect(second!.tools).toEqual(first!.tools); + expect(first!.system).toEqual([]); + expect(second!.system).toEqual([{ added: ["weather_forecast"], removed: [] }]); + // The third request replays the one system entry; nothing new is declared. + expect(third!.tools).toEqual(first!.tools); + expect(third!.system).toEqual(second!.system); + expect(systemEntries(book)).toHaveLength(1); + expect(systemEntries(book)[0]!.data).toMatchObject({ + toolsAdded: [{ name: "weather_forecast" }], + toolsRemoved: [], + }); + // The loaded tool runs. + expect(outcomes[1]).toMatchObject({ + name: "weather_forecast", + status: "completed", + text: "weather_forecast ran", + }); + + yield* artifact("tools.toolsearch.load.json", { + requests: [first, second, third], + results: outcomes, + ...book, + }); + }), + ); + + it( + "refuses an empty query and a bad limit, and reports no matches", + Effect.gen(function* () { + yield* reset; + const { sessionId, main } = yield* createSession; + yield* run(sessionId, main, "Find me tools", [ + call(search(" "), search("weather", 0), search("weather", 1.5), search("zebra xylophone")), + reply, + ]); + const outcomes = yield* results(main); + const book = yield* ledger(sessionId, main); + + expect(outcomes.map((one) => [one.status, one.text])).toEqual([ + ["error", "query must not be empty"], + ["error", "limit must be a positive integer"], + ["error", "limit must be a positive integer"], + ["completed", "No matching tools found."], + ]); + for (const failed of outcomes.slice(0, 3)) expect(failed.details).toMatchObject({ _tag: "ToolSearchFailed" }); + expect(outcomes[3]!.details).toEqual({ loaded: [] }); + // Nothing loaded: no `agent` change and no system entry. + expect(book.agent.some((revision) => JSON.stringify(revision.value).includes("modelTools"))).toBe(false); + expect(systemEntries(book)).toEqual([]); + expect( + requestsOf(main) + .map(sent) + .map((one) => one.system), + ).toEqual([[], []]); + yield* artifact("tools.toolsearch.refused.json", { results: outcomes, agent: book.agent }); + }), + ); + + it( + "returns at most `limit` tools, best first, and 8 by default", + Effect.gen(function* () { + yield* reset; + const { sessionId, main } = yield* createSession; + // Both run against the same request's candidates: all ten deferred tools. + const broad = "create send search list get query translate run weather issue message quote event"; + yield* run(sessionId, main, "Find me tools", [call(search("github issue", 2), search(broad)), reply]); + const [limited, all] = yield* results(main); + const loadedOf = (details: unknown) => (details as { readonly loaded: ReadonlyArray }).loaded; + + expect(loadedOf(limited!.details)).toEqual(["github_create_issue", "jira_search_issues"]); + expect(loadedOf(all!.details)).toHaveLength(DEFAULT_LIMIT); + for (const never of ["weather_codemode", "weather_hidden", "notes_read", TOOL_SEARCH_NAME]) + expect(loadedOf(all!.details)).not.toContain(never); + // Both loads are on the agent, the second appended after the first. + const add = (yield* (yield* Document.Service).agent(main)).modelTools?.add ?? []; + expect(add.slice(0, 2)).toEqual(loadedOf(limited!.details)); + expect(new Set(add)).toEqual(new Set([...loadedOf(limited!.details), ...loadedOf(all!.details)])); + const [, next] = requestsOf(main).map(sent); + expect(next!.system).toHaveLength(1); + expect(next!.system[0]!.added).toEqual(add); + + yield* artifact("tools.toolsearch.limit.json", { + results: [limited, all], + agentAdd: add, + nextRequest: next, + }); + }), + ); + + it( + "a fork cut before the load does not offer the tool; one cut at the load does", + Effect.gen(function* () { + yield* reset; + const threads = yield* Thread.Service; + const documents = yield* Document.Service; + const { sessionId, main } = yield* createSession; + yield* run(sessionId, main, "Load the forecast tool", [call(search("weather forecast", 1)), reply]); + const line = yield* threads.line(main); + expect(line.map(({ entry }) => entry.type)).toEqual(["user", "assistant", "system", "assistant"]); + const [prompt, searched] = line; + + const before = ThreadSchema.ID.make( + (yield* threads.fork({ threadId: main, entryId: prompt!.entry.id })).thread.id, + ); + const after = ThreadSchema.ID.make( + (yield* threads.fork({ threadId: main, entryId: searched!.entry.id })).thread.id, + ); + const beforeAgent = yield* documents.agent(before); + const afterAgent = yield* documents.agent(after); + expect(beforeAgent.modelTools).toBeUndefined(); + expect(afterAgent.modelTools).toEqual({ add: ["weather_forecast"] }); + + const weather: Call = { name: "weather_forecast", arguments: { city: "Porto" } }; + yield* run(sessionId, before, "Weather in Porto?", [call(weather), reply]); + yield* run(sessionId, after, "Weather in Porto?", [call(weather), reply]); + + const [beforeFirst] = requestsOf(before).map(sent); + const [afterFirst] = requestsOf(after).map(sent); + const beforeResults = yield* results(before); + const afterResults = yield* results(after); + expect(beforeFirst!.system).toEqual([]); + expect(beforeResults[0]).toMatchObject({ + status: "error", + text: 'Tool "weather_forecast" is not available in this thread.', + details: { _tag: "ToolUnavailable", name: "weather_forecast" }, + }); + // The history up to the cut has no system entry; the fork declares the tool itself. + expect(afterFirst!.system).toEqual([{ added: ["weather_forecast"], removed: [] }]); + expect(afterResults[0]).toMatchObject({ status: "completed", text: "weather_forecast ran" }); + const afterBook = yield* ledger(sessionId, after); + expect(systemEntries(afterBook)).toHaveLength(1); + + yield* artifact("tools.toolsearch.fork.json", { + before: { agent: beforeAgent, firstRequest: beforeFirst, results: beforeResults }, + after: { agent: afterAgent, firstRequest: afterFirst, results: afterResults, entries: afterBook.entries }, + }); + }), + ); + + it( + "calling an unloaded deferred tool or a removed declared tool is unavailable", + Effect.gen(function* () { + yield* reset; + const events = yield* Event.Service; + const { sessionId, main } = yield* createSession; + yield* events.publish(EventList.ThreadConfigured, { + timestamp: yield* DateTime.now, + sessionId, + threadId: main, + change: { modelTools: { remove: ["notes_read"] } }, + }); + yield* run(sessionId, main, "Read my notes and the weather", [ + call( + { name: "notes_read", arguments: {} }, + { name: "stock_quote", arguments: { symbol: "ACME" } }, + { name: "no_such_tool", arguments: {} }, + ), + reply, + ]); + const outcomes = yield* results(main); + const book = yield* ledger(sessionId, main); + + expect(outcomes.map((one) => [one.name, one.status, one.text, one.details])).toEqual([ + [ + "notes_read", + "error", + 'Tool "notes_read" is not available in this thread.', + { _tag: "ToolUnavailable", name: "notes_read" }, + ], + [ + "stock_quote", + "error", + 'Tool "stock_quote" is not available in this thread.', + { _tag: "ToolUnavailable", name: "stock_quote" }, + ], + ["no_such_tool", "error", "Unknown Tool: no_such_tool", { error: "unknown-tool", name: "no_such_tool" }], + ]); + const [first] = requestsOf(main).map(sent); + yield* artifact("tools.toolsearch.unavailable.json", { + firstRequest: first, + results: outcomes, + entries: book.entries, + }); + // The removal is declared to the model as a system entry. + expect(systemEntries(book)).toHaveLength(1); + expect(systemEntries(book)[0]!.data).toMatchObject({ toolsRemoved: [{ name: "notes_read" }] }); + }), + ); +}); diff --git a/packages/harness/test/tools.write.test.ts b/packages/harness/test/tools.write.test.ts index 92c7894d..d6173c32 100644 --- a/packages/harness/test/tools.write.test.ts +++ b/packages/harness/test/tools.write.test.ts @@ -48,7 +48,7 @@ const writeAll = (root: string, custom: string) => const path = yield* session.history(); expect(path.every((entry) => entry.entry.state === "committed")).toBe(true); const messages = yield* Effect.forEach(path, ContextCodec.decodeMessage); - return messages.flatMap((message) => message.parts.filter((part) => part.type === "toolCall")); + return messages.flatMap((message) => ContextCodec.parts(message).filter((part) => part.type === "toolCall")); }).pipe( Effect.provide( Harness.layer({