diff --git a/packages/harness/src/mcp/mcp.ts b/packages/harness/src/mcp/mcp.ts index 8d32dbce..1e0d9d2c 100644 --- a/packages/harness/src/mcp/mcp.ts +++ b/packages/harness/src/mcp/mcp.ts @@ -32,7 +32,6 @@ * service down, stops the server process and ends the HTTP session. */ -import type { Message } from "@codeworksh/aikit"; import type { Exposure, ServerConfig } from "@codeworksh/plugin/plugin/mcp"; import type { AuthProvider, Tool } from "@modelcontextprotocol/client"; import { @@ -63,6 +62,7 @@ import { Event } from "../event/event.ts"; import { EventList } from "../event/list.ts"; import { ID as SessionId } from "../session/schema.ts"; import { ID as ThreadId } from "../thread/schema.ts"; +import type { DeferredTool } from "../tool/offer.ts"; import { type RegisteredTool, toAikitTool } from "../tool/tool.ts"; import { McpClient } from "./client.ts"; import { namespace } from "./config.ts"; @@ -135,7 +135,7 @@ export interface Reconciled { * The thread's deferred tools as they stand when it runs, under the names they are registered by. * Waits first, within {@link STARTUP_WAIT}, for those servers' connect attempts in progress. */ - readonly deferred: Effect.Effect>; + readonly deferred: Effect.Effect>; /** The `mcpservers` prompt section, as the servers stand once the ones with declared tools were waited for. */ readonly section: string | undefined; } @@ -925,9 +925,23 @@ export const layer = (options: { yield* Effect.forEach(servers, attempted, { concurrency: "unbounded", discard: true }).pipe( Effect.timeoutOption(STARTUP_WAIT), ); - return toolsOf(thread).flatMap((tool) => - tool.definition.exposure === "deferred" ? [toAikitTool(tool.definition)] : [], + // A tool is searched by its server's namespace text too: its name, description and instructions. + const namespaces = new Map( + [...thread.servers].flatMap(([name, { config, server }]) => { + if (server === undefined) return []; + const text = [namespace(name), config.description ?? "", server.instructions ?? ""].join(" "); + const names = thread.names( + name, + server.tools.map((tool) => tool.name), + ); + return [...names.values()].map((registered) => [registered, text] as const); + }), ); + return toolsOf(thread).flatMap((tool): ReadonlyArray => { + if (tool.definition.exposure !== "deferred") return []; + const text = namespaces.get(tool.definition.name); + return [{ ...toAikitTool(tool.definition), ...(text === undefined ? {} : { namespace: text }) }]; + }); }); const reconcile = Effect.fn("Mcp.reconcile")(function* (input: Input) { diff --git a/packages/harness/src/plugin/builtin/tool/toolsearch.ts b/packages/harness/src/plugin/builtin/tool/toolsearch.ts index d9382438..276a742e 100644 --- a/packages/harness/src/plugin/builtin/tool/toolsearch.ts +++ b/packages/harness/src/plugin/builtin/tool/toolsearch.ts @@ -60,7 +60,7 @@ export const toolSearchHandler: Tool.Handler< const deferred = yield* (yield* ToolOffer).deferred; const loaded = ToolSearch.rank( query, - deferred.map((tool) => ({ name: tool.name, text: ToolSearch.text(tool) })), + deferred.map((tool) => ({ name: tool.name, text: ToolSearch.text(tool), namespaceText: tool.namespace })), max, ); const listed = loaded.map((name) => { diff --git a/packages/harness/src/plugin/registry.ts b/packages/harness/src/plugin/registry.ts index 27d051e9..9510d4ee 100644 --- a/packages/harness/src/plugin/registry.ts +++ b/packages/harness/src/plugin/registry.ts @@ -1,4 +1,4 @@ -import type { Message } from "@codeworksh/aikit"; +import type { DeferredTool } from "../tool/offer.ts"; import type { PluginRegistry } from "@codeworksh/plugin/plugin"; import type { Effect } from "effect"; import type { McpServer } from "@codeworksh/plugin/settings"; @@ -34,7 +34,7 @@ export const make = (servers: ReadonlyArray) => { freeze: ( directory: string, selection?: Selection, - live?: Effect.Effect>, + live?: Effect.Effect>, mcpservers?: string, ) => { close(); diff --git a/packages/harness/src/tool/offer.ts b/packages/harness/src/tool/offer.ts index aead28db..6e67a938 100644 --- a/packages/harness/src/tool/offer.ts +++ b/packages/harness/src/tool/offer.ts @@ -1,12 +1,18 @@ import type { Message } from "@codeworksh/aikit"; import { Context, Effect } from "effect"; +/** A deferred tool as searched. */ +export interface DeferredTool extends Message.Tool { + /** Text of the namespace it belongs to, such as its MCP server's name, description and instructions. */ + readonly namespace?: string; +} + export interface Offer { /** * Deferred tools the model may call and is not offered yet: the registry's, in its order, then * any an MCP server offers since the turn started. Waits, bounded, for servers still connecting. */ - readonly deferred: Effect.Effect>; + readonly deferred: Effect.Effect>; } /** diff --git a/packages/harness/src/tool/registry.ts b/packages/harness/src/tool/registry.ts index e9ca3427..5e4ebb65 100644 --- a/packages/harness/src/tool/registry.ts +++ b/packages/harness/src/tool/registry.ts @@ -2,6 +2,7 @@ import type { ToolRegistration } from "../plugin/tool/schema.ts"; import type { Message } from "@codeworksh/aikit"; import { Context, Effect, Layer } from "effect"; import * as Executor from "./executor.ts"; +import type { DeferredTool } from "./offer.ts"; import { type AnyToolDef, declared, type RegisteredTool, toAikitTool } from "./tool.ts"; /** @@ -63,7 +64,7 @@ export interface Registry { * deferred tools registered after this catalog, as MCP servers connect: what a search finds * beside `deferred`, and the next snapshot registers. */ - readonly resolve: (selection?: Selection, live?: Effect.Effect>) => Resolved; + readonly resolve: (selection?: Selection, live?: Effect.Effect>) => Resolved; } /** @@ -90,7 +91,7 @@ export const make = (tools: ReadonlyArray): R const isDeferred = (entry: ToolRegistration) => entry.tool.definition.exposure === "deferred"; const catalog = Object.freeze(effective.filter(isDeferred).map((entry) => toAikitTool(entry.tool.definition))); - const resolve = (selection?: Selection, live?: Effect.Effect>): Resolved => { + const resolve = (selection?: Selection, live?: Effect.Effect>): Resolved => { const removed = new Set(selection?.remove); const declaredSet = effective.filter((entry) => declared(entry.tool.definition)); const kept = declaredSet.filter((entry) => !removed.has(entry.tool.definition.name)); diff --git a/packages/harness/src/tool/search.ts b/packages/harness/src/tool/search.ts index 80e2fc76..2d12fc40 100644 --- a/packages/harness/src/tool/search.ts +++ b/packages/harness/src/tool/search.ts @@ -5,9 +5,11 @@ import type { Message } from "@codeworksh/aikit"; /** The tool that loads deferred tools. */ export const name = "toolsearch"; +/** A tool as the ranker sees it. The name is its own field, so `text` leaves it out. */ export interface Document { readonly name: string; readonly text: string; + readonly namespaceText?: string | undefined; } const STOP_WORDS: ReadonlySet = new Set([ @@ -42,48 +44,117 @@ const stem = (term: string): string => { return term; }; -/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */ -export const tokenize = (text: string): string[] => - text +const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" }); + +/** Unicode words, split at camelCase boundaries, lowercase, without stop words; ASCII words in singular form. */ +export const tokenize = (text: string): string[] => { + const normalized = text + .normalize("NFC") .replace(/([a-z0-9])([A-Z])/g, "$1 $2") .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") .toLowerCase() - .split(/[^a-z0-9]+/) - .filter((term) => term.length > 0 && !STOP_WORDS.has(term)) - .map(stem); + .replace(/[^\p{L}\p{M}\p{N}]+/gu, " "); + // The segmenter costs about 15 times a split, and plain ASCII splits into the same words at spaces, so only + // other text, such as Chinese written without spaces, goes through it. + const words = /^[a-z0-9 ]*$/.test(normalized) + ? normalized.split(" ") + : Array.from(wordSegmenter.segment(normalized), (segment) => (segment.isWordLike ? segment.segment : "")); + return words + .filter((word) => word.length > 0 && !STOP_WORDS.has(word)) + .map((word) => (/^[a-z0-9]+$/.test(word) ? stem(word) : word)); +}; + +const K1 = 1.2; +const B = 0.75; + +/** + * A query term in the name counts this many times its IDF. The text part of a term is at most `K1 + 1` (2.2) times + * its IDF and about 1 for a single mention, so a name match beats any number of mentions in the text. + */ +const NAME_WEIGHT = 3; + +/** Every tool of a namespace shares its text, so it helps find the namespace but not the tool in it. */ +const NAMESPACE_WEIGHT = 0.5; + +/** Lowercase with characters invalid in identifiers as `_`, so `dev-radius` matches `dev_radius`. */ +const normalizeName = (name: string): string => + name + .trim() + .toLowerCase() + .replace(/[^a-z0-9_$]/g, "_"); + +/** The whole query is the tool's full name or its name after a `__`: `thread_read` names `mcp__app__thread_read`. */ +const isExactName = (query: string, name: string): boolean => { + const normalized = normalizeName(name); + return normalized === query || normalized.endsWith(`__${query}`); +}; -/** Okapi BM25 (k1 1.2, b 0.75): the names of the best matches, best first; ties keep document order. */ +const indexDocument = (document: Document) => { + const textCounts = new Map(); + for (const term of tokenize(document.text)) textCounts.set(term, (textCounts.get(term) ?? 0) + 1); + const segments = document.name.split("__"); + return { + name: document.name, + nameTerms: new Set(tokenize(document.name)), + // The part after the last `__` is what models say: `thread`, `read` of `mcp__app__thread_read`. + localNameTerms: [...new Set(tokenize(segments[segments.length - 1]!))], + textCounts, + textLength: [...textCounts.values()].reduce((sum, count) => sum + count, 0), + namespaceTerms: new Set(tokenize(document.namespaceText ?? "")), + }; +}; + +/** + * Okapi BM25 over `text`, with the name and the namespace as separate fields: the names of the best matches, best + * first; ties keep document order. + * - A query that is exactly a tool's name, or its name after a `__`, ranks that tool first. + * - Then a tool matching more of the query's terms, in any field, ranks above one matching fewer, whatever their + * scores: for `screenshot window`, a tool whose description has both words beats `window`, whose name has one. + * - A tool whose whole local name is in the query gets a bonus worth one more match of every query term, so + * `peekaboo app` finds `mcp__peekaboo__app` over tools that only mention the words. + * + * Only `text` is length-normalized, so a long name or namespace text does not lower a tool's score. + */ export const rank = (query: string, documents: ReadonlyArray, limit: number): string[] => { - const k1 = 1.2; - const b = 0.75; - const queryTerms = [...new Set(tokenize(query))]; - if (queryTerms.length === 0 || documents.length === 0 || limit <= 0) return []; - const termCounts = documents.map((document) => { - const counts = new Map(); - for (const term of tokenize(document.text)) counts.set(term, (counts.get(term) ?? 0) + 1); - return counts; - }); - const lengths = termCounts.map((counts) => [...counts.values()].reduce((sum, count) => sum + count, 0)); - const averageLength = lengths.reduce((sum, length) => sum + length, 0) / documents.length || 1; + const queryTerms = new Set(tokenize(query)); + if (queryTerms.size === 0 || documents.length === 0 || limit <= 0) return []; + const indexed = documents.map(indexDocument); + const averageLength = indexed.reduce((sum, document) => sum + document.textLength, 0) / indexed.length || 1; const idf = new Map( - queryTerms.map((term) => { - const frequency = termCounts.filter((counts) => counts.has(term)).length; - return [term, Math.log(1 + (documents.length - frequency + 0.5) / (frequency + 0.5))] as const; + [...queryTerms].map((term) => { + const frequency = indexed.filter( + (document) => + document.nameTerms.has(term) || document.textCounts.has(term) || document.namespaceTerms.has(term), + ).length; + return [term, Math.log(1 + (indexed.length - frequency + 0.5) / (frequency + 0.5))] as const; }), ); - const matches: { name: string; score: number }[] = []; - documents.forEach((document, index) => { + const localNameBonus = [...idf.values()].reduce((sum, value) => sum + value, 0); + const exactQuery = normalizeName(query); + const matches: { name: string; score: number; covered: number; exact: boolean }[] = []; + for (const document of indexed) { + const norm = K1 * (1 - B + (B * document.textLength) / averageLength); let score = 0; + let covered = 0; for (const term of queryTerms) { - const count = termCounts[index]!.get(term); - if (!count) continue; - const norm = k1 * (1 - b + (b * lengths[index]!) / averageLength); - score += (idf.get(term) ?? 0) * ((count * (k1 + 1)) / (count + norm)); + const count = document.textCounts.get(term) ?? 0; + const weight = + (document.nameTerms.has(term) ? NAME_WEIGHT : 0) + + (document.namespaceTerms.has(term) ? NAMESPACE_WEIGHT : 0) + + (count * (K1 + 1)) / (count + norm); + if (weight > 0) covered++; + score += (idf.get(term) ?? 0) * weight; } - if (score > 0) matches.push({ name: document.name, score }); - }); + if (score === 0) continue; + const local = document.localNameTerms; + if (local.length > 0 && local.every((term) => queryTerms.has(term))) score += localNameBonus; + matches.push({ name: document.name, score, covered, exact: isExactName(exactQuery, document.name) }); + } return matches - .sort((left, right) => right.score - left.score) + .sort( + (left, right) => + Number(right.exact) - Number(left.exact) || right.covered - left.covered || right.score - left.score, + ) .slice(0, limit) .map((match) => match.name); }; @@ -108,9 +179,9 @@ const schemaText = (schema: unknown, parts: string[]): void => { } }; -/** Search text of a tool: its name, the name with `_` as spaces, its description, and its parameters' text. */ +/** Search text of a tool: its description and its parameters' text. The ranker reads the name from `name`. */ export const text = (tool: Message.Tool): string => { - const parts = [tool.name, tool.name.replaceAll("_", " "), tool.description]; + const parts = [tool.description]; schemaText(tool.parameters, parts); return parts.filter((part) => part.trim() !== "").join(" "); }; diff --git a/packages/harness/test/__artifacts__/mcp.section.json b/packages/harness/test/__artifacts__/mcp.section.json index cb5ef350..ff4f4652 100644 --- a/packages/harness/test/__artifacts__/mcp.section.json +++ b/packages/harness/test/__artifacts__/mcp.section.json @@ -19,7 +19,14 @@ { "sections": [], "added": [ - "mcp__atlas__echo" + "mcp__atlas__echo", + "mcp__atlas__args", + "mcp__atlas__fail", + "mcp__atlas__pid", + "mcp__atlas__crash", + "mcp__atlas__my_tool_v2", + "mcp__atlas__secret", + "mcp__atlas__bye" ], "removed": [] }, @@ -36,7 +43,14 @@ ], "added": [], "removed": [ - "mcp__atlas__echo" + "mcp__atlas__echo", + "mcp__atlas__args", + "mcp__atlas__fail", + "mcp__atlas__pid", + "mcp__atlas__crash", + "mcp__atlas__my_tool_v2", + "mcp__atlas__secret", + "mcp__atlas__bye" ] } ], diff --git a/packages/harness/test/__artifacts__/mcp.toolsearch.namespace.json b/packages/harness/test/__artifacts__/mcp.toolsearch.namespace.json new file mode 100644 index 00000000..1e624d8c --- /dev/null +++ b/packages/harness/test/__artifacts__/mcp.toolsearch.namespace.json @@ -0,0 +1,12 @@ +{ + "calls": { + "described": "Loaded 1 tool. They are available from your next call:\n- mcp__forecast__echo: Echo text back, prefixed with the configured greeting.", + "instructed": "Loaded 1 tool. They are available from your next call:\n- mcp__billing__echo: Echo text back, prefixed with the configured greeting." + }, + "loaded": { + "add": [ + "mcp__forecast__echo", + "mcp__billing__echo" + ] + } +} diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.limit.json b/packages/harness/test/__artifacts__/tools.toolsearch.limit.json index f2773cd1..8df75790 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.limit.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.limit.json @@ -21,16 +21,16 @@ "query": "create send search list get query translate run weather issue message quote event" }, "status": "completed", - "text": "Loaded 8 tools. They are available from your next call:\n- jira_search_issues: Search Jira issues with a JQL query.\n- calendar_create_event: Create an event in the user's calendar.\n- stock_quote: Get the latest stock quote for a ticker symbol.\n- database_query: Run a read-only SQL query against the analytics database.\n- slack_send_message: Send a message to a Slack channel.\n- weather_forecast: Get the weather forecast for a city.\n- github_create_issue: Create an issue in a GitHub repository.\n- email_send: Send an email message to a recipient.", + "text": "Loaded 8 tools. They are available from your next call:\n- jira_search_issues: Search Jira issues with a JQL query.\n- calendar_create_event: Create an event in the user's calendar.\n- slack_send_message: Send a message to a Slack channel.\n- github_create_issue: Create an issue in a GitHub repository.\n- stock_quote: Get the latest stock quote for a ticker symbol.\n- weather_forecast: Get the weather forecast for a city.\n- database_query: Run a read-only SQL query against the analytics database.\n- email_send: Send an email message to a recipient.", "details": { "loaded": [ "jira_search_issues", "calendar_create_event", - "stock_quote", - "database_query", "slack_send_message", - "weather_forecast", "github_create_issue", + "stock_quote", + "weather_forecast", + "database_query", "email_send" ] } @@ -40,10 +40,10 @@ "github_create_issue", "jira_search_issues", "calendar_create_event", - "stock_quote", - "database_query", "slack_send_message", + "stock_quote", "weather_forecast", + "database_query", "email_send" ], "nextRequest": { @@ -71,10 +71,10 @@ "github_create_issue", "jira_search_issues", "calendar_create_event", - "stock_quote", - "database_query", "slack_send_message", + "stock_quote", "weather_forecast", + "database_query", "email_send" ], "removed": [] diff --git a/packages/harness/test/__artifacts__/toolsearch-ranking.json b/packages/harness/test/__artifacts__/toolsearch-ranking.json new file mode 100644 index 00000000..9e4f36a9 --- /dev/null +++ b/packages/harness/test/__artifacts__/toolsearch-ranking.json @@ -0,0 +1,128 @@ +[ + { + "query": "mcp__app__thread_read", + "limit": 3, + "loaded": [ + "mcp__app__thread_read", + "mcp__app__queue_read", + "mcp__app__thread_list" + ] + }, + { + "query": "thread_read", + "limit": 3, + "loaded": [ + "mcp__app__thread_read", + "mcp__app__queue_read", + "mcp__app__thread_list" + ] + }, + { + "query": "app__thread_read", + "limit": 3, + "loaded": [ + "mcp__app__thread_read", + "mcp__app__queue_read", + "mcp__app__thread_list" + ] + }, + { + "query": "Thread_Read", + "limit": 3, + "loaded": [ + "mcp__app__thread_read", + "mcp__app__queue_read", + "mcp__app__thread_list" + ] + }, + { + "query": "dev-radius__status", + "limit": 3, + "loaded": [ + "mcp__dev_radius__status", + "mcp__ci__status" + ] + }, + { + "query": "read", + "limit": 3, + "loaded": [ + "mcp__app__queue_read", + "mcp__app__thread_read" + ] + }, + { + "query": "deploy", + "limit": 3, + "loaded": [ + "mcp__ci__deploy_release", + "mcp__ci__status" + ] + }, + { + "query": "screenshot window", + "limit": 3, + "loaded": [ + "mcp__peekaboo__image", + "mcp__peekaboo__window", + "mcp__peekaboo__see" + ] + }, + { + "query": "describe app", + "limit": 3, + "loaded": [ + "mcp__peekaboo__see", + "mcp__peekaboo__app", + "mcp__app__queue_read" + ] + }, + { + "query": "peekaboo app", + "limit": 3, + "loaded": [ + "mcp__peekaboo__app", + "mcp__peekaboo__see", + "mcp__peekaboo__window" + ] + }, + { + "query": "airtable comment", + "limit": 3, + "loaded": [ + "mcp__airtable__create_comment", + "mcp__airtable__list_comments", + "mcp__airtable__list_records" + ] + }, + { + "query": "update records", + "limit": 3, + "loaded": [ + "mcp__airtable__update_records", + "mcp__airtable__list_records", + "mcp__airtable__create_record" + ] + }, + { + "query": "问题", + "limit": 3, + "loaded": [ + "mcp__docs__search_cn" + ] + }, + { + "query": "café", + "limit": 3, + "loaded": [ + "mcp__menu__order" + ] + }, + { + "query": "résumé", + "limit": 3, + "loaded": [ + "mcp__menu__order" + ] + } +] diff --git a/packages/harness/test/mcp.thread.test.ts b/packages/harness/test/mcp.thread.test.ts index d17e09d1..8f7e69b4 100644 --- a/packages/harness/test/mcp.thread.test.ts +++ b/packages/harness/test/mcp.thread.test.ts @@ -800,3 +800,43 @@ it("mcp: toolsearch stops waiting for a server that never answers, at the startu "./__artifacts__/mcp.toolsearch.bounded.json", ); }, 60_000); + +it("mcp: toolsearch matches a server's description and instructions, not only its tools", async () => { + await using dir = await tmpdir(); + const home = join(dir.path, "home"); + const work = join(dir.path, "work"); + await fs.mkdir(work, { recursive: true }); + // The same tools on every server, and the plain one first: only namespace text tells them apart. + await write(join(home, "mcp.json"), { + mcpServers: { + plain: stdio({ exposure: "deferred" }), + billing: stdio({ exposure: "deferred" }, { E2E_INSTRUCTIONS: "Handles invoices." }), + forecast: stdio({ exposure: "deferred", description: "Weather for any city." }), + }, + }); + + const model = scripted(); + const result = await Effect.gen(function* () { + const session = yield* Session.create({ directory: work, model: { provider: "openai", id: "gpt-5.6-luna" } }); + const main = session.main; + model.turn(main.id, [ + pendingCall(TOOL_SEARCH_NAME, { query: "weather echo", limit: 1 }, "described"), + pendingCall(TOOL_SEARCH_NAME, { query: "invoices echo", limit: 1 }, "instructed"), + ]); + yield* main.run("one"); + return { + calls: yield* results(main), + loaded: (yield* Document.Service.use((documents) => documents.agent(main.id))).modelTools, + }; + }).pipe( + Effect.provide(Harness.layer({ home, hostCwd: dir.path, database: ":memory:", llm: model.open })), + Effect.scoped, + Effect.runPromise, + ); + + expect(result.calls.described).toContain("mcp__forecast__echo"); + expect(result.calls.instructed).toContain("mcp__billing__echo"); + await expect(`${JSON.stringify(result, null, "\t")}\n`).toMatchFileSnapshot( + "./__artifacts__/mcp.toolsearch.namespace.json", + ); +}, 60_000); diff --git a/packages/harness/test/tools.toolsearch.test.ts b/packages/harness/test/tools.toolsearch.test.ts index 1ed32fcb..9cbca7cb 100644 --- a/packages/harness/test/tools.toolsearch.test.ts +++ b/packages/harness/test/tools.toolsearch.test.ts @@ -178,7 +178,7 @@ const catalog: Plugin = { }, }; -const layer = () => { +const layer = (tools: Plugin) => { const database = Database.layer(":memory:"); const sandbox = SandboxController.layer({ hostCwd: "/" }).pipe( Layer.provideMerge(SandboxDriverRegistry.layer()), @@ -188,7 +188,7 @@ const layer = () => { const seeded = pooled([ ...builtins.filter((plugin) => plugin.id !== toolSearchPlugin.id), { ...toolSearchPlugin, optIn: false }, - catalog, + tools, ]); return Control.layer.pipe( Layer.provideMerge( @@ -211,7 +211,7 @@ const layer = () => { ); }; -const { effect: it } = testEffect(layer()); +const { effect: it } = testEffect(layer(catalog)); const reset = Effect.sync(() => { script.clear(); @@ -558,3 +558,113 @@ describe("toolsearch", () => { }), ); }); + +/** Built-in-style tools and MCP-style `mcp____` tools, for ranking alone. */ +const rankingCatalog: Plugin = { + id: "test.tool.ranking", + kind: "tool", + setup: (ctx) => { + const tools = [ + deferred("web_fetch", "Fetch a URL and return the page as markdown.", { url: text("URL to fetch") }), + deferred("notebook_edit", "Edit a cell of a Jupyter notebook.", { + path: text("Notebook path"), + cell: Schema.Finite.annotate({ description: "Cell index" }), + }), + deferred("mcp__ci__status", "Deploy status: last deploy, deploy log, deploy errors.", {}), + deferred( + "mcp__ci__deploy_release", + "Start a release of the current branch to an environment, run the migrations, switch traffic over, and wait until the new version is live and healthy.", + { environment: text("Target environment") }, + ), + deferred("mcp__app__queue_read", "Read a queued message of this thread.", {}), + deferred( + "mcp__app__thread_read", + "Read durable state and a paginated timeline from any thread. The default view returns user messages, assistant messages and plans.", + { threadId: text("Thread to read") }, + ), + deferred("mcp__app__thread_list", "List the threads of the current project, newest first.", {}), + deferred("mcp__dev_radius__status", "Report the health of the local development environment.", {}), + deferred("mcp__peekaboo__app", "Launch, quit, focus, or switch applications by name or bundle identifier.", { + name: text("Application name"), + }), + deferred("mcp__peekaboo__see", "Capture the app window and describe the UI elements of the app.", {}), + deferred("mcp__peekaboo__window", "Move, resize, or close a window of an app.", {}), + deferred("mcp__peekaboo__image", "Take a screenshot of the screen, an app, or a window.", { + path: text("Where to save the image"), + }), + deferred("mcp__airtable__list_records", "List records of an Airtable table.", { + baseId: text("Base ID"), + tableId: text("Table ID"), + }), + deferred("mcp__airtable__create_record", "Create a record in an Airtable table.", { + tableId: text("Table ID"), + fields: text("Field values as JSON"), + }), + deferred("mcp__airtable__update_records", "Update fields of one or more records in a table.", { + tableId: text("Table ID"), + }), + deferred("mcp__airtable__list_comments", "List the comments on a record.", { recordId: text("Record ID") }), + deferred("mcp__airtable__create_comment", "Add a comment to a record.", { + recordId: text("Record ID"), + text: text("Comment text"), + }), + deferred("mcp__docs__search_cn", "搜索仓库问题和文档。", { query: text("搜索关键词") }), + deferred("mcp__menu__order", "Order a cafe\u0301 au lait or a résumé review.", {}), + ]; + for (const tool of tools) ctx.plugin.tools.add(tool); + }, +}; + +describe("toolsearch ranking", () => { + const { effect: rankIt } = testEffect(layer(rankingCatalog)); + + rankIt( + "ranks exact names, name words, coverage, local names, CJK and canonically equivalent text", + Effect.gen(function* () { + yield* reset; + const { sessionId, main } = yield* createSession; + const queries: ReadonlyArray<{ readonly query: string; readonly limit?: number }> = [ + { query: "mcp__app__thread_read", limit: 3 }, + { query: "thread_read", limit: 3 }, + { query: "app__thread_read", limit: 3 }, + { query: "Thread_Read", limit: 3 }, + { query: "dev-radius__status", limit: 3 }, + { query: "read", limit: 3 }, + { query: "deploy", limit: 3 }, + { query: "screenshot window", limit: 3 }, + { query: "describe app", limit: 3 }, + { query: "peekaboo app", limit: 3 }, + { query: "airtable comment", limit: 3 }, + { query: "update records", limit: 3 }, + { query: "问题", limit: 3 }, + { query: "caf\u00e9", limit: 3 }, + { query: "re\u0301sume\u0301", limit: 3 }, + ]; + // One step, so every search runs against the same candidates: the whole catalog. + yield* run(sessionId, main, "Find me tools", [ + call(...queries.map((one) => search(one.query, one.limit))), + reply, + ]); + const outcomes = yield* results(main); + const ranked = queries.map((one, index) => ({ + ...one, + loaded: (outcomes[index]!.details as { readonly loaded: ReadonlyArray }).loaded, + })); + const first = (query: string) => ranked.find((one) => one.query === query)!.loaded[0]; + + for (const query of ["mcp__app__thread_read", "thread_read", "app__thread_read", "Thread_Read"]) + expect(first(query)).toBe("mcp__app__thread_read"); + expect(first("dev-radius__status")).toBe("mcp__dev_radius__status"); + expect(first("read")).toBe("mcp__app__queue_read"); + expect(first("deploy")).toBe("mcp__ci__deploy_release"); + expect(first("screenshot window")).toBe("mcp__peekaboo__image"); + expect(first("describe app")).toBe("mcp__peekaboo__see"); + expect(first("peekaboo app")).toBe("mcp__peekaboo__app"); + expect(first("问题")).toBe("mcp__docs__search_cn"); + expect(first("caf\u00e9")).toBe("mcp__menu__order"); + expect(first("re\u0301sume\u0301")).toBe("mcp__menu__order"); + + yield* artifact("toolsearch-ranking.json", ranked); + }), + ); +}); diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 7bb34b49..792ccda3 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -40,5 +40,5 @@ overrides: storeDir: .pnpm-store permissions: - '@tanstack/router-plugin': - skills: true + "@tanstack/router-plugin": + skills: true