Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions packages/harness/src/mcp/mcp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
* service down, stops the server process and ends the HTTP session.
*/

import type { Message } from "@codeworksh/aikit";
import type { Exposure, ServerConfig } from "@codeworksh/plugin/plugin/mcp";
import type { AuthProvider, Tool } from "@modelcontextprotocol/client";
import {
Expand Down Expand Up @@ -63,6 +62,7 @@ import { Event } from "../event/event.ts";
import { EventList } from "../event/list.ts";
import { ID as SessionId } from "../session/schema.ts";
import { ID as ThreadId } from "../thread/schema.ts";
import type { DeferredTool } from "../tool/offer.ts";
import { type RegisteredTool, toAikitTool } from "../tool/tool.ts";
import { McpClient } from "./client.ts";
import { namespace } from "./config.ts";
Expand Down Expand Up @@ -135,7 +135,7 @@ export interface Reconciled {
* The thread's deferred tools as they stand when it runs, under the names they are registered by.
* Waits first, within {@link STARTUP_WAIT}, for those servers' connect attempts in progress.
*/
readonly deferred: Effect.Effect<ReadonlyArray<Message.Tool>>;
readonly deferred: Effect.Effect<ReadonlyArray<DeferredTool>>;
/** The `mcpservers` prompt section, as the servers stand once the ones with declared tools were waited for. */
readonly section: string | undefined;
}
Expand Down Expand Up @@ -925,9 +925,23 @@ export const layer = (options: {
yield* Effect.forEach(servers, attempted, { concurrency: "unbounded", discard: true }).pipe(
Effect.timeoutOption(STARTUP_WAIT),
);
return toolsOf(thread).flatMap((tool) =>
tool.definition.exposure === "deferred" ? [toAikitTool(tool.definition)] : [],
// A tool is searched by its server's namespace text too: its name, description and instructions.
const namespaces = new Map(
[...thread.servers].flatMap(([name, { config, server }]) => {
if (server === undefined) return [];
const text = [namespace(name), config.description ?? "", server.instructions ?? ""].join(" ");
const names = thread.names(
name,
server.tools.map((tool) => tool.name),
);
return [...names.values()].map((registered) => [registered, text] as const);
}),
);
return toolsOf(thread).flatMap((tool): ReadonlyArray<DeferredTool> => {
if (tool.definition.exposure !== "deferred") return [];
const text = namespaces.get(tool.definition.name);
return [{ ...toAikitTool(tool.definition), ...(text === undefined ? {} : { namespace: text }) }];
});
});

const reconcile = Effect.fn("Mcp.reconcile")(function* (input: Input) {
Expand Down
2 changes: 1 addition & 1 deletion packages/harness/src/plugin/builtin/tool/toolsearch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ export const toolSearchHandler: Tool.Handler<
const deferred = yield* (yield* ToolOffer).deferred;
const loaded = ToolSearch.rank(
query,
deferred.map((tool) => ({ name: tool.name, text: ToolSearch.text(tool) })),
deferred.map((tool) => ({ name: tool.name, text: ToolSearch.text(tool), namespaceText: tool.namespace })),
max,
);
const listed = loaded.map((name) => {
Expand Down
4 changes: 2 additions & 2 deletions packages/harness/src/plugin/registry.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { Message } from "@codeworksh/aikit";
import type { DeferredTool } from "../tool/offer.ts";
import type { PluginRegistry } from "@codeworksh/plugin/plugin";
import type { Effect } from "effect";
import type { McpServer } from "@codeworksh/plugin/settings";
Expand Down Expand Up @@ -34,7 +34,7 @@ export const make = (servers: ReadonlyArray<McpServer>) => {
freeze: (
directory: string,
selection?: Selection,
live?: Effect.Effect<ReadonlyArray<Message.Tool>>,
live?: Effect.Effect<ReadonlyArray<DeferredTool>>,
mcpservers?: string,
) => {
close();
Expand Down
8 changes: 7 additions & 1 deletion packages/harness/src/tool/offer.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
import type { Message } from "@codeworksh/aikit";
import { Context, Effect } from "effect";

/** A deferred tool as searched. */
export interface DeferredTool extends Message.Tool {
/** Text of the namespace it belongs to, such as its MCP server's name, description and instructions. */
readonly namespace?: string;
}

export interface Offer {
/**
* Deferred tools the model may call and is not offered yet: the registry's, in its order, then
* any an MCP server offers since the turn started. Waits, bounded, for servers still connecting.
*/
readonly deferred: Effect.Effect<ReadonlyArray<Message.Tool>>;
readonly deferred: Effect.Effect<ReadonlyArray<DeferredTool>>;
}

/**
Expand Down
5 changes: 3 additions & 2 deletions packages/harness/src/tool/registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import type { ToolRegistration } from "../plugin/tool/schema.ts";
import type { Message } from "@codeworksh/aikit";
import { Context, Effect, Layer } from "effect";
import * as Executor from "./executor.ts";
import type { DeferredTool } from "./offer.ts";
import { type AnyToolDef, declared, type RegisteredTool, toAikitTool } from "./tool.ts";

/**
Expand Down Expand Up @@ -63,7 +64,7 @@ export interface Registry {
* deferred tools registered after this catalog, as MCP servers connect: what a search finds
* beside `deferred`, and the next snapshot registers.
*/
readonly resolve: (selection?: Selection, live?: Effect.Effect<ReadonlyArray<Message.Tool>>) => Resolved;
readonly resolve: (selection?: Selection, live?: Effect.Effect<ReadonlyArray<DeferredTool>>) => Resolved;
}

/**
Expand All @@ -90,7 +91,7 @@ export const make = (tools: ReadonlyArray<RegisteredTool | ToolRegistration>): R
const isDeferred = (entry: ToolRegistration) => entry.tool.definition.exposure === "deferred";
const catalog = Object.freeze(effective.filter(isDeferred).map((entry) => toAikitTool(entry.tool.definition)));

const resolve = (selection?: Selection, live?: Effect.Effect<ReadonlyArray<Message.Tool>>): Resolved => {
const resolve = (selection?: Selection, live?: Effect.Effect<ReadonlyArray<DeferredTool>>): Resolved => {
const removed = new Set(selection?.remove);
const declaredSet = effective.filter((entry) => declared(entry.tool.definition));
const kept = declaredSet.filter((entry) => !removed.has(entry.tool.definition.name));
Expand Down
135 changes: 103 additions & 32 deletions packages/harness/src/tool/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@ import type { Message } from "@codeworksh/aikit";
/** The tool that loads deferred tools. */
export const name = "toolsearch";

/** A tool as the ranker sees it. The name is its own field, so `text` leaves it out. */
export interface Document {
readonly name: string;
readonly text: string;
readonly namespaceText?: string | undefined;
}

const STOP_WORDS: ReadonlySet<string> = new Set([
Expand Down Expand Up @@ -42,48 +44,117 @@ const stem = (term: string): string => {
return term;
};

/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */
export const tokenize = (text: string): string[] =>
text
const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" });

/** Unicode words, split at camelCase boundaries, lowercase, without stop words; ASCII words in singular form. */
export const tokenize = (text: string): string[] => {
const normalized = text
.normalize("NFC")
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
.toLowerCase()
.split(/[^a-z0-9]+/)
.filter((term) => term.length > 0 && !STOP_WORDS.has(term))
.map(stem);
.replace(/[^\p{L}\p{M}\p{N}]+/gu, " ");
// The segmenter costs about 15 times a split, and plain ASCII splits into the same words at spaces, so only
// other text, such as Chinese written without spaces, goes through it.
const words = /^[a-z0-9 ]*$/.test(normalized)
? normalized.split(" ")
: Array.from(wordSegmenter.segment(normalized), (segment) => (segment.isWordLike ? segment.segment : ""));
return words
.filter((word) => word.length > 0 && !STOP_WORDS.has(word))
.map((word) => (/^[a-z0-9]+$/.test(word) ? stem(word) : word));
};

const K1 = 1.2;
const B = 0.75;

/**
* A query term in the name counts this many times its IDF. The text part of a term is at most `K1 + 1` (2.2) times
* its IDF and about 1 for a single mention, so a name match beats any number of mentions in the text.
*/
const NAME_WEIGHT = 3;

/** Every tool of a namespace shares its text, so it helps find the namespace but not the tool in it. */
const NAMESPACE_WEIGHT = 0.5;

/** Lowercase with characters invalid in identifiers as `_`, so `dev-radius` matches `dev_radius`. */
const normalizeName = (name: string): string =>
name
.trim()
.toLowerCase()
.replace(/[^a-z0-9_$]/g, "_");

/** The whole query is the tool's full name or its name after a `__`: `thread_read` names `mcp__app__thread_read`. */
const isExactName = (query: string, name: string): boolean => {
const normalized = normalizeName(name);
return normalized === query || normalized.endsWith(`__${query}`);
};

/** Okapi BM25 (k1 1.2, b 0.75): the names of the best matches, best first; ties keep document order. */
const indexDocument = (document: Document) => {
const textCounts = new Map<string, number>();
for (const term of tokenize(document.text)) textCounts.set(term, (textCounts.get(term) ?? 0) + 1);
const segments = document.name.split("__");
return {
name: document.name,
nameTerms: new Set(tokenize(document.name)),
// The part after the last `__` is what models say: `thread`, `read` of `mcp__app__thread_read`.
localNameTerms: [...new Set(tokenize(segments[segments.length - 1]!))],
textCounts,
textLength: [...textCounts.values()].reduce((sum, count) => sum + count, 0),
namespaceTerms: new Set(tokenize(document.namespaceText ?? "")),
};
};

/**
* Okapi BM25 over `text`, with the name and the namespace as separate fields: the names of the best matches, best
* first; ties keep document order.
* - A query that is exactly a tool's name, or its name after a `__`, ranks that tool first.
* - Then a tool matching more of the query's terms, in any field, ranks above one matching fewer, whatever their
* scores: for `screenshot window`, a tool whose description has both words beats `window`, whose name has one.
* - A tool whose whole local name is in the query gets a bonus worth one more match of every query term, so
* `peekaboo app` finds `mcp__peekaboo__app` over tools that only mention the words.
*
* Only `text` is length-normalized, so a long name or namespace text does not lower a tool's score.
*/
export const rank = (query: string, documents: ReadonlyArray<Document>, limit: number): string[] => {
const k1 = 1.2;
const b = 0.75;
const queryTerms = [...new Set(tokenize(query))];
if (queryTerms.length === 0 || documents.length === 0 || limit <= 0) return [];
const termCounts = documents.map((document) => {
const counts = new Map<string, number>();
for (const term of tokenize(document.text)) counts.set(term, (counts.get(term) ?? 0) + 1);
return counts;
});
const lengths = termCounts.map((counts) => [...counts.values()].reduce((sum, count) => sum + count, 0));
const averageLength = lengths.reduce((sum, length) => sum + length, 0) / documents.length || 1;
const queryTerms = new Set(tokenize(query));
if (queryTerms.size === 0 || documents.length === 0 || limit <= 0) return [];
const indexed = documents.map(indexDocument);
const averageLength = indexed.reduce((sum, document) => sum + document.textLength, 0) / indexed.length || 1;
const idf = new Map(
queryTerms.map((term) => {
const frequency = termCounts.filter((counts) => counts.has(term)).length;
return [term, Math.log(1 + (documents.length - frequency + 0.5) / (frequency + 0.5))] as const;
[...queryTerms].map((term) => {
const frequency = indexed.filter(
(document) =>
document.nameTerms.has(term) || document.textCounts.has(term) || document.namespaceTerms.has(term),
).length;
return [term, Math.log(1 + (indexed.length - frequency + 0.5) / (frequency + 0.5))] as const;
}),
);
const matches: { name: string; score: number }[] = [];
documents.forEach((document, index) => {
const localNameBonus = [...idf.values()].reduce((sum, value) => sum + value, 0);
const exactQuery = normalizeName(query);
const matches: { name: string; score: number; covered: number; exact: boolean }[] = [];
for (const document of indexed) {
const norm = K1 * (1 - B + (B * document.textLength) / averageLength);
let score = 0;
let covered = 0;
for (const term of queryTerms) {
const count = termCounts[index]!.get(term);
if (!count) continue;
const norm = k1 * (1 - b + (b * lengths[index]!) / averageLength);
score += (idf.get(term) ?? 0) * ((count * (k1 + 1)) / (count + norm));
const count = document.textCounts.get(term) ?? 0;
const weight =
(document.nameTerms.has(term) ? NAME_WEIGHT : 0) +
(document.namespaceTerms.has(term) ? NAMESPACE_WEIGHT : 0) +
(count * (K1 + 1)) / (count + norm);
if (weight > 0) covered++;
score += (idf.get(term) ?? 0) * weight;
}
if (score > 0) matches.push({ name: document.name, score });
});
if (score === 0) continue;
const local = document.localNameTerms;
if (local.length > 0 && local.every((term) => queryTerms.has(term))) score += localNameBonus;
matches.push({ name: document.name, score, covered, exact: isExactName(exactQuery, document.name) });
}
return matches
.sort((left, right) => right.score - left.score)
.sort(
(left, right) =>
Number(right.exact) - Number(left.exact) || right.covered - left.covered || right.score - left.score,
)
.slice(0, limit)
.map((match) => match.name);
};
Expand All @@ -108,9 +179,9 @@ const schemaText = (schema: unknown, parts: string[]): void => {
}
};

/** Search text of a tool: its name, the name with `_` as spaces, its description, and its parameters' text. */
/** Search text of a tool: its description and its parameters' text. The ranker reads the name from `name`. */
export const text = (tool: Message.Tool): string => {
const parts = [tool.name, tool.name.replaceAll("_", " "), tool.description];
const parts = [tool.description];
schemaText(tool.parameters, parts);
return parts.filter((part) => part.trim() !== "").join(" ");
};
Expand Down
18 changes: 16 additions & 2 deletions packages/harness/test/__artifacts__/mcp.section.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,14 @@
{
"sections": [],
"added": [
"mcp__atlas__echo"
"mcp__atlas__echo",
"mcp__atlas__args",
"mcp__atlas__fail",
"mcp__atlas__pid",
"mcp__atlas__crash",
"mcp__atlas__my_tool_v2",
"mcp__atlas__secret",
"mcp__atlas__bye"
],
"removed": []
},
Expand All @@ -36,7 +43,14 @@
],
"added": [],
"removed": [
"mcp__atlas__echo"
"mcp__atlas__echo",
"mcp__atlas__args",
"mcp__atlas__fail",
"mcp__atlas__pid",
"mcp__atlas__crash",
"mcp__atlas__my_tool_v2",
"mcp__atlas__secret",
"mcp__atlas__bye"
]
}
],
Expand Down
12 changes: 12 additions & 0 deletions packages/harness/test/__artifacts__/mcp.toolsearch.namespace.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"calls": {
"described": "Loaded 1 tool. They are available from your next call:\n- mcp__forecast__echo: Echo text back, prefixed with the configured greeting.",
"instructed": "Loaded 1 tool. They are available from your next call:\n- mcp__billing__echo: Echo text back, prefixed with the configured greeting."
},
"loaded": {
"add": [
"mcp__forecast__echo",
"mcp__billing__echo"
]
}
}
16 changes: 8 additions & 8 deletions packages/harness/test/__artifacts__/tools.toolsearch.limit.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,16 @@
"query": "create send search list get query translate run weather issue message quote event"
},
"status": "completed",
"text": "Loaded 8 tools. They are available from your next call:\n- jira_search_issues: Search Jira issues with a JQL query.\n- calendar_create_event: Create an event in the user's calendar.\n- stock_quote: Get the latest stock quote for a ticker symbol.\n- database_query: Run a read-only SQL query against the analytics database.\n- slack_send_message: Send a message to a Slack channel.\n- weather_forecast: Get the weather forecast for a city.\n- github_create_issue: Create an issue in a GitHub repository.\n- email_send: Send an email message to a recipient.",
"text": "Loaded 8 tools. They are available from your next call:\n- jira_search_issues: Search Jira issues with a JQL query.\n- calendar_create_event: Create an event in the user's calendar.\n- slack_send_message: Send a message to a Slack channel.\n- github_create_issue: Create an issue in a GitHub repository.\n- stock_quote: Get the latest stock quote for a ticker symbol.\n- weather_forecast: Get the weather forecast for a city.\n- database_query: Run a read-only SQL query against the analytics database.\n- email_send: Send an email message to a recipient.",
"details": {
"loaded": [
"jira_search_issues",
"calendar_create_event",
"stock_quote",
"database_query",
"slack_send_message",
"weather_forecast",
"github_create_issue",
"stock_quote",
"weather_forecast",
"database_query",
"email_send"
]
}
Expand All @@ -40,10 +40,10 @@
"github_create_issue",
"jira_search_issues",
"calendar_create_event",
"stock_quote",
"database_query",
"slack_send_message",
"stock_quote",
"weather_forecast",
"database_query",
"email_send"
],
"nextRequest": {
Expand Down Expand Up @@ -71,10 +71,10 @@
"github_create_issue",
"jira_search_issues",
"calendar_create_event",
"stock_quote",
"database_query",
"slack_send_message",
"stock_quote",
"weather_forecast",
"database_query",
"email_send"
],
"removed": []
Expand Down
Loading
Loading