From bbf6a0029d32faf3588de8935ad8f3232d157025 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sun, 1 Nov 2026 16:17:28 +0530 Subject: [PATCH 1/3] test(codework): revoke OAuth tokens only after the secure server's listing The fixture's afterList trigger also fired on the keyed server's listing, which usually connects first, so tokens were revoked before the secure server connected and its tool was never offered. Co-Authored-By: Claude Opus 5.5 --- packages/codework/test/fixtures/mcp-oauth.mjs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/codework/test/fixtures/mcp-oauth.mjs b/packages/codework/test/fixtures/mcp-oauth.mjs index 4d42e1a8..68e46dd5 100644 --- a/packages/codework/test/fixtures/mcp-oauth.mjs +++ b/packages/codework/test/fixtures/mcp-oauth.mjs @@ -44,8 +44,9 @@ export const start = async (options = {}) => { client: undefined, pending: new Map(), /** - * What happens right after the next `tools/list`: `expire` the access tokens (a refresh still - * works), or `revoke` them and the refresh token (it does not). + * What happens right after the next `tools/list` on `/mcp`: `expire` the access tokens (a refresh + * still works), or `revoke` them and the refresh token (it does not). A listing on `/keyed` leaves + * it alone, or a run's keyed server, connecting first, would revoke before `/mcp` connects. */ afterList: undefined, /** Every bearer token presented to `/mcp` or `/keyed`, accepted or not. */ @@ -123,7 +124,11 @@ export const start = async (options = {}) => { ? { content: [{ type: "text", text: `called with ${token}` }] } : {}; if (body.method === "tools/call") state.log.push({ mcp: "tools/call", token }); - if (body.method === "tools/list" && state.afterList !== undefined) { + if ( + body.method === "tools/list" && + new URL(request.url, base).pathname === "/mcp" && + state.afterList !== undefined + ) { state.access.clear(); if (state.afterList === "revoke") state.refresh = undefined; state.log.push({ [state.afterList]: true }); From fdcae19bf2b74503f0e3241ec3ad3210b1e358ac Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sun, 1 Nov 2026 16:17:28 +0530 Subject: [PATCH 2/3] feat(codework): warn and exit 1 when a run's final response is truncated Co-Authored-By: Claude Opus 5.5 --- packages/codework/src/cli/cmd/handlers/run.ts | 13 ++++++++++- .../test/__artifacts__/cli.truncated.json | 5 ++++ packages/codework/test/cli.test.ts | 23 ++++++++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 packages/codework/test/__artifacts__/cli.truncated.json diff --git a/packages/codework/src/cli/cmd/handlers/run.ts b/packages/codework/src/cli/cmd/handlers/run.ts index 09126a1d..ce5ecd6b 100644 --- a/packages/codework/src/cli/cmd/handlers/run.ts +++ b/packages/codework/src/cli/cmd/handlers/run.ts @@ -21,6 +21,7 @@ interface RenderState { readonly usage: UsageSummary; readonly textSeen: boolean; readonly textEndsWithNewline: boolean; + readonly reason?: EventList.LLMEnded["data"]["reason"]; } const initialRenderState: RenderState = { @@ -47,7 +48,11 @@ const render = (state: Ref.Ref, ended?: Queue.Queue) => return; } if (isLLMEnded(event)) { - yield* Ref.update(state, (current) => ({ ...current, usage: addUsage(current.usage, event.data.message) })); + yield* Ref.update(state, (current) => ({ + ...current, + usage: addUsage(current.usage, event.data.message), + reason: event.data.reason, + })); return; } if (ended !== undefined && isTurnEnded(event)) { @@ -59,6 +64,12 @@ const finish = Effect.fn("CLI.finish")(function* (state: Ref.Ref, c const rendered = yield* Ref.get(state); if (rendered.textSeen && !rendered.textEndsWithNewline) yield* writeOut("\n"); yield* writeError(usage(rendered.usage, columns)); + if (rendered.reason === "length") { + yield* writeError("Response was truncated: the model reached its output token limit.\n"); + yield* Effect.sync(() => { + process.exitCode = 1; + }); + } }); const awaitMessage = Effect.fn("CLI.awaitMessage")(function* (ended: Queue.Queue, messageId: string) { diff --git a/packages/codework/test/__artifacts__/cli.truncated.json b/packages/codework/test/__artifacts__/cli.truncated.json new file mode 100644 index 00000000..261e31ae --- /dev/null +++ b/packages/codework/test/__artifacts__/cli.truncated.json @@ -0,0 +1,5 @@ +{ + "status": 1, + "stdout": "The answer starts here and\n", + "warning": "Response was truncated: the model reached its output token limit.\n" +} diff --git a/packages/codework/test/cli.test.ts b/packages/codework/test/cli.test.ts index 4bd2c10b..5d2f3bd3 100644 --- a/packages/codework/test/cli.test.ts +++ b/packages/codework/test/cli.test.ts @@ -5,7 +5,8 @@ import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import { describe, expect, it } from "vite-plus/test"; +import { describe, expect, it, onTestFinished } from "vite-plus/test"; +import { runCli, truncating, writeSettings } from "./fixtures/cli.ts"; const cli = fileURLToPath(new URL("../src/index.ts", import.meta.url)); const models = fileURLToPath(new URL("../../../models.gen.json", import.meta.url)); @@ -459,6 +460,26 @@ describe("codework CLI", () => { } }); + it("prints a response cut off at the output token limit, warns, and exits 1", async () => { + const home = mkdtempSync(join(tmpdir(), "codework-cli-truncated-")); + onTestFinished(() => rmSync(home, { recursive: true, force: true })); + const fake = await truncating("The answer starts here and"); + writeSettings(home, fake.url); + + const result = await runCli( + ["--home", home, "--database", ":memory:", "--user-config-dir", home, "run", "Answer at length."], + { cwd: home, env: { ...process.env, HOME: home, CODEWORK_MODELS_FILE: models } }, + ); + + expect(result.status, result.stderr).toBe(1); + expect(result.stdout, result.stderr).toBe("The answer starts here and\n"); + const warning = "Response was truncated: the model reached its output token limit.\n"; + expect(result.stderr.endsWith(warning), result.stderr).toBe(true); + await expect( + `${JSON.stringify({ status: result.status, stdout: result.stdout, warning }, null, "\t")}\n`, + ).toMatchFileSnapshot("./__artifacts__/cli.truncated.json"); + }, 60_000); + it("reports a failed refresh", () => { const result = runIsolated( { CODEWORK_MODELS_FILE: undefined, CODEWORK_CATALOG_URL: "http://127.0.0.1:1/v1/models.json" }, From bc3860e187378e653c83c1214c44eec0107b1b14 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sun, 1 Nov 2026 16:17:29 +0530 Subject: [PATCH 3/3] feat(harness): keep partial replies and end crash loops visibly - Aborted, failed and crash-healed replies keep their streamed parts; unfinished tool calls in them settle aborted and never run. - Streaming replies are checkpointed durably (session.llm.progress) at most once a second, so a crash keeps the partial. - A reply cut off at the output limit gives its tool calls a re-issue error and the run continues; two in a row end it as truncated. - The in-flight attempt is persisted: a request cut off by a host death counts as a failed retry attempt, and with none left the run ends as interrupted. - A crash while tools ran commits the turn with the unfinished tools aborted; that commit does not reset the restart count. - At the restart limit the run ends with session.run.failed crashloop instead of parking the thread. - Kill/reopen E2E against the real server process. Co-Authored-By: Claude Opus 5.5 --- packages/codework/src/server/contract.ts | 2 - packages/codework/src/server/handlers.ts | 1 - .../__artifacts__/recovery.crashloop.json | 191 +++++++ .../__artifacts__/recovery.interrupted.json | 89 ++++ .../__artifacts__/recovery.mid-stream.json | 110 ++++ .../test/__artifacts__/recovery.tool.json | 115 +++++ .../__artifacts__/recovery.toolcrash.json | 344 ++++++++++++ .../test/__artifacts__/recovery.tools.json | 136 +++++ .../__artifacts__/recovery.truncated.json | 130 +++++ packages/codework/test/fixtures/cli.ts | 138 +++-- packages/codework/test/recovery.test.ts | 488 ++++++++++++++++++ packages/harness/src/control.ts | 20 +- packages/harness/src/db/migrations.ts | 1 + packages/harness/src/db/schema.sql.ts | 3 + packages/harness/src/effect/session.ts | 9 - packages/harness/src/event/list.ts | 52 +- packages/harness/src/runner/event.ts | 136 ++++- packages/harness/src/runner/execute.ts | 14 +- packages/harness/src/runner/execution.ts | 6 + packages/harness/src/runner/loop.ts | 324 ++++++++---- packages/harness/src/runner/part.ts | 36 ++ packages/harness/src/runner/run.ts | 5 + packages/harness/src/session/projector.ts | 23 +- packages/harness/src/thread/thread.ts | 113 +++- .../test/__artifacts__/runner.retry.json | 2 +- packages/harness/test/control.prompt.test.ts | 1 + packages/harness/test/plugin.foreign.test.ts | 7 +- packages/harness/test/runner.llm.live.test.ts | 4 +- packages/harness/test/runner.loop.test.ts | 35 +- packages/plugin/src/event.ts | 30 +- packages/plugin/src/schema.ts | 6 + 31 files changed, 2378 insertions(+), 193 deletions(-) create mode 100644 packages/codework/test/__artifacts__/recovery.crashloop.json create mode 100644 packages/codework/test/__artifacts__/recovery.interrupted.json create mode 100644 packages/codework/test/__artifacts__/recovery.mid-stream.json create mode 100644 packages/codework/test/__artifacts__/recovery.tool.json create mode 100644 packages/codework/test/__artifacts__/recovery.toolcrash.json create mode 100644 packages/codework/test/__artifacts__/recovery.tools.json create mode 100644 packages/codework/test/__artifacts__/recovery.truncated.json create mode 100644 packages/codework/test/recovery.test.ts create mode 100644 packages/harness/src/runner/part.ts diff --git a/packages/codework/src/server/contract.ts b/packages/codework/src/server/contract.ts index d2a2295e..d672f2f0 100644 --- a/packages/codework/src/server/contract.ts +++ b/packages/codework/src/server/contract.ts @@ -93,8 +93,6 @@ export const ThreadInfo = Schema.Struct({ foundation: optional(Schema.String), /** Where it runs instead of the session's space and directory. */ placement: optional(Schema.Struct({ spaceId: Schema.String, directory: Session.AbsolutePath })), - /** Its unfinished run was cut off by restarts too often to resume on its own; the next prompt runs it. */ - parked: optional(Schema.Boolean), usage: ThreadUsage, session: SessionInfo, }); diff --git a/packages/codework/src/server/handlers.ts b/packages/codework/src/server/handlers.ts index 0a6e17c2..6d18fef3 100644 --- a/packages/codework/src/server/handlers.ts +++ b/packages/codework/src/server/handlers.ts @@ -65,7 +65,6 @@ const toThreadInfo = (info: Session.ThreadInfo): ThreadInfo => ({ ...(info.thinkingLevel === undefined ? {} : { thinkingLevel: info.thinkingLevel }), ...(info.foundation === undefined ? {} : { foundation: info.foundation }), ...(info.placement === undefined ? {} : { placement: info.placement }), - ...(info.parked ? { parked: true } : {}), usage: info.usage, session: toSessionInfo(info.session), }); diff --git a/packages/codework/test/__artifacts__/recovery.crashloop.json b/packages/codework/test/__artifacts__/recovery.crashloop.json new file mode 100644 index 00000000..1b909d09 --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.crashloop.json @@ -0,0 +1,191 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Keep crashing." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Keep crashing." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 1", + "parts": [] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 2", + "parts": [] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 3", + "parts": [] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 4", + "parts": [] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The run was interrupted 5 times in a row; it is not resumed again", + "parts": [] + }, + { + "type": "session.run.failed", + "reason": "crashloop", + "message": "The run was interrupted 5 times in a row; it is not resumed again" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Try again." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Try again." + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "stop", + "parts": [ + { + "text": "Back to normal." + } + ] + }, + { + "type": "session.turn.ended" + } + ], + "history": [ + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "Keep crashing." + } + ] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + }, + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "Try again." + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "text": "Back to normal." + } + ] + } + ], + "requests": [ + [ + { + "role": "user", + "text": "Keep crashing." + } + ], + [ + { + "role": "user", + "text": "Keep crashing." + } + ], + [ + { + "role": "user", + "text": "Keep crashing." + } + ], + [ + { + "role": "user", + "text": "Keep crashing." + } + ], + [ + { + "role": "user", + "text": "Keep crashing." + } + ], + [ + { + "role": "user", + "text": "Keep crashing." + }, + { + "role": "user", + "text": "Try again." + } + ] + ] +} diff --git a/packages/codework/test/__artifacts__/recovery.interrupted.json b/packages/codework/test/__artifacts__/recovery.interrupted.json new file mode 100644 index 00000000..41fc3566 --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.interrupted.json @@ -0,0 +1,89 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "First." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "First." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.prompt.admitted", + "delivery": "followUp", + "text": "Then this." + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 1", + "parts": [] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 2", + "parts": [] + }, + { + "type": "session.run.failed", + "reason": "interrupted", + "message": "The host stopped while the model was answering, on attempt 2" + } + ], + "history": [ + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "First." + } + ] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + } + ], + "queue": [ + { + "delivery": "followUp", + "text": "Then this." + } + ], + "requests": [ + [ + { + "role": "user", + "text": "First." + } + ], + [ + { + "role": "user", + "text": "First." + } + ] + ] +} diff --git a/packages/codework/test/__artifacts__/recovery.mid-stream.json b/packages/codework/test/__artifacts__/recovery.mid-stream.json new file mode 100644 index 00000000..f40aebc2 --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.mid-stream.json @@ -0,0 +1,110 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Answer at length." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Answer at length." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.progress", + "changes": [ + { + "_tag": "set", + "partIndex": 0, + "part": { + "text": "Partial answer, streamed before the host died." + } + } + ] + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 1", + "parts": [ + { + "text": "Partial answer, streamed before the host died." + } + ] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "stop", + "parts": [ + { + "text": "The full answer." + } + ] + }, + { + "type": "session.turn.ended" + } + ], + "live": [ + { + "type": "session.retry.finished", + "attempt": 1, + "success": true + } + ], + "history": [ + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "Answer at length." + } + ] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [ + { + "text": "Partial answer, streamed before the host died." + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "text": "The full answer." + } + ] + } + ], + "requests": [ + [ + { + "role": "user", + "text": "Answer at length." + } + ], + [ + { + "role": "user", + "text": "Answer at length." + } + ] + ] +} diff --git a/packages/codework/test/__artifacts__/recovery.tool.json b/packages/codework/test/__artifacts__/recovery.tool.json new file mode 100644 index 00000000..761ff475 --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.tool.json @@ -0,0 +1,115 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Run the slow command." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Run the slow command." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "stop", + "parts": [ + { + "text": "Recovered after 1 result." + } + ] + }, + { + "type": "session.turn.ended" + } + ], + "history": [ + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "Run the slow command." + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "text": "Recovered after 1 result." + } + ] + } + ], + "requests": [ + [ + { + "role": "user", + "text": "Run the slow command." + } + ], + [ + { + "role": "user", + "text": "Run the slow command." + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + } + ] + ], + "ran": "ran\n" +} diff --git a/packages/codework/test/__artifacts__/recovery.toolcrash.json b/packages/codework/test/__artifacts__/recovery.toolcrash.json new file mode 100644 index 00000000..b3a15fca --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.toolcrash.json @@ -0,0 +1,344 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Run the slow command." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Run the slow command." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.run.failed", + "reason": "crashloop", + "message": "The run was interrupted 5 times in a row; it is not resumed again" + } + ], + "history": [ + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "Run the slow command." + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + ] + } + ], + "requests": [ + [ + { + "role": "user", + "text": "Run the slow command." + } + ], + [ + { + "role": "user", + "text": "Run the slow command." + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + } + ], + [ + { + "role": "user", + "text": "Run the slow command." + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + } + ], + [ + { + "role": "user", + "text": "Run the slow command." + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + } + ], + [ + { + "role": "user", + "text": "Run the slow command." + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + } + ] + ], + "ran": "ran\nran\nran\nran\nran\n" +} diff --git a/packages/codework/test/__artifacts__/recovery.tools.json b/packages/codework/test/__artifacts__/recovery.tools.json new file mode 100644 index 00000000..a46fb4ff --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.tools.json @@ -0,0 +1,136 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Find the echo tool." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Find the echo tool." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "toolsearch", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "toolsearch", + "status": "completed", + "result": "Loaded 1 tool. They are available from your next call:\n- mcp__fixture__echo: Echo text back, with the directory the server runs in." + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 1", + "parts": [] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "stop", + "parts": [ + { + "text": "Resumed with the loaded tool." + } + ] + }, + { + "type": "session.turn.ended" + } + ], + "tools": [ + [ + "bash", + "edit", + "read", + "toolsearch", + "write" + ], + [ + "bash", + "edit", + "mcp__fixture__echo", + "read", + "toolsearch", + "write" + ], + [ + "bash", + "edit", + "mcp__fixture__echo", + "read", + "toolsearch", + "write" + ] + ], + "requests": [ + [ + { + "role": "user", + "text": "Find the echo tool." + } + ], + [ + { + "role": "user", + "text": "Find the echo tool." + }, + { + "role": "assistant", + "calls": [ + "toolsearch" + ] + }, + { + "role": "tool", + "text": "Loaded 1 tool. They are available from your next call:\n- mcp__fixture__echo: Echo text back, with the directory the server runs in." + } + ], + [ + { + "role": "user", + "text": "Find the echo tool." + }, + { + "role": "assistant", + "calls": [ + "toolsearch" + ] + }, + { + "role": "tool", + "text": "Loaded 1 tool. They are available from your next call:\n- mcp__fixture__echo: Echo text back, with the directory the server runs in." + } + ] + ] +} diff --git a/packages/codework/test/__artifacts__/recovery.truncated.json b/packages/codework/test/__artifacts__/recovery.truncated.json new file mode 100644 index 00000000..a10d664e --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.truncated.json @@ -0,0 +1,130 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Run a long command." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Run a long command." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "length", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "skipped", + "result": "Tool call \"bash\" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments." + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "length", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "skipped", + "result": "Tool call \"bash\" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments." + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.run.failed", + "reason": "truncated", + "message": "The response hit the output token limit with tool calls 2 times in a row; the run is ended" + } + ], + "history": [ + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "Run a long command." + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "skipped", + "result": "Tool call \"bash\" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments." + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "skipped", + "result": "Tool call \"bash\" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments." + } + ] + } + ], + "requests": [ + [ + { + "role": "user", + "text": "Run a long command." + } + ], + [ + { + "role": "user", + "text": "Run a long command." + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool call \"bash\" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments." + } + ] + ] +} diff --git a/packages/codework/test/fixtures/cli.ts b/packages/codework/test/fixtures/cli.ts index 9b2b9538..a4e922d7 100644 --- a/packages/codework/test/fixtures/cli.ts +++ b/packages/codework/test/fixtures/cli.ts @@ -30,8 +30,6 @@ const read = (request: IncomingMessage) => request.on("error", reject); }); -const sse = (chunks: ReadonlyArray) => - `${chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("")}data: [DONE]\n\n`; const chunk = (choice: object) => ({ id: "chatcmpl-mcp", object: "chat.completion.chunk", @@ -40,56 +38,86 @@ const chunk = (choice: object) => ({ choices: [{ index: 0, ...choice }], }); +/** One provider response as it streams: `send` writes a chunk carrying `delta`, `finish` ends it. */ +export interface Reply { + readonly send: (delta: object) => void; + readonly finish: (reason: string) => void; +} + /** - * An OpenAI-compatible provider. A request without tool results makes the calls `calls()` names, - * all in one turn; the request carrying their results answers with them, joined by ` | `. + * An OpenAI-compatible provider whose `reply` writes each response (`index` counts requests from + * 0). A response it never finishes hangs until the client goes away. */ -export const provider = async (calls: () => ReadonlyArray) => { +export const streaming = async (reply: (body: ChatRequest, response: Reply, index: number) => void) => { const requests: ChatRequest[] = []; const server = createServer(async (request, response) => { const body = JSON.parse(await read(request)) as ChatRequest; requests.push(body); response.writeHead(200, { "content-type": "text/event-stream" }); + reply( + body, + { + send: (delta) => response.write(`data: ${JSON.stringify(chunk({ delta }))}\n\n`), + finish: (reason) => + response.end(`data: ${JSON.stringify(chunk({ delta: {}, finish_reason: reason }))}\n\ndata: [DONE]\n\n`), + }, + requests.length - 1, + ); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + onTestFinished( + () => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }), + ); + return { url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, requests }; +}; + +/** The assistant delta making `calls`, all in one turn. */ +export const toolCalls = (calls: ReadonlyArray) => ({ + role: "assistant", + tool_calls: calls.map((call, index) => ({ + index, + id: `call_${index}`, + type: "function", + function: { name: call.name, arguments: JSON.stringify(call.arguments) }, + })), +}); + +/** + * A request without tool results makes the calls `calls()` names, all in one turn; the request + * carrying their results answers with them, joined by ` | `. + */ +export const provider = (calls: () => ReadonlyArray) => + streaming((body, response) => { const results = body.messages.filter((message) => message.role === "tool"); if (results.length > 0) { const text = results .map((result) => (typeof result.content === "string" ? result.content : JSON.stringify(result.content))) .join(" | "); - response.end( - sse([ - chunk({ delta: { role: "assistant", content: `Tool said: ${text}` } }), - chunk({ delta: {}, finish_reason: "stop" }), - ]), - ); + response.send({ role: "assistant", content: `Tool said: ${text}` }); + response.finish("stop"); return; } - response.end( - sse([ - chunk({ - delta: { - role: "assistant", - tool_calls: calls().map((call, index) => ({ - index, - id: `call_${index}`, - type: "function", - function: { name: call.name, arguments: JSON.stringify(call.arguments) }, - })), - }, - }), - chunk({ delta: {}, finish_reason: "tool_calls" }), - ]), - ); + response.send(toolCalls(calls())); + response.finish("tool_calls"); + }); + +/** Answers every request with `text`, cut off at the output token limit. */ +export const truncating = (text: string) => + streaming((_body, response) => { + response.send({ role: "assistant", content: text }); + response.finish("length"); }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - onTestFinished(() => new Promise((resolve) => server.close(() => resolve()))); - return { url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, requests }; -}; -/** `settings.jsonc` in `directory`, choosing the fake provider's one model. */ -export const writeSettings = (directory: string, url: string) => +/** `settings.jsonc` in `directory`, choosing the fake provider's one model; `extra` adds settings. */ +export const writeSettings = (directory: string, url: string, extra: object = {}) => writeFileSync( join(directory, "settings.jsonc"), JSON.stringify({ + ...extra, model: { provider: "mcp", id: "mcp-model" }, models: [ { @@ -145,3 +173,47 @@ export const runCli = ( resolve({ status, stdout, stderr }); }); }); + +export interface Served { + /** The RPC endpoint the server announced. */ + readonly url: string; + /** SIGKILLs the server, as a host dying would, and resolves once it is gone. */ + readonly kill: () => Promise; +} + +/** Starts `codework serve` from source on an ephemeral port; resolves once it listens. */ +export const serveCli = ( + args: ReadonlyArray, + options: { readonly cwd: string; readonly env: NodeJS.ProcessEnv }, +) => + new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["--conditions=development", cli, ...args, "serve", "--port", "0"], { + cwd: options.cwd, + env: options.env, + }); + const exited = new Promise((done) => child.on("exit", () => done())); + const kill = () => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + return exited; + }; + onTestFinished(kill); + let output = ""; + const deadline = setTimeout(() => { + void kill(); + reject(new Error(`codework serve did not start:\n${output}`)); + }, 30_000); + const listen = (data: Buffer) => { + output += data.toString("utf8"); + const url = /listening on (ws:\/\/\S+)/.exec(output)?.[1]; + if (url === undefined) return; + clearTimeout(deadline); + resolve({ url, kill }); + }; + child.stdout.on("data", listen); + child.stderr.on("data", (data: Buffer) => (output += data.toString("utf8"))); + child.on("error", reject); + child.on("exit", () => { + clearTimeout(deadline); + reject(new Error(`codework serve exited:\n${output}`)); + }); + }); diff --git a/packages/codework/test/recovery.test.ts b/packages/codework/test/recovery.test.ts new file mode 100644 index 00000000..656c9fa3 --- /dev/null +++ b/packages/codework/test/recovery.test.ts @@ -0,0 +1,488 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- this suite writes its home and settings with node:fs. */ +/* @effect-diagnostics globalTimers:off -- polling for a dying server runs outside any Effect. */ +/* @effect-diagnostics globalDate:off -- so does its deadline. */ +import { Control } from "@codeworksh/harness/effect"; +import { Effect, Fiber, Layer, Stream } from "effect"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it, onTestFinished } from "vite-plus/test"; +import { Client } from "../src/server/client.ts"; +import type { EventEnvelope } from "../src/server/envelope.ts"; +import { type ChatRequest, models, type Reply, serveCli, streaming, toolCalls, writeSettings } from "./fixtures/cli.ts"; + +/* + * A host that dies mid-run (SIGKILL, no shutdown) and a new server on the same home: what the + * reopened server makes of the run the dead one left, observed only over RPC and at the provider. + */ + +const mcpEcho = fileURLToPath(new URL("./fixtures/mcp-echo.mjs", import.meta.url)); + +/** A throwaway home whose settings choose the fake provider; the server runs in its `project`. */ +const setup = (url: string, settings: object = {}) => { + const home = realpathSync(mkdtempSync(join(tmpdir(), "codework-recovery-"))); + onTestFinished(() => rmSync(home, { recursive: true, force: true })); + const project = join(home, "project"); + mkdirSync(project); + writeSettings(home, url, settings); + const start = () => + serveCli(["--home", home, "--user-config-dir", home], { + cwd: project, + env: { ...process.env, HOME: home, CODEWORK_MODELS_FILE: models }, + }); + return { home, project, start }; +}; + +/** One RPC exchange over its own connection. */ +const call = (url: string, use: (rpc: Client.Client) => Effect.Effect) => + Effect.runPromise( + Effect.gen(function* () { + const context = yield* Layer.build(Client.layer(url)); + const rpc = yield* Client.make.pipe(Effect.provideContext(context)); + return yield* use(rpc); + }).pipe(Effect.scoped), + ); + +type Ids = { readonly sessionId: Parameters[0]["sessionId"] }; + +/** The session's durable log, oldest first. */ +const log = (url: string, ids: Ids) => + call(url, (rpc) => + rpc["session.log"]({ sessionId: ids.sessionId }).pipe( + Stream.runCollect, + Effect.map((items) => Array.from(items).filter((item): item is EventEnvelope => item.type !== "log.synced")), + ), + ); + +const count = (events: ReadonlyArray, type: string) => + events.filter((event) => event.type === type).length; + +/** Polls `check` until it holds; fails naming `what` after `ms`. */ +const until = async (what: string, check: () => Promise | A | undefined, ms = 30_000) => { + const deadline = Date.now() + ms; + while (true) { + const value = await check(); + if (value !== undefined && value !== false) return value; + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await new Promise((resolve) => setTimeout(resolve, 50)); + } +}; + +/** Waits for the log to hold `n` events of `type`, and returns it. */ +const logged = (url: string, ids: Ids, type: string, n = 1) => + until(`${n} × ${type}`, async () => { + const events = await log(url, ids); + return count(events, type) >= n ? events : undefined; + }); + +/** Live events of one thread, collected from the moment `ready` resolves. */ +const subscribe = (url: string, threadId: Parameters[0]["threadId"]) => { + const events: EventEnvelope[] = []; + let connected: () => void = () => {}; + const ready = new Promise((resolve) => (connected = resolve)); + const fiber = Effect.runFork( + Effect.gen(function* () { + const context = yield* Layer.build(Client.layer(url)); + const rpc = yield* Client.make.pipe(Effect.provideContext(context)); + yield* rpc["event.subscribe"]({ threadId }).pipe( + Stream.runForEach((event) => + Effect.sync(() => (event.type === "server.connected" ? connected() : events.push(event))), + ), + ); + }).pipe(Effect.scoped, Effect.ignore), + ); + onTestFinished(() => Effect.runPromise(Fiber.interrupt(fiber))); + return { events, ready }; +}; + +/** A gate a test opens: what waits on it holds until then. */ +const gate = () => { + let open: () => void = () => {}; + const opened = new Promise((resolve) => (open = resolve)); + return { open, opened }; +}; + +type Json = Record; + +const part = (value: unknown) => { + const p = value as Json; + if (p["type"] === "text") return { text: p["text"] }; + if (p["type"] === "thinking") return { thinking: p["thinking"] }; + if (p["type"] === "toolCall") { + const result = p["result"] as { content?: ReadonlyArray<{ text?: string }> } | undefined; + return { + tool: p["name"], + status: p["status"], + ...(result === undefined ? {} : { result: result.content?.map((one) => one.text).join("") }), + }; + } + return { type: p["type"] }; +}; + +/** What an event says, without ids, times or paths. */ +const summary = (event: EventEnvelope) => { + const data = event.data as Json; + const message = data["message"] as Json; + switch (event.type) { + case "session.prompt.admitted": + case "session.prompt.promoted": { + const prompt = data["prompt"] as { parts: ReadonlyArray<{ text?: string }> }; + return { type: event.type, delivery: data["delivery"], text: prompt.parts.map((one) => one.text).join("") }; + } + case "session.llm.progress": + return { + type: event.type, + changes: (data["changes"] as ReadonlyArray).map((change) => + change["_tag"] === "append" ? change : { ...change, part: part(change["part"]) }, + ), + }; + case "session.llm.ended": + case "session.llm.failed": + return { + type: event.type, + reason: data["reason"], + ...(message["errorMessage"] === undefined ? {} : { errorMessage: message["errorMessage"] }), + parts: (message["parts"] as ReadonlyArray).map(part), + }; + case "session.tool.settled": + return { type: event.type, part: part(data["part"]) }; + case "session.run.failed": + return { type: event.type, reason: data["reason"], message: data["message"] }; + case "session.retry.scheduled": + case "session.retry.finished": { + const { sessionId: _s, threadId: _t, timestamp: _ts, ...rest } = data; + return { type: event.type, ...rest }; + } + default: + return { type: event.type }; + } +}; + +/** The conversation a request carries, system prompt left out. */ +const sent = (request: ChatRequest) => + request.messages + .filter((message) => message.role !== "system" && message.role !== "developer") + .map((message) => { + const content = message.content; + const text = + typeof content === "string" + ? content + : Array.isArray(content) + ? content.map((one: { text?: string }) => one.text ?? "").join("") + : undefined; + const calls = (message as { tool_calls?: ReadonlyArray<{ function: { name: string } }> }).tool_calls; + return { + role: message.role, + ...(text === undefined || text === "" ? {} : { text }), + ...(calls === undefined ? {} : { calls: calls.map((one) => one.function.name) }), + }; + }); + +const history = async (url: string, ids: Ids) => + (await call(url, (rpc) => rpc["thread.history"](ids))).entries.map((entry) => ({ + role: entry.role, + state: entry.state, + parts: entry.parts.map(part), + })); + +const artifact = (value: unknown) => `${JSON.stringify(value, null, "\t")}\n`; + +const answer = (response: Reply, text: string) => { + response.send({ role: "assistant", content: text }); + response.finish("stop"); +}; + +/** Opens the response and then never finishes it: the request stays in flight until the host dies. */ +const hang = (response: Reply) => response.send({ role: "assistant" }); + +const retry = { retry: { maxRetries: 3, baseDelayMs: 10, maxDelayMs: 1_000 } }; + +describe("recovery after the host is killed", () => { + it("a reply killed mid-stream keeps its partial text, and the request is resent without it", async () => { + const resend = gate(); + const fake = await streaming((_body, response, index) => { + if (index === 0) { + response.send({ role: "assistant", content: "Partial answer, " }); + response.send({ content: "streamed before " }); + response.send({ content: "the host died." }); + return; + } + void resend.opened.then(() => answer(response, "The full answer.")); + }); + const { start } = setup(fake.url, retry); + + const first = await start(); + const created = await call(first.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(first.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Answer at length." })); + // The checkpoint lands a second into the stream: what a crash after it must not lose. + await logged(first.url, ids, "session.llm.progress"); + await first.kill(); + + const second = await start(); + const live = subscribe(second.url, ids.threadId); + await live.ready; + await until("the resend", () => fake.requests.length === 2); + resend.open(); + const events = await logged(second.url, ids, "session.turn.ended"); + await until("retry.finished", () => live.events.find((event) => event.type === "session.retry.finished")); + await call(second.url, (rpc) => rpc["thread.wait"](ids)); + + const failed = events.find((event) => event.type === "session.llm.failed"); + expect(summary(failed!)).toMatchObject({ + reason: "error", + errorMessage: "The host stopped while the model was answering, on attempt 1", + parts: [{ text: "Partial answer, streamed before the host died." }], + }); + // The aborted partial stays out of the model's context. + expect(sent(fake.requests[1]!)).toEqual(sent(fake.requests[0]!)); + + await expect( + artifact({ + log: events.map(summary), + // `session.retry.scheduled` is live only and may go out before a client can attach to the + // new server; `retry.finished` follows the resend this test holds until it has. + live: live.events.filter((event) => event.type === "session.retry.finished").map(summary), + history: await history(second.url, ids), + requests: fake.requests.map(sent), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.mid-stream.json"); + }, 90_000); + + it("a tool killed while it ran settles aborted, and the next request carries that without running it again", async () => { + const marker = join(realpathSync(mkdtempSync(join(tmpdir(), "codework-recovery-marker-"))), "ran"); + onTestFinished(() => rmSync(join(marker, ".."), { recursive: true, force: true })); + const fake = await streaming((body, response, index) => { + if (index === 0) { + response.send(toolCalls([{ name: "bash", arguments: { command: `echo ran >> ${marker} && sleep 20` } }])); + response.finish("tool_calls"); + return; + } + answer(response, `Recovered after ${body.messages.filter((one) => one.role === "tool").length} result.`); + }); + const { start } = setup(fake.url, retry); + + const first = await start(); + const created = await call(first.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(first.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Run the slow command." })); + await until("the tool to run", () => existsSync(marker)); + await first.kill(); + + const second = await start(); + const events = await logged(second.url, ids, "session.turn.ended", 2); + await call(second.url, (rpc) => rpc["thread.wait"](ids)); + + expect(readFileSync(marker, "utf8")).toBe("ran\n"); + expect(fake.requests).toHaveLength(2); + const tool = fake.requests[1]!.messages.find((one) => one.role === "tool"); + expect(tool?.content).toContain("Tool Execution Interrupted"); + + await expect( + artifact({ + log: events.map(summary), + history: await history(second.url, ids), + requests: fake.requests.map(sent), + ran: readFileSync(marker, "utf8"), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.tool.json"); + }, 90_000); + + it("a request killed with no retry left ends the run as interrupted, its follow-up still queued", async () => { + const fake = await streaming((_body, response) => hang(response)); + const { start } = setup(fake.url, { retry: { maxRetries: 1, baseDelayMs: 10, maxDelayMs: 1_000 } }); + + const first = await start(); + const created = await call(first.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(first.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "First." })); + await logged(first.url, ids, "session.llm.started"); + await call(first.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Then this.", delivery: "followUp" })); + await logged(first.url, ids, "session.prompt.admitted", 2); + await first.kill(); + + // The one retry: the request is resent, and the host dies with it in flight again. + const second = await start(); + await logged(second.url, ids, "session.llm.started", 2); + await second.kill(); + + const third = await start(); + const events = await logged(third.url, ids, "session.run.failed"); + await call(third.url, (rpc) => rpc["thread.wait"](ids)); + const queue = await call(third.url, (rpc) => rpc["thread.queue"](ids)); + + expect(fake.requests).toHaveLength(2); + expect(summary(events.find((event) => event.type === "session.run.failed")!)).toEqual({ + type: "session.run.failed", + reason: "interrupted", + message: "The host stopped while the model was answering, on attempt 2", + }); + expect(queue.entries.map((entry) => entry.text)).toEqual(["Then this."]); + + await expect( + artifact({ + log: events.map(summary), + history: await history(third.url, ids), + queue: queue.entries.map(({ delivery, text }) => ({ delivery, text })), + requests: fake.requests.map(sent), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.interrupted.json"); + }, 90_000); + + it("a run the host keeps dying in ends as a crashloop, and the next prompt runs normally", async () => { + let answering = false; + const fake = await streaming((_body, response) => + answering ? answer(response, "Back to normal.") : hang(response), + ); + const { start } = setup(fake.url, { retry: { maxRetries: 10, baseDelayMs: 10, maxDelayMs: 1_000 } }); + + let server = await start(); + const created = await call(server.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(server.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Keep crashing." })); + // The first run, then a resume each until the limit: each dies with its request in flight. + for (let started = 1; started <= Control.RESTART_LIMIT; started++) { + await logged(server.url, ids, "session.llm.started", started); + await server.kill(); + server = await start(); + } + // The resume past the limit is one too many: the run ends instead. + const ended = await logged(server.url, ids, "session.run.failed"); + await call(server.url, (rpc) => rpc["thread.wait"](ids)); + expect(fake.requests).toHaveLength(Control.RESTART_LIMIT); + expect(summary(ended.find((event) => event.type === "session.run.failed")!)).toMatchObject({ + reason: "crashloop", + }); + + answering = true; + await call(server.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Try again." })); + const events = await logged(server.url, ids, "session.turn.ended"); + await call(server.url, (rpc) => rpc["thread.wait"](ids)); + expect(fake.requests).toHaveLength(Control.RESTART_LIMIT + 1); + + await expect( + artifact({ + log: events.map(summary), + history: await history(server.url, ids), + requests: fake.requests.map(sent), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.crashloop.json"); + }, 120_000); + + it("a tool the host keeps dying in ends as a crashloop, each run of it the model's own", async () => { + const marker = join(realpathSync(mkdtempSync(join(tmpdir(), "codework-recovery-marker-"))), "ran"); + onTestFinished(() => rmSync(join(marker, ".."), { recursive: true, force: true })); + const runs = () => (existsSync(marker) ? readFileSync(marker, "utf8").split("\n").length - 1 : 0); + const fake = await streaming((_body, response) => { + response.send(toolCalls([{ name: "bash", arguments: { command: `echo ran >> ${marker} && sleep 20` } }])); + response.finish("tool_calls"); + }); + const { start } = setup(fake.url, retry); + + let server = await start(); + const created = await call(server.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(server.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Run the slow command." })); + // The first run, then a resume each until the limit: each dies with the re-issued tool running. + for (let ran = 1; ran <= Control.RESTART_LIMIT; ran++) { + await until(`run ${ran} of the tool`, () => runs() >= ran); + await server.kill(); + server = await start(); + } + const events = await logged(server.url, ids, "session.run.failed"); + await call(server.url, (rpc) => rpc["thread.wait"](ids)); + + expect(runs()).toBe(Control.RESTART_LIMIT); + expect(fake.requests).toHaveLength(Control.RESTART_LIMIT); + expect(summary(events.find((event) => event.type === "session.run.failed")!)).toMatchObject({ + reason: "crashloop", + }); + + await expect( + artifact({ + log: events.map(summary), + history: await history(server.url, ids), + requests: fake.requests.map(sent), + ran: readFileSync(marker, "utf8"), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.toolcrash.json"); + }, 180_000); + + it("replies cut off at the output limit with tool calls twice in a row end the run as truncated", async () => { + const fake = await streaming((_body, response) => { + response.send(toolCalls([{ name: "bash", arguments: { command: "echo cut" } }])); + response.finish("length"); + }); + const { start } = setup(fake.url, retry); + + const server = await start(); + const created = await call(server.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(server.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Run a long command." })); + const events = await logged(server.url, ids, "session.run.failed"); + await call(server.url, (rpc) => rpc["thread.wait"](ids)); + + expect(fake.requests).toHaveLength(2); + const settled = events.filter((event) => event.type === "session.tool.settled").map(summary); + expect(settled).toHaveLength(2); + for (const one of settled) expect(JSON.stringify(one)).toContain("Re-issue the tool call"); + expect(summary(events.find((event) => event.type === "session.run.failed")!)).toEqual({ + type: "session.run.failed", + reason: "truncated", + message: "The response hit the output token limit with tool calls 2 times in a row; the run is ended", + }); + + await expect( + artifact({ + log: events.map(summary), + history: await history(server.url, ids), + requests: fake.requests.map(sent), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.truncated.json"); + }, 90_000); + + it("a deferred tool loaded before the host died is still offered to the resent request", async () => { + const tools = (request: ChatRequest) => (request.tools ?? []).map((tool) => tool.function.name).sort(); + const fake = await streaming((_body, response, index) => { + if (index === 0) { + response.send(toolCalls([{ name: "toolsearch", arguments: { query: "echo" } }])); + response.finish("tool_calls"); + return; + } + if (index === 1) return hang(response); + answer(response, "Resumed with the loaded tool."); + }); + const { home, start } = setup(fake.url, retry); + writeFileSync( + join(home, "mcp.json"), + JSON.stringify({ + mcpServers: { fixture: { command: process.execPath, args: [mcpEcho], exposure: "deferred" } }, + }), + ); + + const first = await start(); + const created = await call(first.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(first.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Find the echo tool." })); + await logged(first.url, ids, "session.llm.started", 2); + await first.kill(); + + const second = await start(); + const events = await logged(second.url, ids, "session.turn.ended", 2); + await call(second.url, (rpc) => rpc["thread.wait"](ids)); + + expect(fake.requests).toHaveLength(3); + expect(tools(fake.requests[0]!)).not.toContain("mcp__fixture__echo"); + expect(tools(fake.requests[1]!)).toContain("mcp__fixture__echo"); + expect(tools(fake.requests[2]!)).toContain("mcp__fixture__echo"); + + await expect( + artifact({ + log: events.map(summary), + tools: fake.requests.map(tools), + requests: fake.requests.map(sent), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.tools.json"); + }, 90_000); +}); diff --git a/packages/harness/src/control.ts b/packages/harness/src/control.ts index 547c3bfc..eb292386 100644 --- a/packages/harness/src/control.ts +++ b/packages/harness/src/control.ts @@ -55,8 +55,8 @@ export interface PromptInput extends ThreadRef { export type ResolveError = Session.SessionNotFoundError | Thread.ThreadNotFoundError; /** - * Times a run may be cut off by a dying process before the thread is parked instead of resumed. - * A committed turn starts the count over. + * Times in a row a run may be cut off by a dying process before it ends (`session.run.failed`, + * `crashloop`) instead of resuming. A committed turn starts the count over. */ export const RESTART_LIMIT = 5; @@ -100,8 +100,9 @@ export interface Interface { readonly active: Effect.Effect>; /** * Resumes the runs the last process left unfinished, as the server starts. One a crash cut off - * counts a restart, one a shutdown closed does not; at {@link RESTART_LIMIT} the thread is parked - * and waits for its next prompt. Resolves once every resumed run has settled. + * counts a restart, one a shutdown closed does not; at {@link RESTART_LIMIT} the run ends visibly + * instead, the thread idle with its queue kept for the next prompt. Resolves once every resumed + * run has settled. */ readonly resumeUnfinished: Effect.Effect; /** Prompts admitted but not yet in the conversation, steers and follow-ups, in admission order. */ @@ -271,8 +272,15 @@ export const layer = Layer.effect( // Claimed again here, not from the list: a stop since then cleared it, and must stand. // Counted before resuming, so a crash inside the resumed run is counted too. const restarts = yield* threads.claimRestart(threadId); - if (Option.isNone(restarts) || restarts.value >= RESTART_LIMIT) return; - yield* execution.recover(threadId).pipe( + if (Option.isNone(restarts)) return; + const recovery = + restarts.value >= RESTART_LIMIT + ? execution.abandon( + threadId, + `The run was interrupted ${restarts.value} times in a row; it is not resumed again`, + ) + : execution.recover(threadId); + yield* recovery.pipe( Effect.tapCause((cause) => Effect.logWarning("Resuming an unfinished thread failed", cause).pipe( Effect.annotateLogs({ threadId }), diff --git a/packages/harness/src/db/migrations.ts b/packages/harness/src/db/migrations.ts index e520d0bc..e3c9f3a1 100644 --- a/packages/harness/src/db/migrations.ts +++ b/packages/harness/src/db/migrations.ts @@ -124,6 +124,7 @@ export const migrations = { title TEXT, execution TEXT, restarts INTEGER NOT NULL DEFAULT 0, + attempt INTEGER, cost REAL NOT NULL DEFAULT 0, tokens_input INTEGER NOT NULL DEFAULT 0, tokens_output INTEGER NOT NULL DEFAULT 0, diff --git a/packages/harness/src/db/schema.sql.ts b/packages/harness/src/db/schema.sql.ts index da63974a..6213f195 100644 --- a/packages/harness/src/db/schema.sql.ts +++ b/packages/harness/src/db/schema.sql.ts @@ -143,6 +143,9 @@ export class ThreadRow extends Model.Class("ThreadRow")({ title: Model.FieldOption(Schema.String), execution: Model.FieldOption(ThreadSchema.Execution), restarts: Model.GeneratedByDb(Schema.Int), + // The 1-based number of the provider request in flight, from its start until its response ends, the + // turn commits or the run ends: still set on recovery, the host died with that request out. + attempt: Model.FieldOption(Schema.Int), // Billed usage of own entries only, never the inherited prefix. A cache: assistant envelopes are truth. cost: Model.GeneratedByDb(Schema.Finite), tokensInput: Model.GeneratedByDb(Schema.Int), diff --git a/packages/harness/src/effect/session.ts b/packages/harness/src/effect/session.ts index bdd8c7e1..1f5ffc3f 100644 --- a/packages/harness/src/effect/session.ts +++ b/packages/harness/src/effect/session.ts @@ -128,11 +128,6 @@ export interface ThreadInfo { readonly foundation?: string; /** Where it runs instead of the session's space and directory. */ readonly placement?: DocumentSchema.Placement; - /** - * Its unfinished run was cut off {@link Control.RESTART_LIMIT} times, so the server no longer - * resumes it; the next prompt runs it again. - */ - readonly parked: boolean; /** Billed usage of its own entries; what it inherits through a fork point counts for its source. */ readonly usage: Usage; readonly session: Info; @@ -279,14 +274,11 @@ const originOf = (row: ThreadRow): ThreadSchema.Origin => { export const threadInfo = Effect.fn("Session.threadInfo")(function* (id: ThreadSchema.ID) { const threads = yield* Thread.Service; const documents = yield* Document.Service; - const control = yield* Control.Service; const found = yield* threads.get(id); if (Option.isNone(found)) return yield* new Thread.ThreadNotFoundError({ threadId: id }); const row = found.value; const agent = yield* documents.agent(id); const title = Option.getOrUndefined(row.title); - const parked = - Option.isSome(row.execution) && row.restarts >= Control.RESTART_LIMIT && !(yield* control.active).has(id); return { id, sessionId: SessionSchema.ID.make(row.sessionId), @@ -297,7 +289,6 @@ export const threadInfo = Effect.fn("Session.threadInfo")(function* (id: ThreadS ...(agent.thinkingLevel === undefined ? {} : { thinkingLevel: agent.thinkingLevel }), ...(agent.foundation === undefined ? {} : { foundation: agent.foundation }), ...(agent.placement === undefined ? {} : { placement: agent.placement }), - parked, usage: { cost: row.cost, input: row.tokensInput, diff --git a/packages/harness/src/event/list.ts b/packages/harness/src/event/list.ts index cd893099..35ae9012 100644 --- a/packages/harness/src/event/list.ts +++ b/packages/harness/src/event/list.ts @@ -147,10 +147,14 @@ export const TurnStarted = EventSchema.define({ }); export type TurnStarted = typeof TurnStarted.Type; +/** + * A draft committed. `recovered` marks the commit a restart makes of a turn the host died in while + * its tools ran: no progress, so the restart count keeps counting. + */ export const TurnEnded = EventSchema.define({ type: "session.turn.ended", ...durableOptions, - schema: LLMFields, + schema: { ...LLMFields, recovered: optional(Schema.Literal(true)) }, }); export type TurnEnded = typeof TurnEnded.Type; @@ -232,6 +236,27 @@ export const LLMThinkingEnd = EventSchema.define({ }); export type LLMThinkingEnd = typeof LLMThinkingEnd.Type; +/** + * What one streamed part gained since the draft's last checkpoint: `append` extends a text or + * thinking part's text, `set` writes a new or otherwise changed part whole. + */ +export const LLMProgressChange = Schema.Union([ + Schema.TaggedStruct("append", { partIndex: NonNegativeInt, text: Schema.String }), + Schema.TaggedStruct("set", { partIndex: NonNegativeInt, part: EventSchema.AikitAssistantPart }), +]); +export type LLMProgressChange = typeof LLMProgressChange.Type; + +/** + * A checkpoint of the streaming reply into its draft, at most one a second, so a host that dies + * mid-reply leaves what was streamed. Carries only what changed since the previous checkpoint. + */ +export const LLMProgress = EventSchema.define({ + type: "session.llm.progress", + ...durableOptions, + schema: { ...LLMFields, changes: Schema.Array(LLMProgressChange) }, +}); +export type LLMProgress = typeof LLMProgress.Type; + /** aikit `done`: replace the LLMStarted payload in place; state stays draft. */ export const LLMEnded = EventSchema.define({ type: "session.llm.ended", @@ -244,7 +269,10 @@ export const LLMEnded = EventSchema.define({ }); export type LLMEnded = typeof LLMEnded.Type; -/** Abort the whole draft and discard every part. */ +/** + * End the draft as aborted, keeping the output it streamed. Its unfinished tool calls are aborted, + * never run. An aborted entry stays out of the model's context. + */ export const LLMFailed = EventSchema.define({ type: "session.llm.failed", ...durableOptions, @@ -317,6 +345,22 @@ export const ExecutionInterrupted = EventSchema.define({ }); export type ExecutionInterrupted = typeof ExecutionInterrupted.Type; +/** + * A run ended without its answer and is not resumed: the host died in it too often (`crashloop`), + * with no retry left for the request it cut off (`interrupted`), or replies kept hitting the output + * limit with tool calls (`truncated`). Queued prompts stay queued. + */ +export const RunFailed = EventSchema.define({ + type: "session.run.failed", + ...durableOptions, + schema: { + ...baseOptions, + reason: Schema.Literals(["crashloop", "interrupted", "truncated"]), + message: Schema.String, + }, +}); +export type RunFailed = typeof RunFailed.Type; + /** * Kernel lifecycle notice for one plugin module. * The `plugin..` namespace belongs to plugin-declared types, so this kernel type can never collide with one. @@ -411,8 +455,10 @@ export const Definitions = EventSchema.inventory( LLMThinkingStart, LLMThinkingDelta, LLMThinkingEnd, + LLMProgress, LLMEnded, LLMFailed, + RunFailed, ToolStarted, ToolProgress, ToolSettled, @@ -438,8 +484,10 @@ export const DurableDefinitions = EventSchema.inventory( SystemChanged, TurnEnded, LLMStarted, + LLMProgress, LLMEnded, LLMFailed, + RunFailed, ToolSettled, ); diff --git a/packages/harness/src/runner/event.ts b/packages/harness/src/runner/event.ts index 7c8568fa..054c3079 100644 --- a/packages/harness/src/runner/event.ts +++ b/packages/harness/src/runner/event.ts @@ -16,19 +16,57 @@ * events carry only fragments, so it has to reassemble text across deltas before * it can persist a block. Every aikit event already carries a complete assistant * message, and the terminal one carries the whole response. Successful output - * can therefore replace the draft in one write; failed output becomes a - * partless tombstone without needing a fragment buffer. + * can therefore replace the draft in one write, and failed output keeps the + * latest partial without needing a fragment buffer. While streaming, that + * partial is checkpointed into the draft (`LLMProgress`) so a dead host leaves it. */ import type { Event as AikitEvent, Message } from "@codeworksh/aikit"; -import { DateTime, Effect } from "effect"; +import { DateTime, Duration, Effect, Fiber, Option, Schema } from "effect"; import { Event } from "../event/event.ts"; import { EventList } from "../event/list.ts"; +import { EventSchema } from "../event/schema.ts"; import { SessionMessageSchema } from "../session/message/schema.ts"; import type { SessionSchema } from "../session/schema.ts"; import type { ThreadSchema } from "../thread/schema.ts"; +import { RunnerPart } from "./part.ts"; import { Runner } from "./run.ts"; +type Part = Message.AssistantMessage["parts"][number]; + +const CHECKPOINT_INTERVAL = Duration.seconds(1); + +const decodePartial = Schema.decodeUnknownEffect(EventSchema.AikitAssistantMessage); + +const textOf = (part: Part) => + part.type === "text" ? part.text : part.type === "thinking" ? part.thinking : undefined; +const withoutText = (part: Part): Part => + part.type === "text" ? { ...part, text: "" } : part.type === "thinking" ? { ...part, thinking: "" } : part; + +/** What `after` changed since `before`: grown text appends, every other difference writes the part whole. */ +const changesSince = (before: ReadonlyArray, after: ReadonlyArray) => { + const changes: Array = []; + for (const [partIndex, part] of after.entries()) { + const previous = before[partIndex]; + const was = previous === undefined ? undefined : textOf(previous); + const now = textOf(part); + if ( + previous !== undefined && + was !== undefined && + now !== undefined && + now.startsWith(was) && + JSON.stringify(withoutText(previous)) === JSON.stringify(withoutText(part)) + ) { + if (now.length > was.length) changes.push({ _tag: "append", partIndex, text: now.slice(was.length) }); + continue; + } + if (previous === undefined || JSON.stringify(previous) !== JSON.stringify(part)) { + changes.push({ _tag: "set", partIndex, part }); + } + } + return changes; +}; + /** How the response settled. The loop branches on this, never on `result()`. */ export type Terminal = | { @@ -45,6 +83,10 @@ export type Terminal = export interface Publisher { /** Set only after LLMStarted committed its draft row. */ readonly startedMessageId: SessionMessageSchema.ID | undefined; + /** A copy of the latest streamed message until the response settles: what a failure it did not report keeps. */ + readonly partial: Message.AssistantMessage | undefined; + /** Ends checkpointing, waiting out one in flight: no checkpoint lands after this returns. */ + readonly stop: Effect.Effect; /** Interpret one provider event. Durable ones commit before this returns. */ readonly publish: (event: AikitEvent.LLMMessageEvent) => Effect.Effect; /** How the response settled; fails if the stream ended without terminating. */ @@ -62,6 +104,69 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { let settled: Terminal | undefined; const clone = (message: Message.AssistantMessage): Message.AssistantMessage => structuredClone(message); + // aikit rewrites one partial object in place, so this is a live view: copied when read. + let latest: Message.AssistantMessage | undefined; + let checkpointed: ReadonlyArray = []; + let dirty = false; + let stopped = false; + let ticker: Fiber.Fiber | undefined; + + const checkpoint = Effect.gen(function* () { + dirty = false; + if (latest === undefined || startedMessageId === undefined) return; + // A partial mid-way through a part may not be a valid message yet; the next checkpoint takes it. + const decoded = yield* decodePartial(clone(latest)).pipe(Effect.option); + if (Option.isNone(decoded)) return; + const parts = decoded.value.parts; + const changes = changesSince(checkpointed, parts); + if (changes.length === 0) return; + yield* events.publish(EventList.LLMProgress, { + sessionId: input.sessionId, + threadId: input.threadId, + timestamp: yield* DateTime.now, + messageId: startedMessageId, + changes, + }); + checkpointed = parts; + }); + + // One checkpoint a second at most, one in flight; the ticker ends once a checkpoint finds nothing new. + const tick = Effect.gen(function* () { + while (true) { + yield* Effect.sleep(CHECKPOINT_INTERVAL); + yield* Effect.uninterruptible(checkpoint); + // Cleared with the decision, no yield between: an `observe` after it forks a fresh ticker. + if (!dirty) { + ticker = undefined; + return; + } + } + }).pipe( + // Not `ensuring`: after a normal exit `ticker` may already be the fresh one. + Effect.onError(() => + Effect.sync(() => { + ticker = undefined; + }), + ), + ); + + const observe = (partial: Message.AssistantMessage) => + Effect.suspend(() => { + // A partial without content, such as the opening one, shows nothing worth keeping. + if (stopped || partial.parts.length === 0) return Effect.void; + latest = partial; + dirty = true; + if (ticker !== undefined) return Effect.void; + return Effect.map(Effect.forkDetach(tick), (fiber) => { + ticker = fiber; + }); + }); + + const stop = Effect.suspend(() => { + stopped = true; + return ticker === undefined ? Effect.void : Fiber.interrupt(ticker); + }); + const fault = (reason: string) => new Runner.LLMStreamError({ threadId: input.threadId, reason }); /** @@ -95,6 +200,14 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { } const timestamp = yield* DateTime.now; const base = { sessionId: input.sessionId, threadId: input.threadId, timestamp }; + if ( + event.type !== "start" && + event.type !== "done" && + event.type !== "error" && + event.partial.messageId === startedMessageId + ) { + yield* observe(event.partial); + } switch (event.type) { case "start": { @@ -116,9 +229,8 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { /* * Live only, all of them. The terminal message contains every finished - * block, so persisting these would write the same text twice and, worse, - * leave an entry that a crash could strand mid-response -- which - * `Context.assemble` would then replay to the model as a complete turn. + * block; what a crash would lose is kept by the throttled checkpoints of + * the draft instead, which stays out of the model's context. * * Only scalars computed at push time are carried. `partial` is excluded * deliberately: it is one object aikit rewrites in place and pushes by @@ -193,6 +305,7 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { if (startedMessageId === undefined) return yield* fault("received done before start"); const message = clone(event.message); const identified = yield* identify(message.messageId); + yield* stop; yield* Effect.uninterruptible( Effect.gen(function* () { yield* events.publish(EventList.LLMEnded, { @@ -211,12 +324,13 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { /* * aikit calls the payload `error`, but it is a complete assistant - * message. LLMFailed uses its envelope as an aborted tombstone and the - * projector deliberately discards every partial part. + * message, with whatever streamed before the failure. LLMFailed keeps it + * as the aborted entry; a tool call in it never runs. */ case "error": { - const message = clone(event.error); + const message = RunnerPart.abortCalls(clone(event.error), DateTime.toEpochMillis(timestamp)); const identified = yield* identify(message.messageId); + yield* stop; yield* Effect.uninterruptible( Effect.gen(function* () { if (startedMessageId !== undefined) { @@ -245,6 +359,10 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { get startedMessageId() { return startedMessageId; }, + get partial() { + return settled !== undefined || latest === undefined ? undefined : clone(latest); + }, + stop, publish, terminal, } satisfies Publisher; diff --git a/packages/harness/src/runner/execute.ts b/packages/harness/src/runner/execute.ts index 07c3712c..7847dbc4 100644 --- a/packages/harness/src/runner/execute.ts +++ b/packages/harness/src/runner/execute.ts @@ -80,6 +80,8 @@ export const layer = Layer.effect( // Threads being recovered: their run neither marks the thread nor runs once it is unmarked. const recovering = new Set(); + // Recovered threads whose run is ended instead of resumed, with the message it ends with. + const abandoning = new Map(); // The log position at each user stop: prompts admitted after it are new work, not parked by it. const stoppedAt = new Map(); @@ -133,6 +135,9 @@ export const layer = Layer.effect( const sessionId = thread.value.sessionId; const recovered = recovering.delete(threadId); if (recovered && Option.isNone(thread.value.execution)) return; + // Only the drain the abandon itself started: one it joined is a live run, which must stand. + const abandoned = recovered && force ? abandoning.get(threadId) : undefined; + if (abandoned !== undefined) return yield* runner.abandon({ threadId, message: abandoned }); // The thread's placement, else the session's space and directory. A placement names its // space in a document no FK guards, so a space gone since is typed: `relink --thread` // is the repair. @@ -201,14 +206,18 @@ export const layer = Layer.effect( return accepted; }); - const recover = (threadId: ThreadId) => + // Under the thread's execution like any run, so a prompt that wakes it meanwhile cannot heal + // the same draft. + const recover = (threadId: ThreadId, abandon?: string) => Effect.suspend(() => { recovering.add(threadId); + if (abandon !== undefined) abandoning.set(threadId, abandon); return coordinator.run(threadId); }).pipe( Effect.ensuring( Effect.sync(() => { recovering.delete(threadId); + abandoning.delete(threadId); }), ), ); @@ -217,7 +226,8 @@ export const layer = Layer.effect( active: coordinator.active, interrupt, resume: coordinator.run, - recover, + recover: (threadId) => recover(threadId), + abandon: recover, wake: coordinator.wake, awaitIdle: coordinator.awaitIdle, }); diff --git a/packages/harness/src/runner/execution.ts b/packages/harness/src/runner/execution.ts index 21114270..fc3725f6 100644 --- a/packages/harness/src/runner/execution.ts +++ b/packages/harness/src/runner/execution.ts @@ -19,6 +19,11 @@ export interface Interface { * recovery chose the thread stands. */ readonly recover: (threadId: ThreadSchema.ID) => Effect.Effect; + /** + * Ends a run the last process left unfinished instead of resuming it, under the thread's execution + * like `recover`; see `Runner.Interface.abandon`. + */ + readonly abandon: (threadId: ThreadSchema.ID, message: string) => Effect.Effect; /** Registers newly recorded work. Repeated wakeups may coalesce. */ readonly wake: (threadId: ThreadSchema.ID) => Effect.Effect; /** @@ -43,6 +48,7 @@ export const noopLayer = Layer.succeed( active: Effect.succeed(new Set()), resume: () => Effect.void, recover: () => Effect.void, + abandon: () => Effect.void, wake: () => Effect.void, interrupt: () => Effect.succeed(false), awaitIdle: () => Effect.void, diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index 76f8de26..a8fce6b3 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -7,6 +7,7 @@ import { type Message, Transcript } from "@codeworksh/aikit"; import { Cause, DateTime, Duration, Effect, Exit, Layer, Option, Schema } from "effect"; +import type { ContextAssembly } from "../context/assemble.ts"; import { ContextCodec } from "../context/codec.ts"; import { Context } from "../context/context.ts"; import { Event } from "../event/event.ts"; @@ -14,6 +15,7 @@ import { EventList } from "../event/list.ts"; import { Inbox } from "../inbox/inbox.ts"; import { McpSection } from "../mcp/section.ts"; import { SessionMessageSchema } from "../session/message/schema.ts"; +import { SessionSchema } from "../session/schema.ts"; import type { ThreadSchema } from "../thread/schema.ts"; import { Thread } from "../thread/thread.ts"; import { State } from "../state/state.ts"; @@ -23,29 +25,30 @@ import { Hash } from "../util/hash.ts"; import { LLMEventPublisher } from "./event.ts"; import { Hooks } from "./hooks.ts"; import { LLM } from "./llm.ts"; +import { RunnerPart } from "./part.ts"; import { Retry } from "./retry.ts"; import { Runner } from "./run.ts"; export type Promotion = "steer" | "followUp" | undefined; /** A turn that committed an assistant message; `undefined` when it sent no request. */ -export type TurnResult = Pick | undefined; +export type TurnResult = + | (Pick & { + /** Whether the reply called tools, every one of them settled by now. */ + readonly toolCalls: boolean; + }) + | undefined; export interface Options { /** Deterministic provider seam for tests. */ readonly request?: LLM.Request; } -const terminalResult = (text: string) => ({ - content: [{ type: "text" as const, text }], - isError: true as const, -}); +const { abortedPart, endTime, terminalResult } = RunnerPart; const encodeJson = (value: unknown) => Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))(value).pipe(Effect.orDie); -const endTime = (call: Message.ToolCallPendingPart, now: number) => Math.max(call.time.end, now); - const skippedPart = (call: Message.ToolCallPendingPart, message: string, now: number): Message.ToolCallSkippedPart => ({ ...call, status: "skipped", @@ -53,12 +56,37 @@ const skippedPart = (call: Message.ToolCallPendingPart, message: string, now: nu time: { ...call.time, end: endTime(call, now) }, }); -const abortedPart = (call: Message.ToolCallPendingPart, now: number): Message.ToolCallAbortedPart => ({ - ...call, - status: "aborted", - result: terminalResult("Tool Execution Interrupted"), - time: { ...call.time, end: endTime(call, now) }, -}); +const truncatedCall = (name: string) => + `Tool call "${name}" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments.`; + +/** Replies in a row cut off at the output limit with tool calls before the run ends. */ +const TRUNCATION_LIMIT = 2; + +const hasToolCalls = (message: Message.AssistantMessage) => message.parts.some((part) => part.type === "toolCall"); + +/** Whether the leaf is a reply the model finished: a cut-off one with tool calls is answered once their results are in. */ +const leafAnswered = (assembled: ContextAssembly.Snapshot) => { + const leaf = + assembled.lastAssistant?.entryId === assembled.leafEntryId ? assembled.lastAssistant.message : undefined; + return leaf !== undefined && (leaf.stopReason === "stop" || (leaf.stopReason === "length" && !hasToolCalls(leaf))); +}; + +/** + * A request the host died with in flight counts as a failed attempt, retried like a transient + * provider failure. + */ +const hostStopped = (snapshot: State.Snapshot, message: string) => + new Runner.ProviderError({ + provider: snapshot.provider, + model: snapshot.model, + reason: new Runner.ProviderTransportError({ message, isRetryable: true }), + }); + +/** A request still in flight when the host died, as recovery finds it. */ +interface Crashed { + readonly attempt: number; + readonly message: string; +} const errorPart = ( call: Message.ToolCallPendingPart, @@ -109,26 +137,37 @@ export const layer = (options: Options = {}) => const state = yield* State.Service; const requestLLM = options.request ?? LLM.run; + const decodeDraft = Effect.fn("Loop.decodeDraft")(function* (draft: Thread.HydratedEntry) { + const decoded = yield* ContextCodec.decodeMessage(draft).pipe(Effect.orDie); + if (decoded.role !== "assistant") return yield* Effect.die(`draft ${draft.entry.id} is not an assistant`); + return decoded; + }); + + /** The aborted entry: what streamed (`partial`, else what the draft holds), its tool calls never run. */ const failureStub = Effect.fn("Loop.failureStub")(function* ( draft: Thread.HydratedEntry, reason: "aborted" | "error", message: string, + partial: Message.AssistantMessage | undefined, ) { - const decoded = yield* ContextCodec.decodeMessage(draft).pipe(Effect.orDie); - if (decoded.role !== "assistant") return yield* Effect.die(`draft ${draft.entry.id} is not an assistant`); + const decoded = yield* decodeDraft(draft); const completed = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - return { - ...decoded, - stopReason: reason, - errorMessage: message, - time: { ...decoded.time, completed }, - parts: [], - } satisfies Message.AssistantMessage; + return RunnerPart.abortCalls( + { + ...decoded, + stopReason: reason, + errorMessage: message, + time: { ...decoded.time, completed }, + parts: (partial ?? decoded).parts, + }, + completed, + ); }); const publishFailure = Effect.fn("Loop.publishFailure")(function* ( draft: Thread.HydratedEntry, cause: EventList.TurnAbortCause, + partial?: Message.AssistantMessage, ) { const reason = cause._tag === "interrupted" ? "aborted" : "error"; const message = cause._tag === "interrupted" ? "turn interrupted before commit" : cause.message; @@ -138,14 +177,106 @@ export const layer = (options: Options = {}) => threadId: draft.entry.threadId, messageId: SessionMessageSchema.ID.from(draft.entry.id), reason, - message: yield* failureStub(draft, reason, message), + message: yield* failureStub(draft, reason, message, partial), + }); + }); + + const publishSettled = Effect.fn("Loop.publishSettled")(function* ( + ids: { readonly sessionId: SessionSchema.ID; readonly threadId: ThreadSchema.ID }, + messageId: SessionMessageSchema.ID, + part: Message.ToolCallTerminalPart, + ) { + yield* events.publish(EventList.ToolSettled, { + timestamp: yield* DateTime.now, + sessionId: ids.sessionId, + threadId: ids.threadId, + messageId, + callID: part.callID, + part, + }); + }); + + const publishCommit = Effect.fn("Loop.publishCommit")(function* ( + ids: { readonly sessionId: SessionSchema.ID; readonly threadId: ThreadSchema.ID }, + messageId: SessionMessageSchema.ID, + recovered?: true, + ) { + yield* events.publish(EventList.TurnEnded, { + timestamp: yield* DateTime.now, + sessionId: ids.sessionId, + threadId: ids.threadId, + messageId, + ...(recovered === undefined ? {} : { recovered }), }); }); - const healDanglingDraft = Effect.fn("Loop.healDanglingDraft")(function* (threadId: ThreadSchema.ID) { + /** + * Settles the draft a dead process left. One whose request was in flight (`inFlight`, its + * failure) is aborted with what streamed. One whose response ended with tool calls had them + * running: those not settled are aborted, never run again, and the turn commits so the model + * sees what ran. Anything else is aborted as interrupted. + */ + const healDraft = Effect.fn("Loop.healDraft")(function* ( + threadId: ThreadSchema.ID, + inFlight: string | undefined, + ) { const draft = yield* threads.draft(threadId); if (Option.isNone(draft)) return; - yield* publishFailure(draft.value, { _tag: "interrupted" }); + if (inFlight !== undefined) return yield* publishFailure(draft.value, { _tag: "error", message: inFlight }); + const decoded = yield* decodeDraft(draft.value); + if (!hasToolCalls(decoded)) return yield* publishFailure(draft.value, { _tag: "interrupted" }); + const ids = { sessionId: draft.value.entry.sessionId, threadId }; + const messageId = SessionMessageSchema.ID.from(decoded.messageId); + yield* Effect.uninterruptible( + Effect.gen(function* () { + for (const part of decoded.parts) { + if (part.type !== "toolCall" || (part.status !== "pending" && part.status !== "running")) continue; + const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); + yield* publishSettled(ids, messageId, abortedPart(part, now)); + } + yield* publishCommit(ids, messageId, true); + }), + ); + }); + + /** Announces a retry of a failed request, then waits it out. */ + const scheduleRetry = Effect.fn("Loop.scheduleRetry")(function* ( + snapshot: State.Snapshot, + attempt: number, + delayMs: number, + message: string, + ) { + yield* events.publish(EventList.RetryScheduled, { + timestamp: yield* DateTime.now, + sessionId: snapshot.sessionId, + threadId: snapshot.threadId, + attempt, + maxRetries: snapshot.retry.maxRetries, + delayMs, + message, + }); + yield* Effect.sleep(Duration.millis(delayMs)); + }); + + /** Ends a run that is not resumed, leaving the thread idle with its queue kept. */ + const endRun = Effect.fn("Loop.endRun")(function* ( + sessionId: SessionSchema.ID, + threadId: ThreadSchema.ID, + reason: EventList.RunFailed["data"]["reason"], + message: string, + ) { + yield* Effect.uninterruptible( + Effect.gen(function* () { + yield* events.publish(EventList.RunFailed, { + timestamp: yield* DateTime.now, + sessionId, + threadId, + reason, + message, + }); + yield* threads.idle(threadId); + }), + ); }); const settleTools = Effect.fn("Loop.settleTools", { level: "Info" })(function* ( @@ -160,35 +291,16 @@ export const layer = (options: Options = {}) => let interruptedCause: Cause.Cause | undefined; const messageId = SessionMessageSchema.ID.from(message.messageId); - const settle = Effect.fn("Loop.settleTool")(function* ( - call: Message.ToolCallPendingPart, - part: Message.ToolCallTerminalPart, - ) { - yield* events.publish(EventList.ToolSettled, { - timestamp: yield* DateTime.now, - sessionId: snapshot.sessionId, - threadId: snapshot.threadId, - messageId, - callID: call.callID, - part, - }); - }); - const commit = Effect.fn("Loop.commitTurn")(function* () { - yield* events.publish(EventList.TurnEnded, { - timestamp: yield* DateTime.now, - sessionId: snapshot.sessionId, - threadId: snapshot.threadId, - messageId, - }); - markCommitted(); - }); + const settle = (part: Message.ToolCallTerminalPart) => publishSettled(snapshot, messageId, part); + const commit = () => publishCommit(snapshot, messageId).pipe(Effect.tap(() => Effect.sync(markCommitted))); yield* Effect.uninterruptibleMask((restore) => Effect.gen(function* () { + // The model re-issues them from these results: the drain continues as after tool use. if (reason === "length") { for (const call of pending) { const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - yield* settle(call, skippedPart(call, "tool call arguments were truncated", now)); + yield* settle(skippedPart(call, truncatedCall(call.name), now)); } yield* commit(); return; @@ -197,7 +309,7 @@ export const layer = (options: Options = {}) => if (reason !== "toolUse") { for (const call of pending) { const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - yield* settle(call, skippedPart(call, "unexpected tool call for terminal response", now)); + yield* settle(skippedPart(call, "unexpected tool call for terminal response", now)); } yield* commit(); return; @@ -258,7 +370,7 @@ export const layer = (options: Options = {}) => // cleanup publish a second terminal transition for the same call. yield* Effect.uninterruptible( Effect.gen(function* () { - yield* settle(call, handled); + yield* settle(handled); yield* Effect.sync(() => settled.add(call.callID)); }), ); @@ -275,7 +387,7 @@ export const layer = (options: Options = {}) => for (const call of pending) { if (settled.has(call.callID)) continue; const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - yield* settle(call, abortedPart(call, now)); + yield* settle(abortedPart(call, now)); } } yield* commit(); @@ -323,6 +435,7 @@ export const layer = (options: Options = {}) => promotion: Promotion, snapshot: State.Snapshot, continuing: boolean, + carried: Crashed | undefined, ) { const { sessionId, threadId } = snapshot; if (promotion !== undefined) { @@ -335,26 +448,25 @@ export const layer = (options: Options = {}) => } const assembled = yield* context.assemble(threadId); - const leaf = - assembled.lastAssistant?.entryId === assembled.leafEntryId ? assembled.lastAssistant.message : undefined; - if (leaf !== undefined && (leaf.stopReason === "stop" || leaf.stopReason === "length")) { - return undefined; - } + if (leafAnswered(assembled)) return undefined; const messages = [...assembled.messages, ...(yield* declareSystemChanges(snapshot, assembled.messages))]; // The latest attempt's publisher; a retried turn has one per provider request. let publisher: LLMEventPublisher.Publisher | undefined; let committed = false; - let retries = 0; - let failed: string | undefined; + // A recovered run goes on counting from the request the host died with. + let retries = carried?.attempt ?? 0; + let failed: string | undefined = carried?.message; /** Abort the latest attempt's draft, unless its own terminal event already did. */ const settleDraft = (cause: EventList.TurnAbortCause) => Effect.gen(function* () { - if (publisher?.startedMessageId === undefined) return; + if (publisher === undefined) return; + yield* publisher.stop; + if (publisher.startedMessageId === undefined) return; const stored = yield* threads.entry(publisher.startedMessageId); if (Option.isSome(stored) && stored.value.entry.state === "draft") { - yield* publishFailure(stored.value, cause); + yield* publishFailure(stored.value, cause, publisher.partial); } }); @@ -379,6 +491,7 @@ export const layer = (options: Options = {}) => }, { retry: retries, ...(failed === undefined ? {} : { failed }) }, ); + yield* threads.setAttempt(threadId, retries + 1); const terminal = yield* requestLLM({ threadId, ...prepared, publisher }); if (terminal.outcome === "failed") { @@ -410,16 +523,7 @@ export const layer = (options: Options = {}) => retries += 1; failed = error.message; yield* Effect.uninterruptible(settleDraft({ _tag: "error", message: error.message })); - yield* events.publish(EventList.RetryScheduled, { - timestamp: yield* DateTime.now, - sessionId, - threadId, - attempt: retries, - maxRetries: snapshot.retry.maxRetries, - delayMs, - message: error.message, - }); - yield* Effect.sleep(Duration.millis(delayMs)); + yield* scheduleRetry(snapshot, retries, delayMs, error.message); return yield* request(); }); }), @@ -454,7 +558,11 @@ export const layer = (options: Options = {}) => committed = true; }); if (interrupted !== undefined) return yield* Effect.failCause(interrupted); - return { messageId: terminal.message.messageId, reason: terminal.reason } satisfies TurnResult; + return { + messageId: terminal.message.messageId, + reason: terminal.reason, + toolCalls: hasToolCalls(terminal.message), + } satisfies TurnResult; }); // `onError`, not `catchCause`: a catch handler never runs on an interrupted fiber, and @@ -483,8 +591,9 @@ export const layer = (options: Options = {}) => promotion: Promotion, snapshot: State.Snapshot, continuing: boolean, + carried: Crashed | undefined, ) { - return yield* runTurnAttempt(promotion, snapshot, continuing); + return yield* runTurnAttempt(promotion, snapshot, continuing, carried); }); const run = Effect.fn("Loop.run", { level: "Info" })(function* (input: { @@ -492,7 +601,18 @@ export const layer = (options: Options = {}) => readonly force: boolean; readonly recovered?: boolean; }) { - yield* healDanglingDraft(input.threadId); + // Only a dead host leaves a request marked in flight: every run that ends clears it. + const crashed = Option.getOrUndefined( + Option.flatMap(yield* threads.get(input.threadId), (row) => row.attempt), + ); + const carried: Crashed | undefined = + crashed === undefined + ? undefined + : { + attempt: crashed, + message: `The host stopped while the model was answering, on attempt ${crashed}`, + }; + yield* healDraft(input.threadId, carried?.message); const hasSteer = yield* inputs.hasPending(input.threadId, "steer"); const hasFollowUp = hasSteer ? false : yield* inputs.hasPending(input.threadId, "followUp"); @@ -510,17 +630,9 @@ export const layer = (options: Options = {}) => (input.recovered === true || input.force) && !hasSteer && hasFollowUp && - (yield* context.assemble(input.threadId).pipe( - Effect.map((assembled) => { - const leaf = - assembled.lastAssistant?.entryId === assembled.leafEntryId - ? assembled.lastAssistant.message - : undefined; - return ( - assembled.messages.length > 0 && leaf?.stopReason !== "stop" && leaf?.stopReason !== "length" - ); - }), - )); + (yield* context + .assemble(input.threadId) + .pipe(Effect.map((assembled) => assembled.messages.length > 0 && !leafAnswered(assembled)))); // A cut-off run still promotes steers that land while it loads: they join the reply being answered. let promotion: Promotion = hasSteer || cutOff ? "steer" : hasFollowUp ? "followUp" : undefined; /** Steers when there are any, else the oldest follow-up: what a run promotes first. */ @@ -532,19 +644,45 @@ export const layer = (options: Options = {}) => const drain = Effect.gen(function* () { const start = yield* state.snapshot(input.threadId); + if (carried !== undefined) { + const delayMs = Retry.delay(start.retry, carried.attempt, hostStopped(start, carried.message)); + if (delayMs === undefined) { + return yield* endRun(start.sessionId, input.threadId, "interrupted", carried.message); + } + yield* scheduleRetry(start, carried.attempt, delayMs, carried.message); + } let snapshot = start; let continuing = input.force; + let resent = carried; + let truncations = 0; // Tool results and steers continue first; a follow-up only once both are exhausted. while (true) { - const result: TurnResult = yield* runTurn(promotion, snapshot, continuing); - const toolUse = result?.reason === "toolUse"; + const result: TurnResult = yield* runTurn(promotion, snapshot, continuing, resent); + resent = undefined; + // A reply cut off at the output limit gets its tool calls back as errors to re-issue. + const truncated = result?.reason === "length" && result.toolCalls; + const toolUse = result?.reason === "toolUse" || truncated; + truncations = truncated ? truncations + 1 : 0; const completed: Hooks.Completed | undefined = result === undefined ? undefined - : { ...result, snapshot, pending: yield* peekPending(input.threadId) }; + : { + messageId: result.messageId, + reason: result.reason, + snapshot, + pending: yield* peekPending(input.threadId), + }; const decision = completed === undefined ? undefined : yield* hooks.finishTurn(completed); // Queued steers and follow-ups stay for the next run. if (decision === "end") return; + if (truncations >= TRUNCATION_LIMIT) { + return yield* endRun( + snapshot.sessionId, + input.threadId, + "truncated", + `The response hit the output token limit with tool calls ${truncations} times in a row; the run is ended`, + ); + } // A forced continuation after a finished reply sends nothing: the leaf guard in // `runTurnAttempt` ends the turn unless something was appended after it. @@ -565,7 +703,17 @@ export const layer = (options: Options = {}) => ); }); - return Runner.Service.of({ run }); + const abandon = Effect.fn("Loop.abandon")(function* (input: { + readonly threadId: ThreadSchema.ID; + readonly message: string; + }) { + const thread = yield* threads.get(input.threadId); + if (Option.isNone(thread)) return; + yield* healDraft(input.threadId, Option.isSome(thread.value.attempt) ? input.message : undefined); + yield* endRun(SessionSchema.ID.make(thread.value.sessionId), input.threadId, "crashloop", input.message); + }); + + return Runner.Service.of({ run, abandon }); }), ); diff --git a/packages/harness/src/runner/part.ts b/packages/harness/src/runner/part.ts new file mode 100644 index 00000000..0a6e7d83 --- /dev/null +++ b/packages/harness/src/runner/part.ts @@ -0,0 +1,36 @@ +/* + * @file Terminal tool-call parts the runner writes for calls that did not run to completion. + */ + +import type { Message } from "@codeworksh/aikit"; + +type OpenCall = Message.ToolCallPendingPart | Message.ToolCallRunningPart; + +export const terminalResult = (text: string) => ({ + content: [{ type: "text" as const, text }], + isError: true as const, +}); + +export const endTime = (call: OpenCall, now: number) => Math.max(call.time.end, now); + +export const abortedPart = (call: OpenCall, now: number): Message.ToolCallAbortedPart => { + const { status: _status, ...base } = call; + return { + ...base, + status: "aborted", + result: terminalResult("Tool Execution Interrupted"), + time: { ...call.time, end: endTime(call, now) }, + }; +}; + +/** The message with every tool call still pending or running aborted: an ended reply never runs them. */ +export const abortCalls = (message: Message.AssistantMessage, now: number): Message.AssistantMessage => ({ + ...message, + parts: message.parts.map((part) => + part.type === "toolCall" && (part.status === "pending" || part.status === "running") + ? abortedPart(part, now) + : part, + ), +}); + +export * as RunnerPart from "./part.ts"; diff --git a/packages/harness/src/runner/run.ts b/packages/harness/src/runner/run.ts index 67877973..51f95907 100644 --- a/packages/harness/src/runner/run.ts +++ b/packages/harness/src/runner/run.ts @@ -193,6 +193,11 @@ export interface Interface { /** Resuming a run a previous process left unfinished: its cut-off reply is answered first. */ readonly recovered?: boolean; }) => Effect.Effect; + /** + * Ends a run the server no longer resumes: settles what the dead process left of its reply, + * records `session.run.failed` with `message`, and leaves the thread idle, its queue kept. + */ + readonly abandon: (input: { readonly threadId: ThreadSchema.ID; readonly message: string }) => Effect.Effect; } export class Service extends Context.Service()("@codeworksh/harness/runner/run/Service") {} diff --git a/packages/harness/src/session/projector.ts b/packages/harness/src/session/projector.ts index 94e9e760..bdf2bbf3 100644 --- a/packages/harness/src/session/projector.ts +++ b/packages/harness/src/session/projector.ts @@ -174,6 +174,24 @@ export const layer = Layer.effectDiscard( }), ); + yield* events.project(EventList.LLMProgress, (event) => + Effect.gen(function* () { + const changes = yield* Effect.forEach(event.data.changes, (change) => + change._tag === "append" + ? Effect.succeed(change) + : ContextCodec.encodePart(change.part).pipe( + Effect.orDie, + Effect.map((part): Thread.ProgressChange => ({ + _tag: "set", + partIndex: change.partIndex, + part, + })), + ), + ); + yield* threads.progress({ id: event.data.messageId, threadId: event.data.threadId, changes }); + }), + ); + yield* events.project(EventList.LLMEnded, (event) => Effect.gen(function* () { const encoded = yield* encodeAssistant(event.data); @@ -183,6 +201,7 @@ export const layer = Layer.effectDiscard( data: encoded.data, parts: encoded.parts, }); + yield* threads.setAttempt(event.data.threadId, null); }), ); @@ -194,6 +213,7 @@ export const layer = Layer.effectDiscard( threadId: event.data.threadId, seq: yield* seqOf(event), data: encoded.data, + parts: encoded.parts, }); }), ); @@ -255,7 +275,8 @@ export const layer = Layer.effectDiscard( threadId: event.data.threadId, seq: yield* seqOf(event), }); - yield* threads.resetRestarts(event.data.threadId); + if (event.data.recovered !== true) yield* threads.resetRestarts(event.data.threadId); + yield* threads.setAttempt(event.data.threadId, null); }), ); }), diff --git a/packages/harness/src/thread/thread.ts b/packages/harness/src/thread/thread.ts index ce5bf81b..6c6ada30 100644 --- a/packages/harness/src/thread/thread.ts +++ b/packages/harness/src/thread/thread.ts @@ -98,7 +98,7 @@ export interface ReplaceAssistant { readonly parts: ReadonlyArray; } -/** Ends a draft assistant: committed once every tool part is settled, or aborted (parts dropped). */ +/** Ends a draft assistant: committed once every tool part is settled, or aborted. */ export interface EndAssistant { readonly id: string; readonly threadId: ThreadSchema.ID; @@ -106,6 +106,17 @@ export interface EndAssistant { readonly seq: number; } +/** One part change of a streaming draft: `append` extends a text or thinking part, `set` writes one whole. */ +export type ProgressChange = + | { readonly _tag: "append"; readonly partIndex: number; readonly text: string } + | { readonly _tag: "set"; readonly partIndex: number; readonly part: AppendPart }; + +export interface ProgressAssistant { + readonly id: string; + readonly threadId: ThreadSchema.ID; + readonly changes: ReadonlyArray; +} + export interface ForkInput { readonly threadId: ThreadSchema.ID; readonly entryId: string; @@ -158,12 +169,14 @@ export interface Interface { ) => Effect.Effect; /** Projector-only: replace the payload of an existing draft assistant in place. */ readonly replaceAssistant: (input: ReplaceAssistant) => Effect.Effect; + /** Projector-only: apply a streaming checkpoint to a draft assistant's parts. */ + readonly progress: (input: ProgressAssistant) => Effect.Effect; /** Projector-only: settle one pending/running tool part on a draft assistant. */ readonly settleToolCall: (input: SettleToolCall) => Effect.Effect; /** Projector-only: commit a draft after every tool part is terminal. */ readonly commit: (input: EndAssistant) => Effect.Effect; - /** Projector-only: turn a draft into a partless aborted tombstone. */ - readonly abort: (input: EndAssistant & { readonly data: string }) => Effect.Effect; + /** Projector-only: end a draft as aborted with the payload and parts given. */ + readonly abort: (input: ReplaceAssistant & EndAssistant) => Effect.Effect; /** Cut a new thread off `threadId`'s line at a committed entry. */ readonly fork: (input: ForkInput) => Effect.Effect; /** Projector of `session.thread.forked`: the thread and its copied documents, at the event's seq. */ @@ -191,6 +204,13 @@ export interface Interface { readonly claimRestart: (threadId: ThreadSchema.ID) => Effect.Effect>; /** A committed turn is progress: the restart count starts over. Runs inside `TurnEnded`'s commit. */ readonly resetRestarts: (threadId: ThreadSchema.ID) => Effect.Effect; + /** Records the 1-based provider request now in flight, or none (null) once its response ended. */ + readonly setAttempt: (threadId: ThreadSchema.ID, attempt: number | null) => Effect.Effect; + /** + * Leaves a run that is not resumed idle: no mark, no restarts, no request in flight, whatever is + * queued. Its queued prompts wait for the next prompt to wake the thread. + */ + readonly idle: (threadId: ThreadSchema.ID) => Effect.Effect; /** Root threads with a run left unfinished, oldest first. */ readonly unfinished: Effect.Effect>; /** Set or clear (null) the bookmark label on an entry of the thread. */ @@ -211,6 +231,7 @@ const decodeEnvelopeUsage = Schema.decodeUnknownEffect(SessionSchema.AssistantEn const decodeMessageEnvelopeIdentity = Schema.decodeUnknownEffect(SessionSchema.MessageEnvelopeIdentity); const decodeCompactionData = Schema.decodeUnknownEffect(SessionSchema.CompactionData); const decodeJsonObject = Schema.decodeUnknownEffect(SessionSchema.JsonObject); +const encodeJsonObject = Schema.encodeEffect(SessionSchema.JsonObject); const decodeUserText = Schema.decodeUnknownOption( Schema.fromJsonString(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })), ); @@ -395,30 +416,38 @@ export const layer = Layer.effect( originCallId: origin.kind === "subagent" ? Option.some(origin.callId) : Option.none(), title: Option.fromUndefinedOr(input.title), execution: Option.none(), + attempt: Option.none(), createdSeq: input.seq, }); yield* insertThread(row); }, Effect.orDie); + const writePart = Effect.fnUntraced(function* ( + entryId: string, + threadId: ThreadSchema.ID, + partIndex: number, + part: AppendPart, + ) { + const partRow = yield* EntryPartRow.insert.makeEffect({ + id: part.id ?? uuidv7(), + entryId, + threadId, + partIndex, + type: part.type, + status: Option.fromUndefinedOr(part.status), + callId: Option.fromUndefinedOr(part.callId), + toolName: Option.fromUndefinedOr(part.toolName), + data: part.data, + }); + yield* insertPart(partRow); + }); + const writeParts = Effect.fnUntraced(function* ( entryId: string, threadId: ThreadSchema.ID, parts: ReadonlyArray, ) { - for (const [partIndex, part] of parts.entries()) { - const partRow = yield* EntryPartRow.insert.makeEffect({ - id: part.id ?? uuidv7(), - entryId, - threadId, - partIndex, - type: part.type, - status: Option.fromUndefinedOr(part.status), - callId: Option.fromUndefinedOr(part.callId), - toolName: Option.fromUndefinedOr(part.toolName), - data: part.data, - }); - yield* insertPart(partRow); - } + for (const [partIndex, part] of parts.entries()) yield* writePart(entryId, threadId, partIndex, part); }); // Usage lives on the thread and moves with the envelope stored; the session's last activity @@ -605,6 +634,38 @@ export const layer = Layer.effect( ); }, Effect.orDie); + const progress = Effect.fn("Thread.progress")(function* (input: ProgressAssistant) { + yield* sql.withTransaction( + Effect.gen(function* () { + yield* requireDraft(input.id, input.threadId, "progress"); + const now = yield* epochNow; + for (const change of input.changes) { + if (change._tag === "set") { + yield* sql`DELETE FROM part WHERE entry_id = ${input.id} AND part_index = ${change.partIndex}`; + yield* writePart(input.id, input.threadId, change.partIndex, change.part); + continue; + } + const [row] = yield* sql<{ readonly data: string }>` + SELECT data FROM part WHERE entry_id = ${input.id} AND part_index = ${change.partIndex} + `; + const data = row === undefined ? undefined : yield* decodeJsonObject(row.data); + const field = + data?.["type"] === "text" ? "text" : data?.["type"] === "thinking" ? "thinking" : undefined; + const text = field === undefined ? undefined : data?.[field]; + if (data === undefined || field === undefined || typeof text !== "string") { + return yield* Effect.die(`progress cannot append to part ${change.partIndex} of ${input.id}`); + } + const next = yield* encodeJsonObject({ ...data, [field]: text + change.text }); + yield* sql` + UPDATE part SET data = ${next}, updated_at = ${now} + WHERE entry_id = ${input.id} AND part_index = ${change.partIndex} + `; + } + yield* sql`UPDATE entry SET updated_at = ${now} WHERE id = ${input.id}`; + }), + ); + }, Effect.orDie); + const settleToolCall = Effect.fn("Thread.settleToolCall")(function* (input: SettleToolCall) { const data = yield* decodeJsonObject(input.data).pipe(Effect.orDie); if (data["type"] !== "toolCall" || data["callID"] !== input.callId || data["status"] !== input.status) { @@ -643,13 +704,14 @@ export const layer = Layer.effect( yield* sql`UPDATE thread SET updated_at = ${now} WHERE id = ${input.threadId}`; }, Effect.orDie); - const abort = Effect.fn("Thread.abort")(function* (input: EndAssistant & { readonly data: string }) { + const abort = Effect.fn("Thread.abort")(function* (input: ReplaceAssistant & EndAssistant) { yield* sql.withTransaction( Effect.gen(function* () { const { entry, thread } = yield* requireDraft(input.id, input.threadId, "abort"); const delta = yield* usageDelta(entry.data, input.data); const now = yield* epochNow; yield* sql`DELETE FROM part WHERE entry_id = ${input.id}`; + yield* writeParts(input.id, input.threadId, input.parts); yield* sql` UPDATE entry SET state = 'aborted', committed_seq = ${input.seq}, data = ${input.data}, updated_at = ${now} @@ -778,10 +840,12 @@ export const layer = Layer.effect( end: "finished" | "stopped" | "closed", stoppedAt?: number, ) { + // A run that ended in this process has no request in flight: only a dead host leaves one. if (end === "closed") { - yield* sql`UPDATE thread SET execution = 'closed' WHERE id = ${threadId}`; + yield* sql`UPDATE thread SET execution = 'closed', attempt = NULL WHERE id = ${threadId}`; return; } + yield* sql`UPDATE thread SET attempt = NULL WHERE id = ${threadId}`; // Pending prompts that keep the mark: any for a finished run; for a stop, only those sent // after it, since what was queued before it stays parked by the user's choice. const after = end === "stopped" ? (stoppedAt ?? Number.MAX_SAFE_INTEGER) : -1; @@ -809,6 +873,14 @@ export const layer = Layer.effect( yield* sql`UPDATE thread SET restarts = 0 WHERE id = ${threadId} AND restarts > 0`; }, Effect.orDie); + const setAttempt = Effect.fn("Thread.setAttempt")(function* (threadId: ThreadSchema.ID, attempt: number | null) { + yield* sql`UPDATE thread SET attempt = ${attempt} WHERE id = ${threadId}`; + }, Effect.orDie); + + const idle = Effect.fn("Thread.idle")(function* (threadId: ThreadSchema.ID) { + yield* sql`UPDATE thread SET execution = NULL, restarts = 0, attempt = NULL WHERE id = ${threadId}`; + }, Effect.orDie); + const findUnfinished = SqlSchema.findAll({ Request: Schema.Void, Result: ThreadRow, @@ -845,6 +917,7 @@ export const layer = Layer.effect( insert, append, replaceAssistant, + progress, settleToolCall, commit, abort, @@ -855,6 +928,8 @@ export const layer = Layer.effect( endExecution, claimRestart, resetRestarts, + setAttempt, + idle, unfinished, setLabel, }); diff --git a/packages/harness/test/__artifacts__/runner.retry.json b/packages/harness/test/__artifacts__/runner.retry.json index 56dbd33f..5abafd7f 100644 --- a/packages/harness/test/__artifacts__/runner.retry.json +++ b/packages/harness/test/__artifacts__/runner.retry.json @@ -87,7 +87,7 @@ "id": "assistant_2", "type": "assistant", "state": "aborted", - "parts": 0 + "parts": 1 }, { "id": "assistant_3", diff --git a/packages/harness/test/control.prompt.test.ts b/packages/harness/test/control.prompt.test.ts index b1feba1c..09da03b5 100644 --- a/packages/harness/test/control.prompt.test.ts +++ b/packages/harness/test/control.prompt.test.ts @@ -395,6 +395,7 @@ const watched = Control.layer.pipe( active: Effect.succeed(new Set()), resume: () => Effect.void, recover: () => Effect.void, + abandon: () => Effect.void, wake: (threadId) => Effect.sync(() => wakes.push(threadId)), interrupt: () => Effect.succeed(false), awaitIdle: () => Effect.void, diff --git a/packages/harness/test/plugin.foreign.test.ts b/packages/harness/test/plugin.foreign.test.ts index 17805ebd..37897d11 100644 --- a/packages/harness/test/plugin.foreign.test.ts +++ b/packages/harness/test/plugin.foreign.test.ts @@ -1,6 +1,6 @@ import "./utils/env.ts"; import type { Message } from "@codeworksh/aikit"; -import { Effect, Schema } from "effect"; +import { Effect, Option, Schema } from "effect"; import { glob, readFile } from "node:fs/promises"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -84,7 +84,10 @@ describe("plugins built against another Effect instance", () => { const assistant = path.at(-1); expect(assistant?.entry.type).toBe("assistant"); expect(assistant?.entry.state).toBe("aborted"); - expect(assistant?.parts).toEqual([]); + // The reply's tool call is kept, aborted: it never ran. + expect(assistant?.parts.map((part) => [part.toolName, part.status])).toEqual([ + [Option.some("foreign_echo"), Option.some("aborted")], + ]); })); it("pins one Effect version, declared and resolved alike, across the workspace", async () => { diff --git a/packages/harness/test/runner.llm.live.test.ts b/packages/harness/test/runner.llm.live.test.ts index 9a8bcc24..2b43744d 100644 --- a/packages/harness/test/runner.llm.live.test.ts +++ b/packages/harness/test/runner.llm.live.test.ts @@ -114,7 +114,7 @@ describe.each(LIVE)("runner LLM — live $name", (live) => { ); liveIt( - "bridges Effect interruption to a partless aborted tombstone", + "bridges Effect interruption to an aborted entry keeping what streamed", Effect.gen(function* () { const { threads, sessionId, threadId } = yield* setup; const events = yield* Event.Service; @@ -149,8 +149,8 @@ describe.each(LIVE)("runner LLM — live $name", (live) => { const stored = yield* ContextCodec.decodeMessage(path[0]!); if (stored.role !== "assistant") return yield* Effect.die(`stored message is ${stored.role}, not assistant`); expect(stored.stopReason).toBe("aborted"); - expect(stored.parts).toEqual([]); expect(["thinking", "text"]).toContain(interruptedPart); + expect(stored.parts.map((part) => part.type)).toContain(interruptedPart); }), { timeout: 180_000 }, ); diff --git a/packages/harness/test/runner.loop.test.ts b/packages/harness/test/runner.loop.test.ts index a5c98be1..a1e56edf 100644 --- a/packages/harness/test/runner.loop.test.ts +++ b/packages/harness/test/runner.loop.test.ts @@ -10,7 +10,6 @@ import { SqlClient, SqlSchema } from "effect/sql"; import { describe, expect } from "vite-plus/test"; import { Context } from "../src/context/context.ts"; import { Control } from "../src/control.ts"; -import { Session as SessionApi } from "../src/effect/session.ts"; import { Database } from "../src/db/db.ts"; import { InboxRow } from "../src/db/schema.sql.ts"; import { Event } from "../src/event/event.ts"; @@ -556,7 +555,7 @@ describe("runner loop — provider interruption", () => { const { live: it } = testEffect(runtime({ open })); it( - "aborts the whole draft as a partless tombstone, then propagates interruption", + "aborts the draft keeping what streamed, then propagates interruption", Effect.gen(function* () { const execution = yield* RunnerExecution.Service; const events = yield* Event.Service; @@ -581,7 +580,7 @@ describe("runner loop — provider interruption", () => { expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); expect(path[2]!.entry.state).toBe("aborted"); expect(JSON.parse(path[2]!.entry.data)).toMatchObject({ stopReason: "aborted" }); - expect(path[2]!.parts).toEqual([]); + expect(path[2]!.parts.map((part) => JSON.parse(part.data))).toEqual([{ type: "text", text: "partial" }]); expect(Array.from(yield* execution.active)).toEqual([]); }), { timeout: 10_000 }, @@ -637,7 +636,7 @@ describe("runner loop — provider failure", () => { const path = yield* threads.line(threadId); expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); expect(path[2]!.entry.state).toBe("aborted"); - expect(path[2]!.parts).toEqual([]); + expect(path[2]!.parts.map((part) => JSON.parse(part.data))).toEqual([{ type: "text", text: "partial" }]); expect(JSON.parse(path[2]!.entry.data)).toMatchObject({ stopReason: "error", errorMessage: "provider failed", @@ -1305,31 +1304,41 @@ describe("runner loop — resume after restart", () => { ); it( - "parks a session at the restart limit, and its next prompt runs it again", + "ends a run at the restart limit visibly, and its next prompt runs it again", Effect.gen(function* () { yield* provider.reset; const control = yield* Control.Service; + const events = yield* Event.Service; + const threads = yield* Thread.Service; const sql = yield* SqlClient.SqlClient; - const { sessionId, threadId } = yield* seedSession(); + const { threadId } = yield* seedSession(); + const failures: Array = []; + yield* events.listen((event) => + Schema.is(EventList.RunFailed)(event) + ? Effect.sync(() => { + failures.push(event.data); + }) + : Effect.void, + ); yield* crashedMidReply(threadId, "A"); yield* sql`UPDATE thread SET restarts = ${Control.RESTART_LIMIT - 1} WHERE id = ${threadId}`; yield* control.resumeUnfinished; expect(contexts).toEqual([]); - expect(yield* stateOf(threadId)).toEqual({ execution: "running", restarts: Control.RESTART_LIMIT }); - expect((yield* (yield* SessionApi.get(sessionId).pipe(Effect.map(Option.getOrThrow))).main.info).parked).toBe( - true, - ); + expect(yield* stateOf(threadId)).toEqual({ execution: null, restarts: 0 }); + expect(failures.map((failure) => [failure.reason, failure.message])).toEqual([ + ["crashloop", `The run was interrupted ${Control.RESTART_LIMIT} times in a row; it is not resumed again`], + ]); + const [, healed] = yield* threads.line(threadId); + expect(healed!.entry.state).toBe("aborted"); + expect(healed!.parts.map((part) => JSON.parse(part.data).text)).toEqual(["half a reply"]); yield* send(threadId, "B"); yield* settle(threadId); expect(contexts.map(texts)).toEqual([["A", "B"]]); expect(yield* stateOf(threadId)).toEqual({ execution: null, restarts: 0 }); - expect((yield* (yield* SessionApi.get(sessionId).pipe(Effect.map(Option.getOrThrow))).main.info).parked).toBe( - false, - ); }), ); diff --git a/packages/plugin/src/event.ts b/packages/plugin/src/event.ts index bf4e26a2..0a42f2a7 100644 --- a/packages/plugin/src/event.ts +++ b/packages/plugin/src/event.ts @@ -3,17 +3,19 @@ import { Effect, Schema, type SchemaAST, SchemaGetter, SchemaIssue } from "effec import { uuidv7 } from "uuidv7"; import { isAikitAssistantMessage, + isAikitAssistantPart, isAikitToolCallTerminalPart, optional, validateAikitAssistantMessage, + validateAikitAssistantPart, validateAikitToolCallTerminalPart, withStatics, } from "./schema.ts"; -type AikitAssistantPart = Message.AssistantMessage["parts"][number]; +type AssistantPart = Message.AssistantMessage["parts"][number]; /** Undeclared fields used only while aikit is assembling a streamed part. */ -const aikitTransientFields: Readonly>> = { +const aikitTransientFields: Readonly>> = { text: ["streamId"], image: [], thinking: ["streamId"], @@ -27,9 +29,12 @@ const omit = (value: T, keys: ReadonlyArray): T => { return copy; }; +const canonicalizeAikitAssistantPart = (part: AssistantPart): AssistantPart => + omit(part, aikitTransientFields[part.type]); + const canonicalizeAikitAssistantMessage = (message: Message.AssistantMessage): Message.AssistantMessage => ({ ...message, - parts: message.parts.map((part) => omit(part, aikitTransientFields[part.type])), + parts: message.parts.map(canonicalizeAikitAssistantPart), }); const reasonOf = (cause: unknown): string => (cause instanceof Error ? cause.message : String(cause)); @@ -56,6 +61,25 @@ export const AikitAssistantMessage = Schema.Unknown.pipe( ); export type AikitAssistantMessage = typeof AikitAssistantMessage.Type; +const decodeAikitAssistantPart = (value: unknown, options: SchemaAST.ParseOptions) => + Effect.try({ + try: () => canonicalizeAikitAssistantPart(validateAikitAssistantPart(value, "aikit assistant part")), + catch: (cause) => new SchemaIssue.InvalidValue({ message: reasonOf(cause) }, value, options), + }); + +const aikitAssistantPartDeclared = Schema.declare(isAikitAssistantPart, { + expected: "aikit assistant part", +}); + +/** One part of an aikit assistant message, streaming-only fields stripped like {@link AikitAssistantMessage}. */ +export const AikitAssistantPart = Schema.Unknown.pipe( + Schema.decodeTo(aikitAssistantPartDeclared, { + decode: SchemaGetter.transformEffect(decodeAikitAssistantPart), + encode: SchemaGetter.transformEffect(decodeAikitAssistantPart), + }), +); +export type AikitAssistantPart = typeof AikitAssistantPart.Type; + const decodeAikitToolCallTerminalPart = (value: unknown, options: SchemaAST.ParseOptions) => Effect.try({ try: () => validateAikitToolCallTerminalPart(value, "aikit terminal toolcall part"), diff --git a/packages/plugin/src/schema.ts b/packages/plugin/src/schema.ts index 9d21aa6e..31f70eac 100644 --- a/packages/plugin/src/schema.ts +++ b/packages/plugin/src/schema.ts @@ -41,6 +41,9 @@ export const validateAikitUserMessage = (value: unknown, label: string): Message export const validateAikitAssistantMessage = (value: unknown, label: string): Message.AssistantMessage => validateAikitSchema(Message.AssistantMessageSchema, value, label); +export const validateAikitAssistantPart = (value: unknown, label: string): Message.AssistantMessage["parts"][number] => + validateAikitSchema(Message.AssistantMessageSchema.properties.parts.items, value, label); + export const validateAikitToolCallTerminalPart = (value: unknown, label: string): Message.ToolCallTerminalPart => validateAikitSchema(Message.ToolCallTerminalPartSchema, value, label); @@ -52,6 +55,9 @@ export const isAikitModelInfo = (value: unknown): value is Model.Info => aikitVa export const isAikitAssistantMessage = (value: unknown): value is Message.AssistantMessage => aikitValidatorFor(Message.AssistantMessageSchema).Check(value); +export const isAikitAssistantPart = (value: unknown): value is Message.AssistantMessage["parts"][number] => + aikitValidatorFor(Message.AssistantMessageSchema.properties.parts.items).Check(value); + export const isAikitToolCallTerminalPart = (value: unknown): value is Message.ToolCallTerminalPart => aikitValidatorFor(Message.ToolCallTerminalPartSchema).Check(value);