diff --git a/packages/codework/src/cli/cmd/handlers/run.ts b/packages/codework/src/cli/cmd/handlers/run.ts index 09126a1d..ce5ecd6b 100644 --- a/packages/codework/src/cli/cmd/handlers/run.ts +++ b/packages/codework/src/cli/cmd/handlers/run.ts @@ -21,6 +21,7 @@ interface RenderState { readonly usage: UsageSummary; readonly textSeen: boolean; readonly textEndsWithNewline: boolean; + readonly reason?: EventList.LLMEnded["data"]["reason"]; } const initialRenderState: RenderState = { @@ -47,7 +48,11 @@ const render = (state: Ref.Ref, ended?: Queue.Queue) => return; } if (isLLMEnded(event)) { - yield* Ref.update(state, (current) => ({ ...current, usage: addUsage(current.usage, event.data.message) })); + yield* Ref.update(state, (current) => ({ + ...current, + usage: addUsage(current.usage, event.data.message), + reason: event.data.reason, + })); return; } if (ended !== undefined && isTurnEnded(event)) { @@ -59,6 +64,12 @@ const finish = Effect.fn("CLI.finish")(function* (state: Ref.Ref, c const rendered = yield* Ref.get(state); if (rendered.textSeen && !rendered.textEndsWithNewline) yield* writeOut("\n"); yield* writeError(usage(rendered.usage, columns)); + if (rendered.reason === "length") { + yield* writeError("Response was truncated: the model reached its output token limit.\n"); + yield* Effect.sync(() => { + process.exitCode = 1; + }); + } }); const awaitMessage = Effect.fn("CLI.awaitMessage")(function* (ended: Queue.Queue, messageId: string) { diff --git a/packages/codework/src/server/contract.ts b/packages/codework/src/server/contract.ts index d2a2295e..d672f2f0 100644 --- a/packages/codework/src/server/contract.ts +++ b/packages/codework/src/server/contract.ts @@ -93,8 +93,6 @@ export const ThreadInfo = Schema.Struct({ foundation: optional(Schema.String), /** Where it runs instead of the session's space and directory. */ placement: optional(Schema.Struct({ spaceId: Schema.String, directory: Session.AbsolutePath })), - /** Its unfinished run was cut off by restarts too often to resume on its own; the next prompt runs it. */ - parked: optional(Schema.Boolean), usage: ThreadUsage, session: SessionInfo, }); diff --git a/packages/codework/src/server/handlers.ts b/packages/codework/src/server/handlers.ts index 0a6e17c2..6d18fef3 100644 --- a/packages/codework/src/server/handlers.ts +++ b/packages/codework/src/server/handlers.ts @@ -65,7 +65,6 @@ const toThreadInfo = (info: Session.ThreadInfo): ThreadInfo => ({ ...(info.thinkingLevel === undefined ? {} : { thinkingLevel: info.thinkingLevel }), ...(info.foundation === undefined ? {} : { foundation: info.foundation }), ...(info.placement === undefined ? {} : { placement: info.placement }), - ...(info.parked ? { parked: true } : {}), usage: info.usage, session: toSessionInfo(info.session), }); diff --git a/packages/codework/test/__artifacts__/cli.truncated.json b/packages/codework/test/__artifacts__/cli.truncated.json new file mode 100644 index 00000000..261e31ae --- /dev/null +++ b/packages/codework/test/__artifacts__/cli.truncated.json @@ -0,0 +1,5 @@ +{ + "status": 1, + "stdout": "The answer starts here and\n", + "warning": "Response was truncated: the model reached its output token limit.\n" +} diff --git a/packages/codework/test/__artifacts__/recovery.crashloop.json b/packages/codework/test/__artifacts__/recovery.crashloop.json new file mode 100644 index 00000000..1b909d09 --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.crashloop.json @@ -0,0 +1,191 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Keep crashing." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Keep crashing." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 1", + "parts": [] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 2", + "parts": [] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 3", + "parts": [] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 4", + "parts": [] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The run was interrupted 5 times in a row; it is not resumed again", + "parts": [] + }, + { + "type": "session.run.failed", + "reason": "crashloop", + "message": "The run was interrupted 5 times in a row; it is not resumed again" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Try again." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Try again." + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "stop", + "parts": [ + { + "text": "Back to normal." + } + ] + }, + { + "type": "session.turn.ended" + } + ], + "history": [ + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "Keep crashing." + } + ] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + }, + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "Try again." + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "text": "Back to normal." + } + ] + } + ], + "requests": [ + [ + { + "role": "user", + "text": "Keep crashing." + } + ], + [ + { + "role": "user", + "text": "Keep crashing." + } + ], + [ + { + "role": "user", + "text": "Keep crashing." + } + ], + [ + { + "role": "user", + "text": "Keep crashing." + } + ], + [ + { + "role": "user", + "text": "Keep crashing." + } + ], + [ + { + "role": "user", + "text": "Keep crashing." + }, + { + "role": "user", + "text": "Try again." + } + ] + ] +} diff --git a/packages/codework/test/__artifacts__/recovery.interrupted.json b/packages/codework/test/__artifacts__/recovery.interrupted.json new file mode 100644 index 00000000..41fc3566 --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.interrupted.json @@ -0,0 +1,89 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "First." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "First." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.prompt.admitted", + "delivery": "followUp", + "text": "Then this." + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 1", + "parts": [] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 2", + "parts": [] + }, + { + "type": "session.run.failed", + "reason": "interrupted", + "message": "The host stopped while the model was answering, on attempt 2" + } + ], + "history": [ + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "First." + } + ] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [] + } + ], + "queue": [ + { + "delivery": "followUp", + "text": "Then this." + } + ], + "requests": [ + [ + { + "role": "user", + "text": "First." + } + ], + [ + { + "role": "user", + "text": "First." + } + ] + ] +} diff --git a/packages/codework/test/__artifacts__/recovery.mid-stream.json b/packages/codework/test/__artifacts__/recovery.mid-stream.json new file mode 100644 index 00000000..f40aebc2 --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.mid-stream.json @@ -0,0 +1,110 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Answer at length." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Answer at length." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.progress", + "changes": [ + { + "_tag": "set", + "partIndex": 0, + "part": { + "text": "Partial answer, streamed before the host died." + } + } + ] + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 1", + "parts": [ + { + "text": "Partial answer, streamed before the host died." + } + ] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "stop", + "parts": [ + { + "text": "The full answer." + } + ] + }, + { + "type": "session.turn.ended" + } + ], + "live": [ + { + "type": "session.retry.finished", + "attempt": 1, + "success": true + } + ], + "history": [ + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "Answer at length." + } + ] + }, + { + "role": "assistant", + "state": "aborted", + "parts": [ + { + "text": "Partial answer, streamed before the host died." + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "text": "The full answer." + } + ] + } + ], + "requests": [ + [ + { + "role": "user", + "text": "Answer at length." + } + ], + [ + { + "role": "user", + "text": "Answer at length." + } + ] + ] +} diff --git a/packages/codework/test/__artifacts__/recovery.tool.json b/packages/codework/test/__artifacts__/recovery.tool.json new file mode 100644 index 00000000..761ff475 --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.tool.json @@ -0,0 +1,115 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Run the slow command." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Run the slow command." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "stop", + "parts": [ + { + "text": "Recovered after 1 result." + } + ] + }, + { + "type": "session.turn.ended" + } + ], + "history": [ + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "Run the slow command." + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "text": "Recovered after 1 result." + } + ] + } + ], + "requests": [ + [ + { + "role": "user", + "text": "Run the slow command." + } + ], + [ + { + "role": "user", + "text": "Run the slow command." + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + } + ] + ], + "ran": "ran\n" +} diff --git a/packages/codework/test/__artifacts__/recovery.toolcrash.json b/packages/codework/test/__artifacts__/recovery.toolcrash.json new file mode 100644 index 00000000..b3a15fca --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.toolcrash.json @@ -0,0 +1,344 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Run the slow command." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Run the slow command." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.run.failed", + "reason": "crashloop", + "message": "The run was interrupted 5 times in a row; it is not resumed again" + } + ], + "history": [ + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "Run the slow command." + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "aborted", + "result": "Tool Execution Interrupted" + } + ] + } + ], + "requests": [ + [ + { + "role": "user", + "text": "Run the slow command." + } + ], + [ + { + "role": "user", + "text": "Run the slow command." + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + } + ], + [ + { + "role": "user", + "text": "Run the slow command." + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + } + ], + [ + { + "role": "user", + "text": "Run the slow command." + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + } + ], + [ + { + "role": "user", + "text": "Run the slow command." + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool Execution Interrupted" + } + ] + ], + "ran": "ran\nran\nran\nran\nran\n" +} diff --git a/packages/codework/test/__artifacts__/recovery.tools.json b/packages/codework/test/__artifacts__/recovery.tools.json new file mode 100644 index 00000000..a46fb4ff --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.tools.json @@ -0,0 +1,136 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Find the echo tool." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Find the echo tool." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "toolUse", + "parts": [ + { + "tool": "toolsearch", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "toolsearch", + "status": "completed", + "result": "Loaded 1 tool. They are available from your next call:\n- mcp__fixture__echo: Echo text back, with the directory the server runs in." + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.failed", + "reason": "error", + "errorMessage": "The host stopped while the model was answering, on attempt 1", + "parts": [] + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "stop", + "parts": [ + { + "text": "Resumed with the loaded tool." + } + ] + }, + { + "type": "session.turn.ended" + } + ], + "tools": [ + [ + "bash", + "edit", + "read", + "toolsearch", + "write" + ], + [ + "bash", + "edit", + "mcp__fixture__echo", + "read", + "toolsearch", + "write" + ], + [ + "bash", + "edit", + "mcp__fixture__echo", + "read", + "toolsearch", + "write" + ] + ], + "requests": [ + [ + { + "role": "user", + "text": "Find the echo tool." + } + ], + [ + { + "role": "user", + "text": "Find the echo tool." + }, + { + "role": "assistant", + "calls": [ + "toolsearch" + ] + }, + { + "role": "tool", + "text": "Loaded 1 tool. They are available from your next call:\n- mcp__fixture__echo: Echo text back, with the directory the server runs in." + } + ], + [ + { + "role": "user", + "text": "Find the echo tool." + }, + { + "role": "assistant", + "calls": [ + "toolsearch" + ] + }, + { + "role": "tool", + "text": "Loaded 1 tool. They are available from your next call:\n- mcp__fixture__echo: Echo text back, with the directory the server runs in." + } + ] + ] +} diff --git a/packages/codework/test/__artifacts__/recovery.truncated.json b/packages/codework/test/__artifacts__/recovery.truncated.json new file mode 100644 index 00000000..a10d664e --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.truncated.json @@ -0,0 +1,130 @@ +{ + "log": [ + { + "type": "session.created" + }, + { + "type": "session.prompt.admitted", + "delivery": "steer", + "text": "Run a long command." + }, + { + "type": "session.prompt.promoted", + "delivery": "steer", + "text": "Run a long command." + }, + { + "type": "session.thread.system.changed" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "length", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "skipped", + "result": "Tool call \"bash\" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments." + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.llm.started" + }, + { + "type": "session.llm.ended", + "reason": "length", + "parts": [ + { + "tool": "bash", + "status": "pending" + } + ] + }, + { + "type": "session.tool.settled", + "part": { + "tool": "bash", + "status": "skipped", + "result": "Tool call \"bash\" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments." + } + }, + { + "type": "session.turn.ended" + }, + { + "type": "session.run.failed", + "reason": "truncated", + "message": "The response hit the output token limit with tool calls 2 times in a row; the run is ended" + } + ], + "history": [ + { + "role": "user", + "state": "committed", + "parts": [ + { + "text": "Run a long command." + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "skipped", + "result": "Tool call \"bash\" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments." + } + ] + }, + { + "role": "assistant", + "state": "committed", + "parts": [ + { + "tool": "bash", + "status": "skipped", + "result": "Tool call \"bash\" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments." + } + ] + } + ], + "requests": [ + [ + { + "role": "user", + "text": "Run a long command." + } + ], + [ + { + "role": "user", + "text": "Run a long command." + }, + { + "role": "assistant", + "calls": [ + "bash" + ] + }, + { + "role": "tool", + "text": "Tool call \"bash\" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments." + } + ] + ] +} diff --git a/packages/codework/test/cli.test.ts b/packages/codework/test/cli.test.ts index 4bd2c10b..5d2f3bd3 100644 --- a/packages/codework/test/cli.test.ts +++ b/packages/codework/test/cli.test.ts @@ -5,7 +5,8 @@ import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import { describe, expect, it } from "vite-plus/test"; +import { describe, expect, it, onTestFinished } from "vite-plus/test"; +import { runCli, truncating, writeSettings } from "./fixtures/cli.ts"; const cli = fileURLToPath(new URL("../src/index.ts", import.meta.url)); const models = fileURLToPath(new URL("../../../models.gen.json", import.meta.url)); @@ -459,6 +460,26 @@ describe("codework CLI", () => { } }); + it("prints a response cut off at the output token limit, warns, and exits 1", async () => { + const home = mkdtempSync(join(tmpdir(), "codework-cli-truncated-")); + onTestFinished(() => rmSync(home, { recursive: true, force: true })); + const fake = await truncating("The answer starts here and"); + writeSettings(home, fake.url); + + const result = await runCli( + ["--home", home, "--database", ":memory:", "--user-config-dir", home, "run", "Answer at length."], + { cwd: home, env: { ...process.env, HOME: home, CODEWORK_MODELS_FILE: models } }, + ); + + expect(result.status, result.stderr).toBe(1); + expect(result.stdout, result.stderr).toBe("The answer starts here and\n"); + const warning = "Response was truncated: the model reached its output token limit.\n"; + expect(result.stderr.endsWith(warning), result.stderr).toBe(true); + await expect( + `${JSON.stringify({ status: result.status, stdout: result.stdout, warning }, null, "\t")}\n`, + ).toMatchFileSnapshot("./__artifacts__/cli.truncated.json"); + }, 60_000); + it("reports a failed refresh", () => { const result = runIsolated( { CODEWORK_MODELS_FILE: undefined, CODEWORK_CATALOG_URL: "http://127.0.0.1:1/v1/models.json" }, diff --git a/packages/codework/test/fixtures/cli.ts b/packages/codework/test/fixtures/cli.ts index 9b2b9538..a4e922d7 100644 --- a/packages/codework/test/fixtures/cli.ts +++ b/packages/codework/test/fixtures/cli.ts @@ -30,8 +30,6 @@ const read = (request: IncomingMessage) => request.on("error", reject); }); -const sse = (chunks: ReadonlyArray) => - `${chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("")}data: [DONE]\n\n`; const chunk = (choice: object) => ({ id: "chatcmpl-mcp", object: "chat.completion.chunk", @@ -40,56 +38,86 @@ const chunk = (choice: object) => ({ choices: [{ index: 0, ...choice }], }); +/** One provider response as it streams: `send` writes a chunk carrying `delta`, `finish` ends it. */ +export interface Reply { + readonly send: (delta: object) => void; + readonly finish: (reason: string) => void; +} + /** - * An OpenAI-compatible provider. A request without tool results makes the calls `calls()` names, - * all in one turn; the request carrying their results answers with them, joined by ` | `. + * An OpenAI-compatible provider whose `reply` writes each response (`index` counts requests from + * 0). A response it never finishes hangs until the client goes away. */ -export const provider = async (calls: () => ReadonlyArray) => { +export const streaming = async (reply: (body: ChatRequest, response: Reply, index: number) => void) => { const requests: ChatRequest[] = []; const server = createServer(async (request, response) => { const body = JSON.parse(await read(request)) as ChatRequest; requests.push(body); response.writeHead(200, { "content-type": "text/event-stream" }); + reply( + body, + { + send: (delta) => response.write(`data: ${JSON.stringify(chunk({ delta }))}\n\n`), + finish: (reason) => + response.end(`data: ${JSON.stringify(chunk({ delta: {}, finish_reason: reason }))}\n\ndata: [DONE]\n\n`), + }, + requests.length - 1, + ); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + onTestFinished( + () => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }), + ); + return { url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, requests }; +}; + +/** The assistant delta making `calls`, all in one turn. */ +export const toolCalls = (calls: ReadonlyArray) => ({ + role: "assistant", + tool_calls: calls.map((call, index) => ({ + index, + id: `call_${index}`, + type: "function", + function: { name: call.name, arguments: JSON.stringify(call.arguments) }, + })), +}); + +/** + * A request without tool results makes the calls `calls()` names, all in one turn; the request + * carrying their results answers with them, joined by ` | `. + */ +export const provider = (calls: () => ReadonlyArray) => + streaming((body, response) => { const results = body.messages.filter((message) => message.role === "tool"); if (results.length > 0) { const text = results .map((result) => (typeof result.content === "string" ? result.content : JSON.stringify(result.content))) .join(" | "); - response.end( - sse([ - chunk({ delta: { role: "assistant", content: `Tool said: ${text}` } }), - chunk({ delta: {}, finish_reason: "stop" }), - ]), - ); + response.send({ role: "assistant", content: `Tool said: ${text}` }); + response.finish("stop"); return; } - response.end( - sse([ - chunk({ - delta: { - role: "assistant", - tool_calls: calls().map((call, index) => ({ - index, - id: `call_${index}`, - type: "function", - function: { name: call.name, arguments: JSON.stringify(call.arguments) }, - })), - }, - }), - chunk({ delta: {}, finish_reason: "tool_calls" }), - ]), - ); + response.send(toolCalls(calls())); + response.finish("tool_calls"); + }); + +/** Answers every request with `text`, cut off at the output token limit. */ +export const truncating = (text: string) => + streaming((_body, response) => { + response.send({ role: "assistant", content: text }); + response.finish("length"); }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - onTestFinished(() => new Promise((resolve) => server.close(() => resolve()))); - return { url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, requests }; -}; -/** `settings.jsonc` in `directory`, choosing the fake provider's one model. */ -export const writeSettings = (directory: string, url: string) => +/** `settings.jsonc` in `directory`, choosing the fake provider's one model; `extra` adds settings. */ +export const writeSettings = (directory: string, url: string, extra: object = {}) => writeFileSync( join(directory, "settings.jsonc"), JSON.stringify({ + ...extra, model: { provider: "mcp", id: "mcp-model" }, models: [ { @@ -145,3 +173,47 @@ export const runCli = ( resolve({ status, stdout, stderr }); }); }); + +export interface Served { + /** The RPC endpoint the server announced. */ + readonly url: string; + /** SIGKILLs the server, as a host dying would, and resolves once it is gone. */ + readonly kill: () => Promise; +} + +/** Starts `codework serve` from source on an ephemeral port; resolves once it listens. */ +export const serveCli = ( + args: ReadonlyArray, + options: { readonly cwd: string; readonly env: NodeJS.ProcessEnv }, +) => + new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["--conditions=development", cli, ...args, "serve", "--port", "0"], { + cwd: options.cwd, + env: options.env, + }); + const exited = new Promise((done) => child.on("exit", () => done())); + const kill = () => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + return exited; + }; + onTestFinished(kill); + let output = ""; + const deadline = setTimeout(() => { + void kill(); + reject(new Error(`codework serve did not start:\n${output}`)); + }, 30_000); + const listen = (data: Buffer) => { + output += data.toString("utf8"); + const url = /listening on (ws:\/\/\S+)/.exec(output)?.[1]; + if (url === undefined) return; + clearTimeout(deadline); + resolve({ url, kill }); + }; + child.stdout.on("data", listen); + child.stderr.on("data", (data: Buffer) => (output += data.toString("utf8"))); + child.on("error", reject); + child.on("exit", () => { + clearTimeout(deadline); + reject(new Error(`codework serve exited:\n${output}`)); + }); + }); diff --git a/packages/codework/test/fixtures/mcp-oauth.mjs b/packages/codework/test/fixtures/mcp-oauth.mjs index 4d42e1a8..68e46dd5 100644 --- a/packages/codework/test/fixtures/mcp-oauth.mjs +++ b/packages/codework/test/fixtures/mcp-oauth.mjs @@ -44,8 +44,9 @@ export const start = async (options = {}) => { client: undefined, pending: new Map(), /** - * What happens right after the next `tools/list`: `expire` the access tokens (a refresh still - * works), or `revoke` them and the refresh token (it does not). + * What happens right after the next `tools/list` on `/mcp`: `expire` the access tokens (a refresh + * still works), or `revoke` them and the refresh token (it does not). A listing on `/keyed` leaves + * it alone, or a run's keyed server, connecting first, would revoke before `/mcp` connects. */ afterList: undefined, /** Every bearer token presented to `/mcp` or `/keyed`, accepted or not. */ @@ -123,7 +124,11 @@ export const start = async (options = {}) => { ? { content: [{ type: "text", text: `called with ${token}` }] } : {}; if (body.method === "tools/call") state.log.push({ mcp: "tools/call", token }); - if (body.method === "tools/list" && state.afterList !== undefined) { + if ( + body.method === "tools/list" && + new URL(request.url, base).pathname === "/mcp" && + state.afterList !== undefined + ) { state.access.clear(); if (state.afterList === "revoke") state.refresh = undefined; state.log.push({ [state.afterList]: true }); diff --git a/packages/codework/test/recovery.test.ts b/packages/codework/test/recovery.test.ts new file mode 100644 index 00000000..656c9fa3 --- /dev/null +++ b/packages/codework/test/recovery.test.ts @@ -0,0 +1,488 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- this suite writes its home and settings with node:fs. */ +/* @effect-diagnostics globalTimers:off -- polling for a dying server runs outside any Effect. */ +/* @effect-diagnostics globalDate:off -- so does its deadline. */ +import { Control } from "@codeworksh/harness/effect"; +import { Effect, Fiber, Layer, Stream } from "effect"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it, onTestFinished } from "vite-plus/test"; +import { Client } from "../src/server/client.ts"; +import type { EventEnvelope } from "../src/server/envelope.ts"; +import { type ChatRequest, models, type Reply, serveCli, streaming, toolCalls, writeSettings } from "./fixtures/cli.ts"; + +/* + * A host that dies mid-run (SIGKILL, no shutdown) and a new server on the same home: what the + * reopened server makes of the run the dead one left, observed only over RPC and at the provider. + */ + +const mcpEcho = fileURLToPath(new URL("./fixtures/mcp-echo.mjs", import.meta.url)); + +/** A throwaway home whose settings choose the fake provider; the server runs in its `project`. */ +const setup = (url: string, settings: object = {}) => { + const home = realpathSync(mkdtempSync(join(tmpdir(), "codework-recovery-"))); + onTestFinished(() => rmSync(home, { recursive: true, force: true })); + const project = join(home, "project"); + mkdirSync(project); + writeSettings(home, url, settings); + const start = () => + serveCli(["--home", home, "--user-config-dir", home], { + cwd: project, + env: { ...process.env, HOME: home, CODEWORK_MODELS_FILE: models }, + }); + return { home, project, start }; +}; + +/** One RPC exchange over its own connection. */ +const call = (url: string, use: (rpc: Client.Client) => Effect.Effect) => + Effect.runPromise( + Effect.gen(function* () { + const context = yield* Layer.build(Client.layer(url)); + const rpc = yield* Client.make.pipe(Effect.provideContext(context)); + return yield* use(rpc); + }).pipe(Effect.scoped), + ); + +type Ids = { readonly sessionId: Parameters[0]["sessionId"] }; + +/** The session's durable log, oldest first. */ +const log = (url: string, ids: Ids) => + call(url, (rpc) => + rpc["session.log"]({ sessionId: ids.sessionId }).pipe( + Stream.runCollect, + Effect.map((items) => Array.from(items).filter((item): item is EventEnvelope => item.type !== "log.synced")), + ), + ); + +const count = (events: ReadonlyArray, type: string) => + events.filter((event) => event.type === type).length; + +/** Polls `check` until it holds; fails naming `what` after `ms`. */ +const until = async (what: string, check: () => Promise | A | undefined, ms = 30_000) => { + const deadline = Date.now() + ms; + while (true) { + const value = await check(); + if (value !== undefined && value !== false) return value; + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await new Promise((resolve) => setTimeout(resolve, 50)); + } +}; + +/** Waits for the log to hold `n` events of `type`, and returns it. */ +const logged = (url: string, ids: Ids, type: string, n = 1) => + until(`${n} × ${type}`, async () => { + const events = await log(url, ids); + return count(events, type) >= n ? events : undefined; + }); + +/** Live events of one thread, collected from the moment `ready` resolves. */ +const subscribe = (url: string, threadId: Parameters[0]["threadId"]) => { + const events: EventEnvelope[] = []; + let connected: () => void = () => {}; + const ready = new Promise((resolve) => (connected = resolve)); + const fiber = Effect.runFork( + Effect.gen(function* () { + const context = yield* Layer.build(Client.layer(url)); + const rpc = yield* Client.make.pipe(Effect.provideContext(context)); + yield* rpc["event.subscribe"]({ threadId }).pipe( + Stream.runForEach((event) => + Effect.sync(() => (event.type === "server.connected" ? connected() : events.push(event))), + ), + ); + }).pipe(Effect.scoped, Effect.ignore), + ); + onTestFinished(() => Effect.runPromise(Fiber.interrupt(fiber))); + return { events, ready }; +}; + +/** A gate a test opens: what waits on it holds until then. */ +const gate = () => { + let open: () => void = () => {}; + const opened = new Promise((resolve) => (open = resolve)); + return { open, opened }; +}; + +type Json = Record; + +const part = (value: unknown) => { + const p = value as Json; + if (p["type"] === "text") return { text: p["text"] }; + if (p["type"] === "thinking") return { thinking: p["thinking"] }; + if (p["type"] === "toolCall") { + const result = p["result"] as { content?: ReadonlyArray<{ text?: string }> } | undefined; + return { + tool: p["name"], + status: p["status"], + ...(result === undefined ? {} : { result: result.content?.map((one) => one.text).join("") }), + }; + } + return { type: p["type"] }; +}; + +/** What an event says, without ids, times or paths. */ +const summary = (event: EventEnvelope) => { + const data = event.data as Json; + const message = data["message"] as Json; + switch (event.type) { + case "session.prompt.admitted": + case "session.prompt.promoted": { + const prompt = data["prompt"] as { parts: ReadonlyArray<{ text?: string }> }; + return { type: event.type, delivery: data["delivery"], text: prompt.parts.map((one) => one.text).join("") }; + } + case "session.llm.progress": + return { + type: event.type, + changes: (data["changes"] as ReadonlyArray).map((change) => + change["_tag"] === "append" ? change : { ...change, part: part(change["part"]) }, + ), + }; + case "session.llm.ended": + case "session.llm.failed": + return { + type: event.type, + reason: data["reason"], + ...(message["errorMessage"] === undefined ? {} : { errorMessage: message["errorMessage"] }), + parts: (message["parts"] as ReadonlyArray).map(part), + }; + case "session.tool.settled": + return { type: event.type, part: part(data["part"]) }; + case "session.run.failed": + return { type: event.type, reason: data["reason"], message: data["message"] }; + case "session.retry.scheduled": + case "session.retry.finished": { + const { sessionId: _s, threadId: _t, timestamp: _ts, ...rest } = data; + return { type: event.type, ...rest }; + } + default: + return { type: event.type }; + } +}; + +/** The conversation a request carries, system prompt left out. */ +const sent = (request: ChatRequest) => + request.messages + .filter((message) => message.role !== "system" && message.role !== "developer") + .map((message) => { + const content = message.content; + const text = + typeof content === "string" + ? content + : Array.isArray(content) + ? content.map((one: { text?: string }) => one.text ?? "").join("") + : undefined; + const calls = (message as { tool_calls?: ReadonlyArray<{ function: { name: string } }> }).tool_calls; + return { + role: message.role, + ...(text === undefined || text === "" ? {} : { text }), + ...(calls === undefined ? {} : { calls: calls.map((one) => one.function.name) }), + }; + }); + +const history = async (url: string, ids: Ids) => + (await call(url, (rpc) => rpc["thread.history"](ids))).entries.map((entry) => ({ + role: entry.role, + state: entry.state, + parts: entry.parts.map(part), + })); + +const artifact = (value: unknown) => `${JSON.stringify(value, null, "\t")}\n`; + +const answer = (response: Reply, text: string) => { + response.send({ role: "assistant", content: text }); + response.finish("stop"); +}; + +/** Opens the response and then never finishes it: the request stays in flight until the host dies. */ +const hang = (response: Reply) => response.send({ role: "assistant" }); + +const retry = { retry: { maxRetries: 3, baseDelayMs: 10, maxDelayMs: 1_000 } }; + +describe("recovery after the host is killed", () => { + it("a reply killed mid-stream keeps its partial text, and the request is resent without it", async () => { + const resend = gate(); + const fake = await streaming((_body, response, index) => { + if (index === 0) { + response.send({ role: "assistant", content: "Partial answer, " }); + response.send({ content: "streamed before " }); + response.send({ content: "the host died." }); + return; + } + void resend.opened.then(() => answer(response, "The full answer.")); + }); + const { start } = setup(fake.url, retry); + + const first = await start(); + const created = await call(first.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(first.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Answer at length." })); + // The checkpoint lands a second into the stream: what a crash after it must not lose. + await logged(first.url, ids, "session.llm.progress"); + await first.kill(); + + const second = await start(); + const live = subscribe(second.url, ids.threadId); + await live.ready; + await until("the resend", () => fake.requests.length === 2); + resend.open(); + const events = await logged(second.url, ids, "session.turn.ended"); + await until("retry.finished", () => live.events.find((event) => event.type === "session.retry.finished")); + await call(second.url, (rpc) => rpc["thread.wait"](ids)); + + const failed = events.find((event) => event.type === "session.llm.failed"); + expect(summary(failed!)).toMatchObject({ + reason: "error", + errorMessage: "The host stopped while the model was answering, on attempt 1", + parts: [{ text: "Partial answer, streamed before the host died." }], + }); + // The aborted partial stays out of the model's context. + expect(sent(fake.requests[1]!)).toEqual(sent(fake.requests[0]!)); + + await expect( + artifact({ + log: events.map(summary), + // `session.retry.scheduled` is live only and may go out before a client can attach to the + // new server; `retry.finished` follows the resend this test holds until it has. + live: live.events.filter((event) => event.type === "session.retry.finished").map(summary), + history: await history(second.url, ids), + requests: fake.requests.map(sent), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.mid-stream.json"); + }, 90_000); + + it("a tool killed while it ran settles aborted, and the next request carries that without running it again", async () => { + const marker = join(realpathSync(mkdtempSync(join(tmpdir(), "codework-recovery-marker-"))), "ran"); + onTestFinished(() => rmSync(join(marker, ".."), { recursive: true, force: true })); + const fake = await streaming((body, response, index) => { + if (index === 0) { + response.send(toolCalls([{ name: "bash", arguments: { command: `echo ran >> ${marker} && sleep 20` } }])); + response.finish("tool_calls"); + return; + } + answer(response, `Recovered after ${body.messages.filter((one) => one.role === "tool").length} result.`); + }); + const { start } = setup(fake.url, retry); + + const first = await start(); + const created = await call(first.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(first.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Run the slow command." })); + await until("the tool to run", () => existsSync(marker)); + await first.kill(); + + const second = await start(); + const events = await logged(second.url, ids, "session.turn.ended", 2); + await call(second.url, (rpc) => rpc["thread.wait"](ids)); + + expect(readFileSync(marker, "utf8")).toBe("ran\n"); + expect(fake.requests).toHaveLength(2); + const tool = fake.requests[1]!.messages.find((one) => one.role === "tool"); + expect(tool?.content).toContain("Tool Execution Interrupted"); + + await expect( + artifact({ + log: events.map(summary), + history: await history(second.url, ids), + requests: fake.requests.map(sent), + ran: readFileSync(marker, "utf8"), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.tool.json"); + }, 90_000); + + it("a request killed with no retry left ends the run as interrupted, its follow-up still queued", async () => { + const fake = await streaming((_body, response) => hang(response)); + const { start } = setup(fake.url, { retry: { maxRetries: 1, baseDelayMs: 10, maxDelayMs: 1_000 } }); + + const first = await start(); + const created = await call(first.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(first.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "First." })); + await logged(first.url, ids, "session.llm.started"); + await call(first.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Then this.", delivery: "followUp" })); + await logged(first.url, ids, "session.prompt.admitted", 2); + await first.kill(); + + // The one retry: the request is resent, and the host dies with it in flight again. + const second = await start(); + await logged(second.url, ids, "session.llm.started", 2); + await second.kill(); + + const third = await start(); + const events = await logged(third.url, ids, "session.run.failed"); + await call(third.url, (rpc) => rpc["thread.wait"](ids)); + const queue = await call(third.url, (rpc) => rpc["thread.queue"](ids)); + + expect(fake.requests).toHaveLength(2); + expect(summary(events.find((event) => event.type === "session.run.failed")!)).toEqual({ + type: "session.run.failed", + reason: "interrupted", + message: "The host stopped while the model was answering, on attempt 2", + }); + expect(queue.entries.map((entry) => entry.text)).toEqual(["Then this."]); + + await expect( + artifact({ + log: events.map(summary), + history: await history(third.url, ids), + queue: queue.entries.map(({ delivery, text }) => ({ delivery, text })), + requests: fake.requests.map(sent), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.interrupted.json"); + }, 90_000); + + it("a run the host keeps dying in ends as a crashloop, and the next prompt runs normally", async () => { + let answering = false; + const fake = await streaming((_body, response) => + answering ? answer(response, "Back to normal.") : hang(response), + ); + const { start } = setup(fake.url, { retry: { maxRetries: 10, baseDelayMs: 10, maxDelayMs: 1_000 } }); + + let server = await start(); + const created = await call(server.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(server.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Keep crashing." })); + // The first run, then a resume each until the limit: each dies with its request in flight. + for (let started = 1; started <= Control.RESTART_LIMIT; started++) { + await logged(server.url, ids, "session.llm.started", started); + await server.kill(); + server = await start(); + } + // The resume past the limit is one too many: the run ends instead. + const ended = await logged(server.url, ids, "session.run.failed"); + await call(server.url, (rpc) => rpc["thread.wait"](ids)); + expect(fake.requests).toHaveLength(Control.RESTART_LIMIT); + expect(summary(ended.find((event) => event.type === "session.run.failed")!)).toMatchObject({ + reason: "crashloop", + }); + + answering = true; + await call(server.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Try again." })); + const events = await logged(server.url, ids, "session.turn.ended"); + await call(server.url, (rpc) => rpc["thread.wait"](ids)); + expect(fake.requests).toHaveLength(Control.RESTART_LIMIT + 1); + + await expect( + artifact({ + log: events.map(summary), + history: await history(server.url, ids), + requests: fake.requests.map(sent), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.crashloop.json"); + }, 120_000); + + it("a tool the host keeps dying in ends as a crashloop, each run of it the model's own", async () => { + const marker = join(realpathSync(mkdtempSync(join(tmpdir(), "codework-recovery-marker-"))), "ran"); + onTestFinished(() => rmSync(join(marker, ".."), { recursive: true, force: true })); + const runs = () => (existsSync(marker) ? readFileSync(marker, "utf8").split("\n").length - 1 : 0); + const fake = await streaming((_body, response) => { + response.send(toolCalls([{ name: "bash", arguments: { command: `echo ran >> ${marker} && sleep 20` } }])); + response.finish("tool_calls"); + }); + const { start } = setup(fake.url, retry); + + let server = await start(); + const created = await call(server.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(server.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Run the slow command." })); + // The first run, then a resume each until the limit: each dies with the re-issued tool running. + for (let ran = 1; ran <= Control.RESTART_LIMIT; ran++) { + await until(`run ${ran} of the tool`, () => runs() >= ran); + await server.kill(); + server = await start(); + } + const events = await logged(server.url, ids, "session.run.failed"); + await call(server.url, (rpc) => rpc["thread.wait"](ids)); + + expect(runs()).toBe(Control.RESTART_LIMIT); + expect(fake.requests).toHaveLength(Control.RESTART_LIMIT); + expect(summary(events.find((event) => event.type === "session.run.failed")!)).toMatchObject({ + reason: "crashloop", + }); + + await expect( + artifact({ + log: events.map(summary), + history: await history(server.url, ids), + requests: fake.requests.map(sent), + ran: readFileSync(marker, "utf8"), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.toolcrash.json"); + }, 180_000); + + it("replies cut off at the output limit with tool calls twice in a row end the run as truncated", async () => { + const fake = await streaming((_body, response) => { + response.send(toolCalls([{ name: "bash", arguments: { command: "echo cut" } }])); + response.finish("length"); + }); + const { start } = setup(fake.url, retry); + + const server = await start(); + const created = await call(server.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(server.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Run a long command." })); + const events = await logged(server.url, ids, "session.run.failed"); + await call(server.url, (rpc) => rpc["thread.wait"](ids)); + + expect(fake.requests).toHaveLength(2); + const settled = events.filter((event) => event.type === "session.tool.settled").map(summary); + expect(settled).toHaveLength(2); + for (const one of settled) expect(JSON.stringify(one)).toContain("Re-issue the tool call"); + expect(summary(events.find((event) => event.type === "session.run.failed")!)).toEqual({ + type: "session.run.failed", + reason: "truncated", + message: "The response hit the output token limit with tool calls 2 times in a row; the run is ended", + }); + + await expect( + artifact({ + log: events.map(summary), + history: await history(server.url, ids), + requests: fake.requests.map(sent), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.truncated.json"); + }, 90_000); + + it("a deferred tool loaded before the host died is still offered to the resent request", async () => { + const tools = (request: ChatRequest) => (request.tools ?? []).map((tool) => tool.function.name).sort(); + const fake = await streaming((_body, response, index) => { + if (index === 0) { + response.send(toolCalls([{ name: "toolsearch", arguments: { query: "echo" } }])); + response.finish("tool_calls"); + return; + } + if (index === 1) return hang(response); + answer(response, "Resumed with the loaded tool."); + }); + const { home, start } = setup(fake.url, retry); + writeFileSync( + join(home, "mcp.json"), + JSON.stringify({ + mcpServers: { fixture: { command: process.execPath, args: [mcpEcho], exposure: "deferred" } }, + }), + ); + + const first = await start(); + const created = await call(first.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + await call(first.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Find the echo tool." })); + await logged(first.url, ids, "session.llm.started", 2); + await first.kill(); + + const second = await start(); + const events = await logged(second.url, ids, "session.turn.ended", 2); + await call(second.url, (rpc) => rpc["thread.wait"](ids)); + + expect(fake.requests).toHaveLength(3); + expect(tools(fake.requests[0]!)).not.toContain("mcp__fixture__echo"); + expect(tools(fake.requests[1]!)).toContain("mcp__fixture__echo"); + expect(tools(fake.requests[2]!)).toContain("mcp__fixture__echo"); + + await expect( + artifact({ + log: events.map(summary), + tools: fake.requests.map(tools), + requests: fake.requests.map(sent), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.tools.json"); + }, 90_000); +}); diff --git a/packages/harness/src/control.ts b/packages/harness/src/control.ts index 547c3bfc..eb292386 100644 --- a/packages/harness/src/control.ts +++ b/packages/harness/src/control.ts @@ -55,8 +55,8 @@ export interface PromptInput extends ThreadRef { export type ResolveError = Session.SessionNotFoundError | Thread.ThreadNotFoundError; /** - * Times a run may be cut off by a dying process before the thread is parked instead of resumed. - * A committed turn starts the count over. + * Times in a row a run may be cut off by a dying process before it ends (`session.run.failed`, + * `crashloop`) instead of resuming. A committed turn starts the count over. */ export const RESTART_LIMIT = 5; @@ -100,8 +100,9 @@ export interface Interface { readonly active: Effect.Effect>; /** * Resumes the runs the last process left unfinished, as the server starts. One a crash cut off - * counts a restart, one a shutdown closed does not; at {@link RESTART_LIMIT} the thread is parked - * and waits for its next prompt. Resolves once every resumed run has settled. + * counts a restart, one a shutdown closed does not; at {@link RESTART_LIMIT} the run ends visibly + * instead, the thread idle with its queue kept for the next prompt. Resolves once every resumed + * run has settled. */ readonly resumeUnfinished: Effect.Effect; /** Prompts admitted but not yet in the conversation, steers and follow-ups, in admission order. */ @@ -271,8 +272,15 @@ export const layer = Layer.effect( // Claimed again here, not from the list: a stop since then cleared it, and must stand. // Counted before resuming, so a crash inside the resumed run is counted too. const restarts = yield* threads.claimRestart(threadId); - if (Option.isNone(restarts) || restarts.value >= RESTART_LIMIT) return; - yield* execution.recover(threadId).pipe( + if (Option.isNone(restarts)) return; + const recovery = + restarts.value >= RESTART_LIMIT + ? execution.abandon( + threadId, + `The run was interrupted ${restarts.value} times in a row; it is not resumed again`, + ) + : execution.recover(threadId); + yield* recovery.pipe( Effect.tapCause((cause) => Effect.logWarning("Resuming an unfinished thread failed", cause).pipe( Effect.annotateLogs({ threadId }), diff --git a/packages/harness/src/db/migrations.ts b/packages/harness/src/db/migrations.ts index e520d0bc..e3c9f3a1 100644 --- a/packages/harness/src/db/migrations.ts +++ b/packages/harness/src/db/migrations.ts @@ -124,6 +124,7 @@ export const migrations = { title TEXT, execution TEXT, restarts INTEGER NOT NULL DEFAULT 0, + attempt INTEGER, cost REAL NOT NULL DEFAULT 0, tokens_input INTEGER NOT NULL DEFAULT 0, tokens_output INTEGER NOT NULL DEFAULT 0, diff --git a/packages/harness/src/db/schema.sql.ts b/packages/harness/src/db/schema.sql.ts index da63974a..6213f195 100644 --- a/packages/harness/src/db/schema.sql.ts +++ b/packages/harness/src/db/schema.sql.ts @@ -143,6 +143,9 @@ export class ThreadRow extends Model.Class("ThreadRow")({ title: Model.FieldOption(Schema.String), execution: Model.FieldOption(ThreadSchema.Execution), restarts: Model.GeneratedByDb(Schema.Int), + // The 1-based number of the provider request in flight, from its start until its response ends, the + // turn commits or the run ends: still set on recovery, the host died with that request out. + attempt: Model.FieldOption(Schema.Int), // Billed usage of own entries only, never the inherited prefix. A cache: assistant envelopes are truth. cost: Model.GeneratedByDb(Schema.Finite), tokensInput: Model.GeneratedByDb(Schema.Int), diff --git a/packages/harness/src/effect/session.ts b/packages/harness/src/effect/session.ts index bdd8c7e1..1f5ffc3f 100644 --- a/packages/harness/src/effect/session.ts +++ b/packages/harness/src/effect/session.ts @@ -128,11 +128,6 @@ export interface ThreadInfo { readonly foundation?: string; /** Where it runs instead of the session's space and directory. */ readonly placement?: DocumentSchema.Placement; - /** - * Its unfinished run was cut off {@link Control.RESTART_LIMIT} times, so the server no longer - * resumes it; the next prompt runs it again. - */ - readonly parked: boolean; /** Billed usage of its own entries; what it inherits through a fork point counts for its source. */ readonly usage: Usage; readonly session: Info; @@ -279,14 +274,11 @@ const originOf = (row: ThreadRow): ThreadSchema.Origin => { export const threadInfo = Effect.fn("Session.threadInfo")(function* (id: ThreadSchema.ID) { const threads = yield* Thread.Service; const documents = yield* Document.Service; - const control = yield* Control.Service; const found = yield* threads.get(id); if (Option.isNone(found)) return yield* new Thread.ThreadNotFoundError({ threadId: id }); const row = found.value; const agent = yield* documents.agent(id); const title = Option.getOrUndefined(row.title); - const parked = - Option.isSome(row.execution) && row.restarts >= Control.RESTART_LIMIT && !(yield* control.active).has(id); return { id, sessionId: SessionSchema.ID.make(row.sessionId), @@ -297,7 +289,6 @@ export const threadInfo = Effect.fn("Session.threadInfo")(function* (id: ThreadS ...(agent.thinkingLevel === undefined ? {} : { thinkingLevel: agent.thinkingLevel }), ...(agent.foundation === undefined ? {} : { foundation: agent.foundation }), ...(agent.placement === undefined ? {} : { placement: agent.placement }), - parked, usage: { cost: row.cost, input: row.tokensInput, diff --git a/packages/harness/src/event/list.ts b/packages/harness/src/event/list.ts index cd893099..35ae9012 100644 --- a/packages/harness/src/event/list.ts +++ b/packages/harness/src/event/list.ts @@ -147,10 +147,14 @@ export const TurnStarted = EventSchema.define({ }); export type TurnStarted = typeof TurnStarted.Type; +/** + * A draft committed. `recovered` marks the commit a restart makes of a turn the host died in while + * its tools ran: no progress, so the restart count keeps counting. + */ export const TurnEnded = EventSchema.define({ type: "session.turn.ended", ...durableOptions, - schema: LLMFields, + schema: { ...LLMFields, recovered: optional(Schema.Literal(true)) }, }); export type TurnEnded = typeof TurnEnded.Type; @@ -232,6 +236,27 @@ export const LLMThinkingEnd = EventSchema.define({ }); export type LLMThinkingEnd = typeof LLMThinkingEnd.Type; +/** + * What one streamed part gained since the draft's last checkpoint: `append` extends a text or + * thinking part's text, `set` writes a new or otherwise changed part whole. + */ +export const LLMProgressChange = Schema.Union([ + Schema.TaggedStruct("append", { partIndex: NonNegativeInt, text: Schema.String }), + Schema.TaggedStruct("set", { partIndex: NonNegativeInt, part: EventSchema.AikitAssistantPart }), +]); +export type LLMProgressChange = typeof LLMProgressChange.Type; + +/** + * A checkpoint of the streaming reply into its draft, at most one a second, so a host that dies + * mid-reply leaves what was streamed. Carries only what changed since the previous checkpoint. + */ +export const LLMProgress = EventSchema.define({ + type: "session.llm.progress", + ...durableOptions, + schema: { ...LLMFields, changes: Schema.Array(LLMProgressChange) }, +}); +export type LLMProgress = typeof LLMProgress.Type; + /** aikit `done`: replace the LLMStarted payload in place; state stays draft. */ export const LLMEnded = EventSchema.define({ type: "session.llm.ended", @@ -244,7 +269,10 @@ export const LLMEnded = EventSchema.define({ }); export type LLMEnded = typeof LLMEnded.Type; -/** Abort the whole draft and discard every part. */ +/** + * End the draft as aborted, keeping the output it streamed. Its unfinished tool calls are aborted, + * never run. An aborted entry stays out of the model's context. + */ export const LLMFailed = EventSchema.define({ type: "session.llm.failed", ...durableOptions, @@ -317,6 +345,22 @@ export const ExecutionInterrupted = EventSchema.define({ }); export type ExecutionInterrupted = typeof ExecutionInterrupted.Type; +/** + * A run ended without its answer and is not resumed: the host died in it too often (`crashloop`), + * with no retry left for the request it cut off (`interrupted`), or replies kept hitting the output + * limit with tool calls (`truncated`). Queued prompts stay queued. + */ +export const RunFailed = EventSchema.define({ + type: "session.run.failed", + ...durableOptions, + schema: { + ...baseOptions, + reason: Schema.Literals(["crashloop", "interrupted", "truncated"]), + message: Schema.String, + }, +}); +export type RunFailed = typeof RunFailed.Type; + /** * Kernel lifecycle notice for one plugin module. * The `plugin..` namespace belongs to plugin-declared types, so this kernel type can never collide with one. @@ -411,8 +455,10 @@ export const Definitions = EventSchema.inventory( LLMThinkingStart, LLMThinkingDelta, LLMThinkingEnd, + LLMProgress, LLMEnded, LLMFailed, + RunFailed, ToolStarted, ToolProgress, ToolSettled, @@ -438,8 +484,10 @@ export const DurableDefinitions = EventSchema.inventory( SystemChanged, TurnEnded, LLMStarted, + LLMProgress, LLMEnded, LLMFailed, + RunFailed, ToolSettled, ); diff --git a/packages/harness/src/runner/event.ts b/packages/harness/src/runner/event.ts index 7c8568fa..054c3079 100644 --- a/packages/harness/src/runner/event.ts +++ b/packages/harness/src/runner/event.ts @@ -16,19 +16,57 @@ * events carry only fragments, so it has to reassemble text across deltas before * it can persist a block. Every aikit event already carries a complete assistant * message, and the terminal one carries the whole response. Successful output - * can therefore replace the draft in one write; failed output becomes a - * partless tombstone without needing a fragment buffer. + * can therefore replace the draft in one write, and failed output keeps the + * latest partial without needing a fragment buffer. While streaming, that + * partial is checkpointed into the draft (`LLMProgress`) so a dead host leaves it. */ import type { Event as AikitEvent, Message } from "@codeworksh/aikit"; -import { DateTime, Effect } from "effect"; +import { DateTime, Duration, Effect, Fiber, Option, Schema } from "effect"; import { Event } from "../event/event.ts"; import { EventList } from "../event/list.ts"; +import { EventSchema } from "../event/schema.ts"; import { SessionMessageSchema } from "../session/message/schema.ts"; import type { SessionSchema } from "../session/schema.ts"; import type { ThreadSchema } from "../thread/schema.ts"; +import { RunnerPart } from "./part.ts"; import { Runner } from "./run.ts"; +type Part = Message.AssistantMessage["parts"][number]; + +const CHECKPOINT_INTERVAL = Duration.seconds(1); + +const decodePartial = Schema.decodeUnknownEffect(EventSchema.AikitAssistantMessage); + +const textOf = (part: Part) => + part.type === "text" ? part.text : part.type === "thinking" ? part.thinking : undefined; +const withoutText = (part: Part): Part => + part.type === "text" ? { ...part, text: "" } : part.type === "thinking" ? { ...part, thinking: "" } : part; + +/** What `after` changed since `before`: grown text appends, every other difference writes the part whole. */ +const changesSince = (before: ReadonlyArray, after: ReadonlyArray) => { + const changes: Array = []; + for (const [partIndex, part] of after.entries()) { + const previous = before[partIndex]; + const was = previous === undefined ? undefined : textOf(previous); + const now = textOf(part); + if ( + previous !== undefined && + was !== undefined && + now !== undefined && + now.startsWith(was) && + JSON.stringify(withoutText(previous)) === JSON.stringify(withoutText(part)) + ) { + if (now.length > was.length) changes.push({ _tag: "append", partIndex, text: now.slice(was.length) }); + continue; + } + if (previous === undefined || JSON.stringify(previous) !== JSON.stringify(part)) { + changes.push({ _tag: "set", partIndex, part }); + } + } + return changes; +}; + /** How the response settled. The loop branches on this, never on `result()`. */ export type Terminal = | { @@ -45,6 +83,10 @@ export type Terminal = export interface Publisher { /** Set only after LLMStarted committed its draft row. */ readonly startedMessageId: SessionMessageSchema.ID | undefined; + /** A copy of the latest streamed message until the response settles: what a failure it did not report keeps. */ + readonly partial: Message.AssistantMessage | undefined; + /** Ends checkpointing, waiting out one in flight: no checkpoint lands after this returns. */ + readonly stop: Effect.Effect; /** Interpret one provider event. Durable ones commit before this returns. */ readonly publish: (event: AikitEvent.LLMMessageEvent) => Effect.Effect; /** How the response settled; fails if the stream ended without terminating. */ @@ -62,6 +104,69 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { let settled: Terminal | undefined; const clone = (message: Message.AssistantMessage): Message.AssistantMessage => structuredClone(message); + // aikit rewrites one partial object in place, so this is a live view: copied when read. + let latest: Message.AssistantMessage | undefined; + let checkpointed: ReadonlyArray = []; + let dirty = false; + let stopped = false; + let ticker: Fiber.Fiber | undefined; + + const checkpoint = Effect.gen(function* () { + dirty = false; + if (latest === undefined || startedMessageId === undefined) return; + // A partial mid-way through a part may not be a valid message yet; the next checkpoint takes it. + const decoded = yield* decodePartial(clone(latest)).pipe(Effect.option); + if (Option.isNone(decoded)) return; + const parts = decoded.value.parts; + const changes = changesSince(checkpointed, parts); + if (changes.length === 0) return; + yield* events.publish(EventList.LLMProgress, { + sessionId: input.sessionId, + threadId: input.threadId, + timestamp: yield* DateTime.now, + messageId: startedMessageId, + changes, + }); + checkpointed = parts; + }); + + // One checkpoint a second at most, one in flight; the ticker ends once a checkpoint finds nothing new. + const tick = Effect.gen(function* () { + while (true) { + yield* Effect.sleep(CHECKPOINT_INTERVAL); + yield* Effect.uninterruptible(checkpoint); + // Cleared with the decision, no yield between: an `observe` after it forks a fresh ticker. + if (!dirty) { + ticker = undefined; + return; + } + } + }).pipe( + // Not `ensuring`: after a normal exit `ticker` may already be the fresh one. + Effect.onError(() => + Effect.sync(() => { + ticker = undefined; + }), + ), + ); + + const observe = (partial: Message.AssistantMessage) => + Effect.suspend(() => { + // A partial without content, such as the opening one, shows nothing worth keeping. + if (stopped || partial.parts.length === 0) return Effect.void; + latest = partial; + dirty = true; + if (ticker !== undefined) return Effect.void; + return Effect.map(Effect.forkDetach(tick), (fiber) => { + ticker = fiber; + }); + }); + + const stop = Effect.suspend(() => { + stopped = true; + return ticker === undefined ? Effect.void : Fiber.interrupt(ticker); + }); + const fault = (reason: string) => new Runner.LLMStreamError({ threadId: input.threadId, reason }); /** @@ -95,6 +200,14 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { } const timestamp = yield* DateTime.now; const base = { sessionId: input.sessionId, threadId: input.threadId, timestamp }; + if ( + event.type !== "start" && + event.type !== "done" && + event.type !== "error" && + event.partial.messageId === startedMessageId + ) { + yield* observe(event.partial); + } switch (event.type) { case "start": { @@ -116,9 +229,8 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { /* * Live only, all of them. The terminal message contains every finished - * block, so persisting these would write the same text twice and, worse, - * leave an entry that a crash could strand mid-response -- which - * `Context.assemble` would then replay to the model as a complete turn. + * block; what a crash would lose is kept by the throttled checkpoints of + * the draft instead, which stays out of the model's context. * * Only scalars computed at push time are carried. `partial` is excluded * deliberately: it is one object aikit rewrites in place and pushes by @@ -193,6 +305,7 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { if (startedMessageId === undefined) return yield* fault("received done before start"); const message = clone(event.message); const identified = yield* identify(message.messageId); + yield* stop; yield* Effect.uninterruptible( Effect.gen(function* () { yield* events.publish(EventList.LLMEnded, { @@ -211,12 +324,13 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { /* * aikit calls the payload `error`, but it is a complete assistant - * message. LLMFailed uses its envelope as an aborted tombstone and the - * projector deliberately discards every partial part. + * message, with whatever streamed before the failure. LLMFailed keeps it + * as the aborted entry; a tool call in it never runs. */ case "error": { - const message = clone(event.error); + const message = RunnerPart.abortCalls(clone(event.error), DateTime.toEpochMillis(timestamp)); const identified = yield* identify(message.messageId); + yield* stop; yield* Effect.uninterruptible( Effect.gen(function* () { if (startedMessageId !== undefined) { @@ -245,6 +359,10 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { get startedMessageId() { return startedMessageId; }, + get partial() { + return settled !== undefined || latest === undefined ? undefined : clone(latest); + }, + stop, publish, terminal, } satisfies Publisher; diff --git a/packages/harness/src/runner/execute.ts b/packages/harness/src/runner/execute.ts index 07c3712c..7847dbc4 100644 --- a/packages/harness/src/runner/execute.ts +++ b/packages/harness/src/runner/execute.ts @@ -80,6 +80,8 @@ export const layer = Layer.effect( // Threads being recovered: their run neither marks the thread nor runs once it is unmarked. const recovering = new Set(); + // Recovered threads whose run is ended instead of resumed, with the message it ends with. + const abandoning = new Map(); // The log position at each user stop: prompts admitted after it are new work, not parked by it. const stoppedAt = new Map(); @@ -133,6 +135,9 @@ export const layer = Layer.effect( const sessionId = thread.value.sessionId; const recovered = recovering.delete(threadId); if (recovered && Option.isNone(thread.value.execution)) return; + // Only the drain the abandon itself started: one it joined is a live run, which must stand. + const abandoned = recovered && force ? abandoning.get(threadId) : undefined; + if (abandoned !== undefined) return yield* runner.abandon({ threadId, message: abandoned }); // The thread's placement, else the session's space and directory. A placement names its // space in a document no FK guards, so a space gone since is typed: `relink --thread` // is the repair. @@ -201,14 +206,18 @@ export const layer = Layer.effect( return accepted; }); - const recover = (threadId: ThreadId) => + // Under the thread's execution like any run, so a prompt that wakes it meanwhile cannot heal + // the same draft. + const recover = (threadId: ThreadId, abandon?: string) => Effect.suspend(() => { recovering.add(threadId); + if (abandon !== undefined) abandoning.set(threadId, abandon); return coordinator.run(threadId); }).pipe( Effect.ensuring( Effect.sync(() => { recovering.delete(threadId); + abandoning.delete(threadId); }), ), ); @@ -217,7 +226,8 @@ export const layer = Layer.effect( active: coordinator.active, interrupt, resume: coordinator.run, - recover, + recover: (threadId) => recover(threadId), + abandon: recover, wake: coordinator.wake, awaitIdle: coordinator.awaitIdle, }); diff --git a/packages/harness/src/runner/execution.ts b/packages/harness/src/runner/execution.ts index 21114270..fc3725f6 100644 --- a/packages/harness/src/runner/execution.ts +++ b/packages/harness/src/runner/execution.ts @@ -19,6 +19,11 @@ export interface Interface { * recovery chose the thread stands. */ readonly recover: (threadId: ThreadSchema.ID) => Effect.Effect; + /** + * Ends a run the last process left unfinished instead of resuming it, under the thread's execution + * like `recover`; see `Runner.Interface.abandon`. + */ + readonly abandon: (threadId: ThreadSchema.ID, message: string) => Effect.Effect; /** Registers newly recorded work. Repeated wakeups may coalesce. */ readonly wake: (threadId: ThreadSchema.ID) => Effect.Effect; /** @@ -43,6 +48,7 @@ export const noopLayer = Layer.succeed( active: Effect.succeed(new Set()), resume: () => Effect.void, recover: () => Effect.void, + abandon: () => Effect.void, wake: () => Effect.void, interrupt: () => Effect.succeed(false), awaitIdle: () => Effect.void, diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index 76f8de26..a8fce6b3 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -7,6 +7,7 @@ import { type Message, Transcript } from "@codeworksh/aikit"; import { Cause, DateTime, Duration, Effect, Exit, Layer, Option, Schema } from "effect"; +import type { ContextAssembly } from "../context/assemble.ts"; import { ContextCodec } from "../context/codec.ts"; import { Context } from "../context/context.ts"; import { Event } from "../event/event.ts"; @@ -14,6 +15,7 @@ import { EventList } from "../event/list.ts"; import { Inbox } from "../inbox/inbox.ts"; import { McpSection } from "../mcp/section.ts"; import { SessionMessageSchema } from "../session/message/schema.ts"; +import { SessionSchema } from "../session/schema.ts"; import type { ThreadSchema } from "../thread/schema.ts"; import { Thread } from "../thread/thread.ts"; import { State } from "../state/state.ts"; @@ -23,29 +25,30 @@ import { Hash } from "../util/hash.ts"; import { LLMEventPublisher } from "./event.ts"; import { Hooks } from "./hooks.ts"; import { LLM } from "./llm.ts"; +import { RunnerPart } from "./part.ts"; import { Retry } from "./retry.ts"; import { Runner } from "./run.ts"; export type Promotion = "steer" | "followUp" | undefined; /** A turn that committed an assistant message; `undefined` when it sent no request. */ -export type TurnResult = Pick | undefined; +export type TurnResult = + | (Pick & { + /** Whether the reply called tools, every one of them settled by now. */ + readonly toolCalls: boolean; + }) + | undefined; export interface Options { /** Deterministic provider seam for tests. */ readonly request?: LLM.Request; } -const terminalResult = (text: string) => ({ - content: [{ type: "text" as const, text }], - isError: true as const, -}); +const { abortedPart, endTime, terminalResult } = RunnerPart; const encodeJson = (value: unknown) => Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))(value).pipe(Effect.orDie); -const endTime = (call: Message.ToolCallPendingPart, now: number) => Math.max(call.time.end, now); - const skippedPart = (call: Message.ToolCallPendingPart, message: string, now: number): Message.ToolCallSkippedPart => ({ ...call, status: "skipped", @@ -53,12 +56,37 @@ const skippedPart = (call: Message.ToolCallPendingPart, message: string, now: nu time: { ...call.time, end: endTime(call, now) }, }); -const abortedPart = (call: Message.ToolCallPendingPart, now: number): Message.ToolCallAbortedPart => ({ - ...call, - status: "aborted", - result: terminalResult("Tool Execution Interrupted"), - time: { ...call.time, end: endTime(call, now) }, -}); +const truncatedCall = (name: string) => + `Tool call "${name}" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments.`; + +/** Replies in a row cut off at the output limit with tool calls before the run ends. */ +const TRUNCATION_LIMIT = 2; + +const hasToolCalls = (message: Message.AssistantMessage) => message.parts.some((part) => part.type === "toolCall"); + +/** Whether the leaf is a reply the model finished: a cut-off one with tool calls is answered once their results are in. */ +const leafAnswered = (assembled: ContextAssembly.Snapshot) => { + const leaf = + assembled.lastAssistant?.entryId === assembled.leafEntryId ? assembled.lastAssistant.message : undefined; + return leaf !== undefined && (leaf.stopReason === "stop" || (leaf.stopReason === "length" && !hasToolCalls(leaf))); +}; + +/** + * A request the host died with in flight counts as a failed attempt, retried like a transient + * provider failure. + */ +const hostStopped = (snapshot: State.Snapshot, message: string) => + new Runner.ProviderError({ + provider: snapshot.provider, + model: snapshot.model, + reason: new Runner.ProviderTransportError({ message, isRetryable: true }), + }); + +/** A request still in flight when the host died, as recovery finds it. */ +interface Crashed { + readonly attempt: number; + readonly message: string; +} const errorPart = ( call: Message.ToolCallPendingPart, @@ -109,26 +137,37 @@ export const layer = (options: Options = {}) => const state = yield* State.Service; const requestLLM = options.request ?? LLM.run; + const decodeDraft = Effect.fn("Loop.decodeDraft")(function* (draft: Thread.HydratedEntry) { + const decoded = yield* ContextCodec.decodeMessage(draft).pipe(Effect.orDie); + if (decoded.role !== "assistant") return yield* Effect.die(`draft ${draft.entry.id} is not an assistant`); + return decoded; + }); + + /** The aborted entry: what streamed (`partial`, else what the draft holds), its tool calls never run. */ const failureStub = Effect.fn("Loop.failureStub")(function* ( draft: Thread.HydratedEntry, reason: "aborted" | "error", message: string, + partial: Message.AssistantMessage | undefined, ) { - const decoded = yield* ContextCodec.decodeMessage(draft).pipe(Effect.orDie); - if (decoded.role !== "assistant") return yield* Effect.die(`draft ${draft.entry.id} is not an assistant`); + const decoded = yield* decodeDraft(draft); const completed = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - return { - ...decoded, - stopReason: reason, - errorMessage: message, - time: { ...decoded.time, completed }, - parts: [], - } satisfies Message.AssistantMessage; + return RunnerPart.abortCalls( + { + ...decoded, + stopReason: reason, + errorMessage: message, + time: { ...decoded.time, completed }, + parts: (partial ?? decoded).parts, + }, + completed, + ); }); const publishFailure = Effect.fn("Loop.publishFailure")(function* ( draft: Thread.HydratedEntry, cause: EventList.TurnAbortCause, + partial?: Message.AssistantMessage, ) { const reason = cause._tag === "interrupted" ? "aborted" : "error"; const message = cause._tag === "interrupted" ? "turn interrupted before commit" : cause.message; @@ -138,14 +177,106 @@ export const layer = (options: Options = {}) => threadId: draft.entry.threadId, messageId: SessionMessageSchema.ID.from(draft.entry.id), reason, - message: yield* failureStub(draft, reason, message), + message: yield* failureStub(draft, reason, message, partial), + }); + }); + + const publishSettled = Effect.fn("Loop.publishSettled")(function* ( + ids: { readonly sessionId: SessionSchema.ID; readonly threadId: ThreadSchema.ID }, + messageId: SessionMessageSchema.ID, + part: Message.ToolCallTerminalPart, + ) { + yield* events.publish(EventList.ToolSettled, { + timestamp: yield* DateTime.now, + sessionId: ids.sessionId, + threadId: ids.threadId, + messageId, + callID: part.callID, + part, + }); + }); + + const publishCommit = Effect.fn("Loop.publishCommit")(function* ( + ids: { readonly sessionId: SessionSchema.ID; readonly threadId: ThreadSchema.ID }, + messageId: SessionMessageSchema.ID, + recovered?: true, + ) { + yield* events.publish(EventList.TurnEnded, { + timestamp: yield* DateTime.now, + sessionId: ids.sessionId, + threadId: ids.threadId, + messageId, + ...(recovered === undefined ? {} : { recovered }), }); }); - const healDanglingDraft = Effect.fn("Loop.healDanglingDraft")(function* (threadId: ThreadSchema.ID) { + /** + * Settles the draft a dead process left. One whose request was in flight (`inFlight`, its + * failure) is aborted with what streamed. One whose response ended with tool calls had them + * running: those not settled are aborted, never run again, and the turn commits so the model + * sees what ran. Anything else is aborted as interrupted. + */ + const healDraft = Effect.fn("Loop.healDraft")(function* ( + threadId: ThreadSchema.ID, + inFlight: string | undefined, + ) { const draft = yield* threads.draft(threadId); if (Option.isNone(draft)) return; - yield* publishFailure(draft.value, { _tag: "interrupted" }); + if (inFlight !== undefined) return yield* publishFailure(draft.value, { _tag: "error", message: inFlight }); + const decoded = yield* decodeDraft(draft.value); + if (!hasToolCalls(decoded)) return yield* publishFailure(draft.value, { _tag: "interrupted" }); + const ids = { sessionId: draft.value.entry.sessionId, threadId }; + const messageId = SessionMessageSchema.ID.from(decoded.messageId); + yield* Effect.uninterruptible( + Effect.gen(function* () { + for (const part of decoded.parts) { + if (part.type !== "toolCall" || (part.status !== "pending" && part.status !== "running")) continue; + const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); + yield* publishSettled(ids, messageId, abortedPart(part, now)); + } + yield* publishCommit(ids, messageId, true); + }), + ); + }); + + /** Announces a retry of a failed request, then waits it out. */ + const scheduleRetry = Effect.fn("Loop.scheduleRetry")(function* ( + snapshot: State.Snapshot, + attempt: number, + delayMs: number, + message: string, + ) { + yield* events.publish(EventList.RetryScheduled, { + timestamp: yield* DateTime.now, + sessionId: snapshot.sessionId, + threadId: snapshot.threadId, + attempt, + maxRetries: snapshot.retry.maxRetries, + delayMs, + message, + }); + yield* Effect.sleep(Duration.millis(delayMs)); + }); + + /** Ends a run that is not resumed, leaving the thread idle with its queue kept. */ + const endRun = Effect.fn("Loop.endRun")(function* ( + sessionId: SessionSchema.ID, + threadId: ThreadSchema.ID, + reason: EventList.RunFailed["data"]["reason"], + message: string, + ) { + yield* Effect.uninterruptible( + Effect.gen(function* () { + yield* events.publish(EventList.RunFailed, { + timestamp: yield* DateTime.now, + sessionId, + threadId, + reason, + message, + }); + yield* threads.idle(threadId); + }), + ); }); const settleTools = Effect.fn("Loop.settleTools", { level: "Info" })(function* ( @@ -160,35 +291,16 @@ export const layer = (options: Options = {}) => let interruptedCause: Cause.Cause | undefined; const messageId = SessionMessageSchema.ID.from(message.messageId); - const settle = Effect.fn("Loop.settleTool")(function* ( - call: Message.ToolCallPendingPart, - part: Message.ToolCallTerminalPart, - ) { - yield* events.publish(EventList.ToolSettled, { - timestamp: yield* DateTime.now, - sessionId: snapshot.sessionId, - threadId: snapshot.threadId, - messageId, - callID: call.callID, - part, - }); - }); - const commit = Effect.fn("Loop.commitTurn")(function* () { - yield* events.publish(EventList.TurnEnded, { - timestamp: yield* DateTime.now, - sessionId: snapshot.sessionId, - threadId: snapshot.threadId, - messageId, - }); - markCommitted(); - }); + const settle = (part: Message.ToolCallTerminalPart) => publishSettled(snapshot, messageId, part); + const commit = () => publishCommit(snapshot, messageId).pipe(Effect.tap(() => Effect.sync(markCommitted))); yield* Effect.uninterruptibleMask((restore) => Effect.gen(function* () { + // The model re-issues them from these results: the drain continues as after tool use. if (reason === "length") { for (const call of pending) { const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - yield* settle(call, skippedPart(call, "tool call arguments were truncated", now)); + yield* settle(skippedPart(call, truncatedCall(call.name), now)); } yield* commit(); return; @@ -197,7 +309,7 @@ export const layer = (options: Options = {}) => if (reason !== "toolUse") { for (const call of pending) { const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - yield* settle(call, skippedPart(call, "unexpected tool call for terminal response", now)); + yield* settle(skippedPart(call, "unexpected tool call for terminal response", now)); } yield* commit(); return; @@ -258,7 +370,7 @@ export const layer = (options: Options = {}) => // cleanup publish a second terminal transition for the same call. yield* Effect.uninterruptible( Effect.gen(function* () { - yield* settle(call, handled); + yield* settle(handled); yield* Effect.sync(() => settled.add(call.callID)); }), ); @@ -275,7 +387,7 @@ export const layer = (options: Options = {}) => for (const call of pending) { if (settled.has(call.callID)) continue; const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - yield* settle(call, abortedPart(call, now)); + yield* settle(abortedPart(call, now)); } } yield* commit(); @@ -323,6 +435,7 @@ export const layer = (options: Options = {}) => promotion: Promotion, snapshot: State.Snapshot, continuing: boolean, + carried: Crashed | undefined, ) { const { sessionId, threadId } = snapshot; if (promotion !== undefined) { @@ -335,26 +448,25 @@ export const layer = (options: Options = {}) => } const assembled = yield* context.assemble(threadId); - const leaf = - assembled.lastAssistant?.entryId === assembled.leafEntryId ? assembled.lastAssistant.message : undefined; - if (leaf !== undefined && (leaf.stopReason === "stop" || leaf.stopReason === "length")) { - return undefined; - } + if (leafAnswered(assembled)) return undefined; const messages = [...assembled.messages, ...(yield* declareSystemChanges(snapshot, assembled.messages))]; // The latest attempt's publisher; a retried turn has one per provider request. let publisher: LLMEventPublisher.Publisher | undefined; let committed = false; - let retries = 0; - let failed: string | undefined; + // A recovered run goes on counting from the request the host died with. + let retries = carried?.attempt ?? 0; + let failed: string | undefined = carried?.message; /** Abort the latest attempt's draft, unless its own terminal event already did. */ const settleDraft = (cause: EventList.TurnAbortCause) => Effect.gen(function* () { - if (publisher?.startedMessageId === undefined) return; + if (publisher === undefined) return; + yield* publisher.stop; + if (publisher.startedMessageId === undefined) return; const stored = yield* threads.entry(publisher.startedMessageId); if (Option.isSome(stored) && stored.value.entry.state === "draft") { - yield* publishFailure(stored.value, cause); + yield* publishFailure(stored.value, cause, publisher.partial); } }); @@ -379,6 +491,7 @@ export const layer = (options: Options = {}) => }, { retry: retries, ...(failed === undefined ? {} : { failed }) }, ); + yield* threads.setAttempt(threadId, retries + 1); const terminal = yield* requestLLM({ threadId, ...prepared, publisher }); if (terminal.outcome === "failed") { @@ -410,16 +523,7 @@ export const layer = (options: Options = {}) => retries += 1; failed = error.message; yield* Effect.uninterruptible(settleDraft({ _tag: "error", message: error.message })); - yield* events.publish(EventList.RetryScheduled, { - timestamp: yield* DateTime.now, - sessionId, - threadId, - attempt: retries, - maxRetries: snapshot.retry.maxRetries, - delayMs, - message: error.message, - }); - yield* Effect.sleep(Duration.millis(delayMs)); + yield* scheduleRetry(snapshot, retries, delayMs, error.message); return yield* request(); }); }), @@ -454,7 +558,11 @@ export const layer = (options: Options = {}) => committed = true; }); if (interrupted !== undefined) return yield* Effect.failCause(interrupted); - return { messageId: terminal.message.messageId, reason: terminal.reason } satisfies TurnResult; + return { + messageId: terminal.message.messageId, + reason: terminal.reason, + toolCalls: hasToolCalls(terminal.message), + } satisfies TurnResult; }); // `onError`, not `catchCause`: a catch handler never runs on an interrupted fiber, and @@ -483,8 +591,9 @@ export const layer = (options: Options = {}) => promotion: Promotion, snapshot: State.Snapshot, continuing: boolean, + carried: Crashed | undefined, ) { - return yield* runTurnAttempt(promotion, snapshot, continuing); + return yield* runTurnAttempt(promotion, snapshot, continuing, carried); }); const run = Effect.fn("Loop.run", { level: "Info" })(function* (input: { @@ -492,7 +601,18 @@ export const layer = (options: Options = {}) => readonly force: boolean; readonly recovered?: boolean; }) { - yield* healDanglingDraft(input.threadId); + // Only a dead host leaves a request marked in flight: every run that ends clears it. + const crashed = Option.getOrUndefined( + Option.flatMap(yield* threads.get(input.threadId), (row) => row.attempt), + ); + const carried: Crashed | undefined = + crashed === undefined + ? undefined + : { + attempt: crashed, + message: `The host stopped while the model was answering, on attempt ${crashed}`, + }; + yield* healDraft(input.threadId, carried?.message); const hasSteer = yield* inputs.hasPending(input.threadId, "steer"); const hasFollowUp = hasSteer ? false : yield* inputs.hasPending(input.threadId, "followUp"); @@ -510,17 +630,9 @@ export const layer = (options: Options = {}) => (input.recovered === true || input.force) && !hasSteer && hasFollowUp && - (yield* context.assemble(input.threadId).pipe( - Effect.map((assembled) => { - const leaf = - assembled.lastAssistant?.entryId === assembled.leafEntryId - ? assembled.lastAssistant.message - : undefined; - return ( - assembled.messages.length > 0 && leaf?.stopReason !== "stop" && leaf?.stopReason !== "length" - ); - }), - )); + (yield* context + .assemble(input.threadId) + .pipe(Effect.map((assembled) => assembled.messages.length > 0 && !leafAnswered(assembled)))); // A cut-off run still promotes steers that land while it loads: they join the reply being answered. let promotion: Promotion = hasSteer || cutOff ? "steer" : hasFollowUp ? "followUp" : undefined; /** Steers when there are any, else the oldest follow-up: what a run promotes first. */ @@ -532,19 +644,45 @@ export const layer = (options: Options = {}) => const drain = Effect.gen(function* () { const start = yield* state.snapshot(input.threadId); + if (carried !== undefined) { + const delayMs = Retry.delay(start.retry, carried.attempt, hostStopped(start, carried.message)); + if (delayMs === undefined) { + return yield* endRun(start.sessionId, input.threadId, "interrupted", carried.message); + } + yield* scheduleRetry(start, carried.attempt, delayMs, carried.message); + } let snapshot = start; let continuing = input.force; + let resent = carried; + let truncations = 0; // Tool results and steers continue first; a follow-up only once both are exhausted. while (true) { - const result: TurnResult = yield* runTurn(promotion, snapshot, continuing); - const toolUse = result?.reason === "toolUse"; + const result: TurnResult = yield* runTurn(promotion, snapshot, continuing, resent); + resent = undefined; + // A reply cut off at the output limit gets its tool calls back as errors to re-issue. + const truncated = result?.reason === "length" && result.toolCalls; + const toolUse = result?.reason === "toolUse" || truncated; + truncations = truncated ? truncations + 1 : 0; const completed: Hooks.Completed | undefined = result === undefined ? undefined - : { ...result, snapshot, pending: yield* peekPending(input.threadId) }; + : { + messageId: result.messageId, + reason: result.reason, + snapshot, + pending: yield* peekPending(input.threadId), + }; const decision = completed === undefined ? undefined : yield* hooks.finishTurn(completed); // Queued steers and follow-ups stay for the next run. if (decision === "end") return; + if (truncations >= TRUNCATION_LIMIT) { + return yield* endRun( + snapshot.sessionId, + input.threadId, + "truncated", + `The response hit the output token limit with tool calls ${truncations} times in a row; the run is ended`, + ); + } // A forced continuation after a finished reply sends nothing: the leaf guard in // `runTurnAttempt` ends the turn unless something was appended after it. @@ -565,7 +703,17 @@ export const layer = (options: Options = {}) => ); }); - return Runner.Service.of({ run }); + const abandon = Effect.fn("Loop.abandon")(function* (input: { + readonly threadId: ThreadSchema.ID; + readonly message: string; + }) { + const thread = yield* threads.get(input.threadId); + if (Option.isNone(thread)) return; + yield* healDraft(input.threadId, Option.isSome(thread.value.attempt) ? input.message : undefined); + yield* endRun(SessionSchema.ID.make(thread.value.sessionId), input.threadId, "crashloop", input.message); + }); + + return Runner.Service.of({ run, abandon }); }), ); diff --git a/packages/harness/src/runner/part.ts b/packages/harness/src/runner/part.ts new file mode 100644 index 00000000..0a6e7d83 --- /dev/null +++ b/packages/harness/src/runner/part.ts @@ -0,0 +1,36 @@ +/* + * @file Terminal tool-call parts the runner writes for calls that did not run to completion. + */ + +import type { Message } from "@codeworksh/aikit"; + +type OpenCall = Message.ToolCallPendingPart | Message.ToolCallRunningPart; + +export const terminalResult = (text: string) => ({ + content: [{ type: "text" as const, text }], + isError: true as const, +}); + +export const endTime = (call: OpenCall, now: number) => Math.max(call.time.end, now); + +export const abortedPart = (call: OpenCall, now: number): Message.ToolCallAbortedPart => { + const { status: _status, ...base } = call; + return { + ...base, + status: "aborted", + result: terminalResult("Tool Execution Interrupted"), + time: { ...call.time, end: endTime(call, now) }, + }; +}; + +/** The message with every tool call still pending or running aborted: an ended reply never runs them. */ +export const abortCalls = (message: Message.AssistantMessage, now: number): Message.AssistantMessage => ({ + ...message, + parts: message.parts.map((part) => + part.type === "toolCall" && (part.status === "pending" || part.status === "running") + ? abortedPart(part, now) + : part, + ), +}); + +export * as RunnerPart from "./part.ts"; diff --git a/packages/harness/src/runner/run.ts b/packages/harness/src/runner/run.ts index 67877973..51f95907 100644 --- a/packages/harness/src/runner/run.ts +++ b/packages/harness/src/runner/run.ts @@ -193,6 +193,11 @@ export interface Interface { /** Resuming a run a previous process left unfinished: its cut-off reply is answered first. */ readonly recovered?: boolean; }) => Effect.Effect; + /** + * Ends a run the server no longer resumes: settles what the dead process left of its reply, + * records `session.run.failed` with `message`, and leaves the thread idle, its queue kept. + */ + readonly abandon: (input: { readonly threadId: ThreadSchema.ID; readonly message: string }) => Effect.Effect; } export class Service extends Context.Service()("@codeworksh/harness/runner/run/Service") {} diff --git a/packages/harness/src/session/projector.ts b/packages/harness/src/session/projector.ts index 94e9e760..bdf2bbf3 100644 --- a/packages/harness/src/session/projector.ts +++ b/packages/harness/src/session/projector.ts @@ -174,6 +174,24 @@ export const layer = Layer.effectDiscard( }), ); + yield* events.project(EventList.LLMProgress, (event) => + Effect.gen(function* () { + const changes = yield* Effect.forEach(event.data.changes, (change) => + change._tag === "append" + ? Effect.succeed(change) + : ContextCodec.encodePart(change.part).pipe( + Effect.orDie, + Effect.map((part): Thread.ProgressChange => ({ + _tag: "set", + partIndex: change.partIndex, + part, + })), + ), + ); + yield* threads.progress({ id: event.data.messageId, threadId: event.data.threadId, changes }); + }), + ); + yield* events.project(EventList.LLMEnded, (event) => Effect.gen(function* () { const encoded = yield* encodeAssistant(event.data); @@ -183,6 +201,7 @@ export const layer = Layer.effectDiscard( data: encoded.data, parts: encoded.parts, }); + yield* threads.setAttempt(event.data.threadId, null); }), ); @@ -194,6 +213,7 @@ export const layer = Layer.effectDiscard( threadId: event.data.threadId, seq: yield* seqOf(event), data: encoded.data, + parts: encoded.parts, }); }), ); @@ -255,7 +275,8 @@ export const layer = Layer.effectDiscard( threadId: event.data.threadId, seq: yield* seqOf(event), }); - yield* threads.resetRestarts(event.data.threadId); + if (event.data.recovered !== true) yield* threads.resetRestarts(event.data.threadId); + yield* threads.setAttempt(event.data.threadId, null); }), ); }), diff --git a/packages/harness/src/thread/thread.ts b/packages/harness/src/thread/thread.ts index ce5bf81b..6c6ada30 100644 --- a/packages/harness/src/thread/thread.ts +++ b/packages/harness/src/thread/thread.ts @@ -98,7 +98,7 @@ export interface ReplaceAssistant { readonly parts: ReadonlyArray; } -/** Ends a draft assistant: committed once every tool part is settled, or aborted (parts dropped). */ +/** Ends a draft assistant: committed once every tool part is settled, or aborted. */ export interface EndAssistant { readonly id: string; readonly threadId: ThreadSchema.ID; @@ -106,6 +106,17 @@ export interface EndAssistant { readonly seq: number; } +/** One part change of a streaming draft: `append` extends a text or thinking part, `set` writes one whole. */ +export type ProgressChange = + | { readonly _tag: "append"; readonly partIndex: number; readonly text: string } + | { readonly _tag: "set"; readonly partIndex: number; readonly part: AppendPart }; + +export interface ProgressAssistant { + readonly id: string; + readonly threadId: ThreadSchema.ID; + readonly changes: ReadonlyArray; +} + export interface ForkInput { readonly threadId: ThreadSchema.ID; readonly entryId: string; @@ -158,12 +169,14 @@ export interface Interface { ) => Effect.Effect; /** Projector-only: replace the payload of an existing draft assistant in place. */ readonly replaceAssistant: (input: ReplaceAssistant) => Effect.Effect; + /** Projector-only: apply a streaming checkpoint to a draft assistant's parts. */ + readonly progress: (input: ProgressAssistant) => Effect.Effect; /** Projector-only: settle one pending/running tool part on a draft assistant. */ readonly settleToolCall: (input: SettleToolCall) => Effect.Effect; /** Projector-only: commit a draft after every tool part is terminal. */ readonly commit: (input: EndAssistant) => Effect.Effect; - /** Projector-only: turn a draft into a partless aborted tombstone. */ - readonly abort: (input: EndAssistant & { readonly data: string }) => Effect.Effect; + /** Projector-only: end a draft as aborted with the payload and parts given. */ + readonly abort: (input: ReplaceAssistant & EndAssistant) => Effect.Effect; /** Cut a new thread off `threadId`'s line at a committed entry. */ readonly fork: (input: ForkInput) => Effect.Effect; /** Projector of `session.thread.forked`: the thread and its copied documents, at the event's seq. */ @@ -191,6 +204,13 @@ export interface Interface { readonly claimRestart: (threadId: ThreadSchema.ID) => Effect.Effect>; /** A committed turn is progress: the restart count starts over. Runs inside `TurnEnded`'s commit. */ readonly resetRestarts: (threadId: ThreadSchema.ID) => Effect.Effect; + /** Records the 1-based provider request now in flight, or none (null) once its response ended. */ + readonly setAttempt: (threadId: ThreadSchema.ID, attempt: number | null) => Effect.Effect; + /** + * Leaves a run that is not resumed idle: no mark, no restarts, no request in flight, whatever is + * queued. Its queued prompts wait for the next prompt to wake the thread. + */ + readonly idle: (threadId: ThreadSchema.ID) => Effect.Effect; /** Root threads with a run left unfinished, oldest first. */ readonly unfinished: Effect.Effect>; /** Set or clear (null) the bookmark label on an entry of the thread. */ @@ -211,6 +231,7 @@ const decodeEnvelopeUsage = Schema.decodeUnknownEffect(SessionSchema.AssistantEn const decodeMessageEnvelopeIdentity = Schema.decodeUnknownEffect(SessionSchema.MessageEnvelopeIdentity); const decodeCompactionData = Schema.decodeUnknownEffect(SessionSchema.CompactionData); const decodeJsonObject = Schema.decodeUnknownEffect(SessionSchema.JsonObject); +const encodeJsonObject = Schema.encodeEffect(SessionSchema.JsonObject); const decodeUserText = Schema.decodeUnknownOption( Schema.fromJsonString(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })), ); @@ -395,30 +416,38 @@ export const layer = Layer.effect( originCallId: origin.kind === "subagent" ? Option.some(origin.callId) : Option.none(), title: Option.fromUndefinedOr(input.title), execution: Option.none(), + attempt: Option.none(), createdSeq: input.seq, }); yield* insertThread(row); }, Effect.orDie); + const writePart = Effect.fnUntraced(function* ( + entryId: string, + threadId: ThreadSchema.ID, + partIndex: number, + part: AppendPart, + ) { + const partRow = yield* EntryPartRow.insert.makeEffect({ + id: part.id ?? uuidv7(), + entryId, + threadId, + partIndex, + type: part.type, + status: Option.fromUndefinedOr(part.status), + callId: Option.fromUndefinedOr(part.callId), + toolName: Option.fromUndefinedOr(part.toolName), + data: part.data, + }); + yield* insertPart(partRow); + }); + const writeParts = Effect.fnUntraced(function* ( entryId: string, threadId: ThreadSchema.ID, parts: ReadonlyArray, ) { - for (const [partIndex, part] of parts.entries()) { - const partRow = yield* EntryPartRow.insert.makeEffect({ - id: part.id ?? uuidv7(), - entryId, - threadId, - partIndex, - type: part.type, - status: Option.fromUndefinedOr(part.status), - callId: Option.fromUndefinedOr(part.callId), - toolName: Option.fromUndefinedOr(part.toolName), - data: part.data, - }); - yield* insertPart(partRow); - } + for (const [partIndex, part] of parts.entries()) yield* writePart(entryId, threadId, partIndex, part); }); // Usage lives on the thread and moves with the envelope stored; the session's last activity @@ -605,6 +634,38 @@ export const layer = Layer.effect( ); }, Effect.orDie); + const progress = Effect.fn("Thread.progress")(function* (input: ProgressAssistant) { + yield* sql.withTransaction( + Effect.gen(function* () { + yield* requireDraft(input.id, input.threadId, "progress"); + const now = yield* epochNow; + for (const change of input.changes) { + if (change._tag === "set") { + yield* sql`DELETE FROM part WHERE entry_id = ${input.id} AND part_index = ${change.partIndex}`; + yield* writePart(input.id, input.threadId, change.partIndex, change.part); + continue; + } + const [row] = yield* sql<{ readonly data: string }>` + SELECT data FROM part WHERE entry_id = ${input.id} AND part_index = ${change.partIndex} + `; + const data = row === undefined ? undefined : yield* decodeJsonObject(row.data); + const field = + data?.["type"] === "text" ? "text" : data?.["type"] === "thinking" ? "thinking" : undefined; + const text = field === undefined ? undefined : data?.[field]; + if (data === undefined || field === undefined || typeof text !== "string") { + return yield* Effect.die(`progress cannot append to part ${change.partIndex} of ${input.id}`); + } + const next = yield* encodeJsonObject({ ...data, [field]: text + change.text }); + yield* sql` + UPDATE part SET data = ${next}, updated_at = ${now} + WHERE entry_id = ${input.id} AND part_index = ${change.partIndex} + `; + } + yield* sql`UPDATE entry SET updated_at = ${now} WHERE id = ${input.id}`; + }), + ); + }, Effect.orDie); + const settleToolCall = Effect.fn("Thread.settleToolCall")(function* (input: SettleToolCall) { const data = yield* decodeJsonObject(input.data).pipe(Effect.orDie); if (data["type"] !== "toolCall" || data["callID"] !== input.callId || data["status"] !== input.status) { @@ -643,13 +704,14 @@ export const layer = Layer.effect( yield* sql`UPDATE thread SET updated_at = ${now} WHERE id = ${input.threadId}`; }, Effect.orDie); - const abort = Effect.fn("Thread.abort")(function* (input: EndAssistant & { readonly data: string }) { + const abort = Effect.fn("Thread.abort")(function* (input: ReplaceAssistant & EndAssistant) { yield* sql.withTransaction( Effect.gen(function* () { const { entry, thread } = yield* requireDraft(input.id, input.threadId, "abort"); const delta = yield* usageDelta(entry.data, input.data); const now = yield* epochNow; yield* sql`DELETE FROM part WHERE entry_id = ${input.id}`; + yield* writeParts(input.id, input.threadId, input.parts); yield* sql` UPDATE entry SET state = 'aborted', committed_seq = ${input.seq}, data = ${input.data}, updated_at = ${now} @@ -778,10 +840,12 @@ export const layer = Layer.effect( end: "finished" | "stopped" | "closed", stoppedAt?: number, ) { + // A run that ended in this process has no request in flight: only a dead host leaves one. if (end === "closed") { - yield* sql`UPDATE thread SET execution = 'closed' WHERE id = ${threadId}`; + yield* sql`UPDATE thread SET execution = 'closed', attempt = NULL WHERE id = ${threadId}`; return; } + yield* sql`UPDATE thread SET attempt = NULL WHERE id = ${threadId}`; // Pending prompts that keep the mark: any for a finished run; for a stop, only those sent // after it, since what was queued before it stays parked by the user's choice. const after = end === "stopped" ? (stoppedAt ?? Number.MAX_SAFE_INTEGER) : -1; @@ -809,6 +873,14 @@ export const layer = Layer.effect( yield* sql`UPDATE thread SET restarts = 0 WHERE id = ${threadId} AND restarts > 0`; }, Effect.orDie); + const setAttempt = Effect.fn("Thread.setAttempt")(function* (threadId: ThreadSchema.ID, attempt: number | null) { + yield* sql`UPDATE thread SET attempt = ${attempt} WHERE id = ${threadId}`; + }, Effect.orDie); + + const idle = Effect.fn("Thread.idle")(function* (threadId: ThreadSchema.ID) { + yield* sql`UPDATE thread SET execution = NULL, restarts = 0, attempt = NULL WHERE id = ${threadId}`; + }, Effect.orDie); + const findUnfinished = SqlSchema.findAll({ Request: Schema.Void, Result: ThreadRow, @@ -845,6 +917,7 @@ export const layer = Layer.effect( insert, append, replaceAssistant, + progress, settleToolCall, commit, abort, @@ -855,6 +928,8 @@ export const layer = Layer.effect( endExecution, claimRestart, resetRestarts, + setAttempt, + idle, unfinished, setLabel, }); diff --git a/packages/harness/test/__artifacts__/runner.retry.json b/packages/harness/test/__artifacts__/runner.retry.json index 56dbd33f..5abafd7f 100644 --- a/packages/harness/test/__artifacts__/runner.retry.json +++ b/packages/harness/test/__artifacts__/runner.retry.json @@ -87,7 +87,7 @@ "id": "assistant_2", "type": "assistant", "state": "aborted", - "parts": 0 + "parts": 1 }, { "id": "assistant_3", diff --git a/packages/harness/test/control.prompt.test.ts b/packages/harness/test/control.prompt.test.ts index b1feba1c..09da03b5 100644 --- a/packages/harness/test/control.prompt.test.ts +++ b/packages/harness/test/control.prompt.test.ts @@ -395,6 +395,7 @@ const watched = Control.layer.pipe( active: Effect.succeed(new Set()), resume: () => Effect.void, recover: () => Effect.void, + abandon: () => Effect.void, wake: (threadId) => Effect.sync(() => wakes.push(threadId)), interrupt: () => Effect.succeed(false), awaitIdle: () => Effect.void, diff --git a/packages/harness/test/plugin.foreign.test.ts b/packages/harness/test/plugin.foreign.test.ts index 17805ebd..37897d11 100644 --- a/packages/harness/test/plugin.foreign.test.ts +++ b/packages/harness/test/plugin.foreign.test.ts @@ -1,6 +1,6 @@ import "./utils/env.ts"; import type { Message } from "@codeworksh/aikit"; -import { Effect, Schema } from "effect"; +import { Effect, Option, Schema } from "effect"; import { glob, readFile } from "node:fs/promises"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -84,7 +84,10 @@ describe("plugins built against another Effect instance", () => { const assistant = path.at(-1); expect(assistant?.entry.type).toBe("assistant"); expect(assistant?.entry.state).toBe("aborted"); - expect(assistant?.parts).toEqual([]); + // The reply's tool call is kept, aborted: it never ran. + expect(assistant?.parts.map((part) => [part.toolName, part.status])).toEqual([ + [Option.some("foreign_echo"), Option.some("aborted")], + ]); })); it("pins one Effect version, declared and resolved alike, across the workspace", async () => { diff --git a/packages/harness/test/runner.llm.live.test.ts b/packages/harness/test/runner.llm.live.test.ts index 9a8bcc24..2b43744d 100644 --- a/packages/harness/test/runner.llm.live.test.ts +++ b/packages/harness/test/runner.llm.live.test.ts @@ -114,7 +114,7 @@ describe.each(LIVE)("runner LLM — live $name", (live) => { ); liveIt( - "bridges Effect interruption to a partless aborted tombstone", + "bridges Effect interruption to an aborted entry keeping what streamed", Effect.gen(function* () { const { threads, sessionId, threadId } = yield* setup; const events = yield* Event.Service; @@ -149,8 +149,8 @@ describe.each(LIVE)("runner LLM — live $name", (live) => { const stored = yield* ContextCodec.decodeMessage(path[0]!); if (stored.role !== "assistant") return yield* Effect.die(`stored message is ${stored.role}, not assistant`); expect(stored.stopReason).toBe("aborted"); - expect(stored.parts).toEqual([]); expect(["thinking", "text"]).toContain(interruptedPart); + expect(stored.parts.map((part) => part.type)).toContain(interruptedPart); }), { timeout: 180_000 }, ); diff --git a/packages/harness/test/runner.loop.test.ts b/packages/harness/test/runner.loop.test.ts index a5c98be1..a1e56edf 100644 --- a/packages/harness/test/runner.loop.test.ts +++ b/packages/harness/test/runner.loop.test.ts @@ -10,7 +10,6 @@ import { SqlClient, SqlSchema } from "effect/sql"; import { describe, expect } from "vite-plus/test"; import { Context } from "../src/context/context.ts"; import { Control } from "../src/control.ts"; -import { Session as SessionApi } from "../src/effect/session.ts"; import { Database } from "../src/db/db.ts"; import { InboxRow } from "../src/db/schema.sql.ts"; import { Event } from "../src/event/event.ts"; @@ -556,7 +555,7 @@ describe("runner loop — provider interruption", () => { const { live: it } = testEffect(runtime({ open })); it( - "aborts the whole draft as a partless tombstone, then propagates interruption", + "aborts the draft keeping what streamed, then propagates interruption", Effect.gen(function* () { const execution = yield* RunnerExecution.Service; const events = yield* Event.Service; @@ -581,7 +580,7 @@ describe("runner loop — provider interruption", () => { expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); expect(path[2]!.entry.state).toBe("aborted"); expect(JSON.parse(path[2]!.entry.data)).toMatchObject({ stopReason: "aborted" }); - expect(path[2]!.parts).toEqual([]); + expect(path[2]!.parts.map((part) => JSON.parse(part.data))).toEqual([{ type: "text", text: "partial" }]); expect(Array.from(yield* execution.active)).toEqual([]); }), { timeout: 10_000 }, @@ -637,7 +636,7 @@ describe("runner loop — provider failure", () => { const path = yield* threads.line(threadId); expect(path.map((item) => item.entry.type)).toEqual(["user", "system", "assistant"]); expect(path[2]!.entry.state).toBe("aborted"); - expect(path[2]!.parts).toEqual([]); + expect(path[2]!.parts.map((part) => JSON.parse(part.data))).toEqual([{ type: "text", text: "partial" }]); expect(JSON.parse(path[2]!.entry.data)).toMatchObject({ stopReason: "error", errorMessage: "provider failed", @@ -1305,31 +1304,41 @@ describe("runner loop — resume after restart", () => { ); it( - "parks a session at the restart limit, and its next prompt runs it again", + "ends a run at the restart limit visibly, and its next prompt runs it again", Effect.gen(function* () { yield* provider.reset; const control = yield* Control.Service; + const events = yield* Event.Service; + const threads = yield* Thread.Service; const sql = yield* SqlClient.SqlClient; - const { sessionId, threadId } = yield* seedSession(); + const { threadId } = yield* seedSession(); + const failures: Array = []; + yield* events.listen((event) => + Schema.is(EventList.RunFailed)(event) + ? Effect.sync(() => { + failures.push(event.data); + }) + : Effect.void, + ); yield* crashedMidReply(threadId, "A"); yield* sql`UPDATE thread SET restarts = ${Control.RESTART_LIMIT - 1} WHERE id = ${threadId}`; yield* control.resumeUnfinished; expect(contexts).toEqual([]); - expect(yield* stateOf(threadId)).toEqual({ execution: "running", restarts: Control.RESTART_LIMIT }); - expect((yield* (yield* SessionApi.get(sessionId).pipe(Effect.map(Option.getOrThrow))).main.info).parked).toBe( - true, - ); + expect(yield* stateOf(threadId)).toEqual({ execution: null, restarts: 0 }); + expect(failures.map((failure) => [failure.reason, failure.message])).toEqual([ + ["crashloop", `The run was interrupted ${Control.RESTART_LIMIT} times in a row; it is not resumed again`], + ]); + const [, healed] = yield* threads.line(threadId); + expect(healed!.entry.state).toBe("aborted"); + expect(healed!.parts.map((part) => JSON.parse(part.data).text)).toEqual(["half a reply"]); yield* send(threadId, "B"); yield* settle(threadId); expect(contexts.map(texts)).toEqual([["A", "B"]]); expect(yield* stateOf(threadId)).toEqual({ execution: null, restarts: 0 }); - expect((yield* (yield* SessionApi.get(sessionId).pipe(Effect.map(Option.getOrThrow))).main.info).parked).toBe( - false, - ); }), ); diff --git a/packages/plugin/src/event.ts b/packages/plugin/src/event.ts index bf4e26a2..0a42f2a7 100644 --- a/packages/plugin/src/event.ts +++ b/packages/plugin/src/event.ts @@ -3,17 +3,19 @@ import { Effect, Schema, type SchemaAST, SchemaGetter, SchemaIssue } from "effec import { uuidv7 } from "uuidv7"; import { isAikitAssistantMessage, + isAikitAssistantPart, isAikitToolCallTerminalPart, optional, validateAikitAssistantMessage, + validateAikitAssistantPart, validateAikitToolCallTerminalPart, withStatics, } from "./schema.ts"; -type AikitAssistantPart = Message.AssistantMessage["parts"][number]; +type AssistantPart = Message.AssistantMessage["parts"][number]; /** Undeclared fields used only while aikit is assembling a streamed part. */ -const aikitTransientFields: Readonly>> = { +const aikitTransientFields: Readonly>> = { text: ["streamId"], image: [], thinking: ["streamId"], @@ -27,9 +29,12 @@ const omit = (value: T, keys: ReadonlyArray): T => { return copy; }; +const canonicalizeAikitAssistantPart = (part: AssistantPart): AssistantPart => + omit(part, aikitTransientFields[part.type]); + const canonicalizeAikitAssistantMessage = (message: Message.AssistantMessage): Message.AssistantMessage => ({ ...message, - parts: message.parts.map((part) => omit(part, aikitTransientFields[part.type])), + parts: message.parts.map(canonicalizeAikitAssistantPart), }); const reasonOf = (cause: unknown): string => (cause instanceof Error ? cause.message : String(cause)); @@ -56,6 +61,25 @@ export const AikitAssistantMessage = Schema.Unknown.pipe( ); export type AikitAssistantMessage = typeof AikitAssistantMessage.Type; +const decodeAikitAssistantPart = (value: unknown, options: SchemaAST.ParseOptions) => + Effect.try({ + try: () => canonicalizeAikitAssistantPart(validateAikitAssistantPart(value, "aikit assistant part")), + catch: (cause) => new SchemaIssue.InvalidValue({ message: reasonOf(cause) }, value, options), + }); + +const aikitAssistantPartDeclared = Schema.declare(isAikitAssistantPart, { + expected: "aikit assistant part", +}); + +/** One part of an aikit assistant message, streaming-only fields stripped like {@link AikitAssistantMessage}. */ +export const AikitAssistantPart = Schema.Unknown.pipe( + Schema.decodeTo(aikitAssistantPartDeclared, { + decode: SchemaGetter.transformEffect(decodeAikitAssistantPart), + encode: SchemaGetter.transformEffect(decodeAikitAssistantPart), + }), +); +export type AikitAssistantPart = typeof AikitAssistantPart.Type; + const decodeAikitToolCallTerminalPart = (value: unknown, options: SchemaAST.ParseOptions) => Effect.try({ try: () => validateAikitToolCallTerminalPart(value, "aikit terminal toolcall part"), diff --git a/packages/plugin/src/schema.ts b/packages/plugin/src/schema.ts index 9d21aa6e..31f70eac 100644 --- a/packages/plugin/src/schema.ts +++ b/packages/plugin/src/schema.ts @@ -41,6 +41,9 @@ export const validateAikitUserMessage = (value: unknown, label: string): Message export const validateAikitAssistantMessage = (value: unknown, label: string): Message.AssistantMessage => validateAikitSchema(Message.AssistantMessageSchema, value, label); +export const validateAikitAssistantPart = (value: unknown, label: string): Message.AssistantMessage["parts"][number] => + validateAikitSchema(Message.AssistantMessageSchema.properties.parts.items, value, label); + export const validateAikitToolCallTerminalPart = (value: unknown, label: string): Message.ToolCallTerminalPart => validateAikitSchema(Message.ToolCallTerminalPartSchema, value, label); @@ -52,6 +55,9 @@ export const isAikitModelInfo = (value: unknown): value is Model.Info => aikitVa export const isAikitAssistantMessage = (value: unknown): value is Message.AssistantMessage => aikitValidatorFor(Message.AssistantMessageSchema).Check(value); +export const isAikitAssistantPart = (value: unknown): value is Message.AssistantMessage["parts"][number] => + aikitValidatorFor(Message.AssistantMessageSchema.properties.parts.items).Check(value); + export const isAikitToolCallTerminalPart = (value: unknown): value is Message.ToolCallTerminalPart => aikitValidatorFor(Message.ToolCallTerminalPartSchema).Check(value);