From 57653eed5cb04b6ea4538bf9b1809bd69a9e0517 Mon Sep 17 00:00:00 2001 From: bft-codebot Date: Sun, 3 May 2026 20:59:26 +0000 Subject: [PATCH] sync(bfmono): chore(gambit): cut 1.0.0-rc.2 (+19 more) (bfmono@af2da504c) This PR is an automated gambitmono sync of bfmono Gambit packages. - Source: `packages/gambit/` - Core: `packages/gambit/packages/gambit-core/` - bfmono rev: af2da504c Changes: - af2da504c chore(gambit): cut 1.0.0-rc.2 - 84f610a1d docs(workloop): align Gambit brand hierarchy - ca4aff086 chore(gambit): remove legacy desktop fallbacks - e281e27f1 fix(gambit): hydrate chat transcript from persisted state - 35438f961 chore(gambit): record full precommit verification - e88e4957e docs(gambit): reposition around scenarios and graders - 2d2691076 fix(gambit): reject sandboxed chat runs - 98ab911eb docs(gambit): use canonical graders frontmatter - bb582eeb6 feat(gambit): improve chat event observability - 43e0588ba feat(gambit): stream and control chat turns - 538d0ad1b feat(gambit): add local deck chat repro server - 60078d9f6 fix(workloop): disable Codex websockets in chief runtime - 93d44fb06 fix(gambit): preserve whitespace in streamed assistant deltas - 76e21a05f fix(workloop): preserve Codex auth refresh failures - 224cfdca6 fix(gambit): fall back without host service token - 668e393de feat(gambit): add browser introspection live commands - 4d8a6ad7b feat(workloop): bridge runtime Codex refresh to host services - 81ac0db58 feat(gambit): add live browser pointer refs - 62e132a24 chore(browser): move runtime out of Gambit - e6c80f928 test(gambit): remove legacy chat suites Do not edit this repo directly; make changes in bfmono and re-run the sync. --- AGENTS.md | 8 +- CHANGELOG.md | 33 +- README.md | 183 ++- deno.jsonc | 2 +- docs/external/README.md | 15 +- docs/external/concepts/hourglass.md | 6 +- docs/external/concepts/runtime.md | 13 +- docs/external/examples/README.md | 6 +- .../examples/agent_with_typescript.md | 11 +- docs/external/examples/handlers_ts.md | 2 +- docs/external/getting-started/README.md | 28 +- docs/external/guides/authoring.md | 66 +- docs/external/reference/cli.md | 5 +- docs/external/reference/cli/commands/chat.md | 40 + docs/external/reference/handlers.md | 2 +- docs/external/reference/openai-compat.md | 6 +- .../cards/grader_decks.card.md | 4 +- .../cards/grader_decks.card.md | 4 +- .../cards/grader_decks.card.md | 4 +- .../actions/complete_task.mock.deck.ts | 25 + .../actions/escalate_task.mock.deck.ts | 27 + .../actions/taxo_lookup_account.mock.deck.ts | 33 + .../local-chat/taxo-runtime-tools.mock.md | 11 + examples/local-chat/taxo-support.deck.md | 13 + .../local-chat/workloop-runtime-tools.mock.md | 17 + packages/gambit-core/README.md | 34 +- packages/gambit-core/deno.json | 2 +- packages/gambit-simulator/deno.json | 2 +- .../demo/gambit-ui-demo-script.md | 4 +- .../simulator-ui/src/DocsPage.tsx | 8 +- .../agent_with_multi_actions/PROMPT.md | 2 +- .../advanced/agent_with_typescript/PROMPT.md | 2 +- .../examples/advanced/arena_chatbot/PROMPT.md | 2 +- .../cli_cold_emailer/lookup_profile.deck.ts | 4 +- .../cli_cold_emailer/sample_input.json | 2 +- .../advanced/patient_swapper/PROMPT.md | 2 +- .../advanced/policy_support_bot/PROMPT.md | 2 +- .../cards/grader_decks.card.md | 4 +- .../cards/grader_decks.card.md | 8 +- .../schemas/voice_call_input.zod.ts | 4 +- .../after/faq_bot_after.deck.md | 10 +- .../before/faq_bot_before.deck.md | 10 +- scaffolds/demo/hello.deck.md | 2 +- src/cli.ts | 25 + src/cli_args.test.ts | 29 + src/cli_args.ts | 14 + src/cli_help.test.ts | 13 + src/cli_utils.test.ts | 2 +- src/codex_app_server_debug.ts | 10 +- src/codex_auth.test.ts | 21 +- src/codex_auth.ts | 7 +- src/codex_preflight.test.ts | 19 +- src/commands/chat.test.ts | 943 +++++++++++++ src/commands/chat.ts | 1194 +++++++++++++++++ src/commands/chat_runtime_tools.ts | 109 ++ .../gambit-bot/policy/deck-format-1.0.md | 6 +- .../scenarios/faq_bot_build_flow/PROMPT.md | 2 +- src/mcp_server.test.ts | 8 +- src/mcp_server.ts | 11 +- src/providers/codex.ts | 11 +- src/providers/ollama.test.ts | 4 +- src/runtime_host_service.ts | 8 +- 62 files changed, 2864 insertions(+), 240 deletions(-) create mode 100644 docs/external/reference/cli/commands/chat.md create mode 100644 examples/local-chat/actions/complete_task.mock.deck.ts create mode 100644 examples/local-chat/actions/escalate_task.mock.deck.ts create mode 100644 examples/local-chat/actions/taxo_lookup_account.mock.deck.ts create mode 100644 examples/local-chat/taxo-runtime-tools.mock.md create mode 100644 examples/local-chat/taxo-support.deck.md create mode 100644 examples/local-chat/workloop-runtime-tools.mock.md create mode 100644 src/commands/chat.test.ts create mode 100644 src/commands/chat.ts create mode 100644 src/commands/chat_runtime_tools.ts diff --git a/AGENTS.md b/AGENTS.md index f554ffc2e..dd389f548 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,10 +2,10 @@ You are Codebot, a helpful assistant that works with developers working on the project github.com/bolt-foundry/gambit to navigate and contribute to our codebase. -Gambit is an open source AI assistant orchestration and authoring framework -built by the company Bolt Foundry. Using Gambit, developers and product teams -can prove that their AI works the way they expect, and they can fix it when it -doesn't. +Gambit is an open source synthetic scenario and evaluation layer for agent +systems. Using Gambit, developers and product teams can create realistic +scenarios, validate eval data, grade behavior, preserve trace evidence, and turn +failures into regression suites. ## Core philosophy diff --git a/CHANGELOG.md b/CHANGELOG.md index f9f4e7b45..9cb646c3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,43 @@ +++ [release] -since = "b6ed5a3a8d6a9b272ce520d885d3953d27c2c0d4" +since = "00872eb8b2190de2ba0904c0ac4f4cdbc611718b" +++ # Changelog -## Unreleased +## Unreleased (v1.0.0) - TBD +## v1.0.0-rc.2 + +- fix(gambit): preserve structured text spacing +- fix(gambit): pass codex runtime tools +- feat(gambit): make provider execution responses-only +- feat(gambit): expose structured responses runtime +- fix(gambit): preserve responses continuation context +- chore(gambit): remove dead chat provider adapters +- test(gambit): remove legacy chat suites +- chore(browser): move runtime out of Gambit +- feat(gambit): add live browser pointer refs +- feat(workloop): bridge runtime Codex refresh to host services +- feat(gambit): add browser introspection live commands +- fix(gambit): fall back without host service token +- fix(workloop): preserve Codex auth refresh failures +- fix(gambit): preserve whitespace in streamed assistant deltas +- fix(workloop): disable Codex websockets in chief runtime +- feat(gambit): add local deck chat repro server +- feat(gambit): stream and control chat turns +- feat(gambit): improve chat event observability +- docs(gambit): use canonical graders frontmatter +- fix(gambit): reject sandboxed chat runs +- docs(gambit): reposition around scenarios and graders +- docs(gambit): record agent file naming feedback +- chore(gambit): record full precommit verification +- fix(gambit): hydrate chat transcript from persisted state +- chore(gambit): use generic runtime env names +- docs(workloop): align Gambit brand hierarchy + ## v1.0.0-rc.1 - BREAKING: make Gambit runtime execution Responses-only. diff --git a/README.md b/README.md index aaee69533..56491a766 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,15 @@ # Gambit logo -Gambit is an open-source, developer-first framework that helps you build\ -reliable LLM workflows by composing small, typed “decks”\ -with clear inputs/outputs and guardrails. Run decks locally, stream traces, and\ -debug with a built-in UI. +Agent frameworks help you build agents. Gambit helps you create the evidence +that they work. + +Gambit is the synthetic scenario and evaluation layer for agent systems: create +realistic scenarios, validate their quality, run agents against them, grade the +behavior, capture trace evidence, and turn failures into regression suites. + +Native Gambit agents are still the fastest path to the fully integrated loop: +typed inputs and outputs, local runs, scenarios, graders, traces, permission +evidence, and the simulator's Build/Test/Grade/Verify workflow. [Watch the demo video](https://youtu.be/J_hQ2L_yy60). @@ -19,8 +25,8 @@ export OPENROUTER_API_KEY=... npx @bolt-foundry/gambit demo ``` -Downloads example files (hello decks plus the `examples/` gallery) and sets -environment variables. +Downloads example files (hello agent definitions plus the `examples/` gallery) +and sets environment variables. To start onboarding with the simulator, run: @@ -29,7 +35,7 @@ npx @bolt-foundry/gambit-simulator serve gambit/hello.deck.md open http://localhost:8000/debug ``` -Use the Build tab to draft your own workspace decks and scenarios. +Use the Build tab to draft your own workspace agents and scenarios. Run an example in the terminal (`repl`): @@ -48,35 +54,102 @@ open http://localhost:8000/debug --- +## Why Gambit + +Agent teams already have many ways to build and orchestrate agents: native +Gambit, Mastra, LangGraph, OpenAI Agents SDK, CrewAI, Google ADK, LlamaIndex, +Pydantic AI, and custom stacks. The harder product problem is creating the +situations those agents need to survive, checking whether those situations are +good tests, and preserving the evidence when behavior regresses. + +Gambit focuses on that reliability loop: + +- **Generate scenarios** for realistic user, tool, workflow, policy, and edge + case pressure. +- **Evaluate the scenario data** for realism, coverage, difficulty, grounding, + duplication, and expected-outcome clarity. +- **Run agent evals** against native Gambit, Mastra, LangGraph, OpenAI, or + custom agents. +- **Grade behavior** from transcripts, artifacts, traces, and typed outputs. +- **Diagnose failures** with trace evidence and permission evidence. +- **Regenerate regression suites** from failures so the same behavior does not + quietly break again. + +For a native Gambit agent, the same system defines, runs, traces, tests, grades, +and debugs the agent end to end. For a Mastra, LangGraph, OpenAI, or custom +agent, Gambit sits on the other side of the framework: the test-data engine, +grader loop, local reproduction harness, and CI behavior check. + +## Common workflows + +### Native Gambit path + +Define the agent in Gambit, run it locally, add scenarios for the behavior that +must keep working, attach graders, inspect traces in the simulator, and reuse +the same checks in CI. This is the most direct path when you want Gambit to own +both the agent definition and the verification loop. + +### Bring your own agent + +Use Mastra to build the TypeScript agent application. Use Gambit to create and +validate scenario suites around the important Mastra behaviors, then grade the +transcripts and artifacts those runs produce. A thin wrapper can record run +inputs, transcript turns, artifacts, state paths, and trace references so Gambit +can grade them and keep failing cases reproducible. + +### Pull request gate + +Run important scenarios on every pull request, grade the resulting transcripts +or artifacts, and fail the check when behavior drops below the expected +standard. Failed checks should keep the trace, state, and reproduction inputs so +the regression can be debugged locally. + +```yaml +# Proposed workflow shape. This is positioning guidance, not a published +# bolt-foundry/gambit-action release. +name: Agent behavior checks + +on: + pull_request: + +jobs: + gambit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: npx @bolt-foundry/gambit scenario gambit/root.deck.md --test-deck gambit/scenarios/smoke.deck.md --grade gambit/graders/smoke.deck.md --state .gambit/ci-smoke.json --trace .gambit/ci-smoke.jsonl +``` + ## Status quo -- Most teams wire one long prompt to several tools and hope the model routes\ - correctly. -- Context often arrives as a single giant fetch or RAG blob, so costs climb and\ - hallucinations slip in. -- Input/outputs are rarely typed, which makes orchestration brittle and hard to\ - test offline. -- Debugging leans on provider logs instead of local traces, so reproducing\ - failures is slow. +- Teams have more ways than ever to build agents, but fewer ways to know whether + their eval data covers the behavior that will matter in production. +- Synthetic scenarios can look plausible while duplicating each other, missing + policy edges, or failing to state the expected outcome clearly. +- Agent failures often disappear into provider logs, so the team cannot replay + the exact inputs, transcript, tool calls, and artifacts that caused the + regression. +- CI usually checks code shape more reliably than agent behavior. ## Our vision -- Treat each step as a small deck with explicit inputs/outputs and guardrails;\ - model calls are just one kind of action. -- Mix LLM and compute tasks interchangeably and effortlessly inside the same\ - deck tree. -- Feed models only what they need per step; inject references and cards instead\ - of dumping every document. -- Keep orchestration logic local and testable; run decks offline with\ - predictable traces. -- Ship with built-in observability (streaming, REPL, debug UI) so debugging\ - feels like regular software, not guesswork. +- Generate the situations your agents need to survive: users, tasks, workflows, + tool pressure, policy edges, and hard failure modes. +- Grade the scenario data itself before it becomes trusted eval data. +- Run any target agent against the curated suite and preserve the transcript, + state, artifacts, trace events, and permission evidence. +- Diagnose failures by capability gap, tool issue, prompt issue, policy + ambiguity, retrieval miss, or unsafe action. +- Feed those failures back into sharper follow-up scenarios and regression + checks. --- ## Using the CLI -Use the CLI to run decks locally, stream output, and capture traces/state. +Use the CLI to run agent definitions locally, stream output, and capture +traces/state. The current CLI and file format still use `deck` as the exact +implementation term. Run with npx (no install): @@ -84,7 +157,7 @@ Run with npx (no install): npx @bolt-foundry/gambit ``` -Run a deck once: +Run an agent definition once: ``` npx @bolt-foundry/gambit run --context --message @@ -99,7 +172,34 @@ Drop into a REPL (streams by default): npx @bolt-foundry/gambit repl ``` -Run a persona against a root deck (scenario): +Start a focused browser chat for an agent definition: + +``` +npx @bolt-foundry/gambit chat --state .gambit/chat/workspace.sqlite --trace .gambit/chat/trace.jsonl +``` + +Use `chat` when you need a localhost transcript, saved state, trace output, and +runtime-supplied tools without the full simulator workbench. Use `repl` for a +terminal loop, `run` for one-shot automation, and `gambit-simulator serve` for +Build/Test/Grade/Verify workflows. + +For repeatable repros, pass `--repro-message ` to attach the original user +ask to the session payload without sending it automatically. + +Supply runtime tools with Markdown/TOML files: + +``` +npx @bolt-foundry/gambit chat MANAGER.md --runtime-tools ./workloop-runtime-tools.mock.md +npx @bolt-foundry/gambit chat support.deck.md --runtime-tools ./taxo-runtime-tools.mock.md +``` + +The runtime-tool file uses `[[tools]]` entries with `name`, `description`, +optional `inputSchema`, and optional `action`. Action bindings run Gambit agent +definitions with the tool arguments as context, keeping product-specific tools +outside the portable root agent. See `examples/local-chat/` for Workloop-style +and Taxo-style mock tool fixtures. + +Run a scenario persona against a root agent: ``` npx @bolt-foundry/gambit scenario --test-deck @@ -125,7 +225,7 @@ Tracing and state:  ### Worker sandbox defaults -- Deck-executing CLI surfaces default to worker sandbox execution. +- CLI commands that execute decks default to worker sandbox execution. - Use `--no-worker-sandbox` (or `--legacy-exec`) to force legacy in-process execution. - `--worker-sandbox` explicitly forces worker execution on. @@ -204,7 +304,9 @@ header (left of `New chat`). ## Using the Library -Use the library when you want TypeScript decks/cards or custom compute steps. +Use the library when you want TypeScript agent definitions, reusable instruction +snippets, or custom compute steps. The exported helper names remain `defineDeck` +and `defineCard` for compatibility. Import the helpers from JSR: @@ -216,10 +318,10 @@ import { defineDeck, defineCard } from "jsr:@bolt-foundry/gambit"; - `reviews/2026-04-15-AAR-raw-response-items.md` -Define `contextSchema`/`responseSchema` with Zod to validate IO, and implement\ -`run`/`execute` for compute decks. To call a child deck from code, use\ -`ctx.spawnAndWait({ path, input })`. Emit structured trace events with\ -`ctx.log(...)`. +Define `contextSchema`/`responseSchema` with Zod to validate IO, and implement +`run`/`execute` for compute agent definitions. To call a child agent definition +from code, use `ctx.spawnAndWait({ path, input })`. Emit structured trace events +with `ctx.log(...)`. ### Runtime defaults for programmatic `runDeckResponses` @@ -288,9 +390,9 @@ Replacement mapping: --- -## Author your first deck +## Author your first native Gambit agent -### Minimal Markdown deck (model-powered): `hello_world.deck.md` +### Minimal Markdown agent definition (model-powered): `hello_world.deck.md` ``` +++ @@ -310,7 +412,7 @@ Run it: npx @bolt-foundry/gambit run ./hello_world.deck.md --context '"Gambit"' --stream ``` -### Compute deck in TypeScript (no model call): `echo.deck.ts` +### Compute agent definition in TypeScript (no model call): `echo.deck.ts` ```typescript // echo.deck.ts @@ -333,7 +435,7 @@ Run it: npx @bolt-foundry/gambit run ./echo.deck.ts --context '{"text":"ping"}' ``` -### Deck with a child action (calls a TypeScript tool): `agent_with_time.deck.md` +### Agent definition with a child action (calls a TypeScript tool): `agent_with_time.deck.md` ``` +++ @@ -374,8 +476,9 @@ npx @bolt-foundry/gambit run ./agent_with_time.deck.md --context '"hello"' --str ### Legacy respond-flow demo (historical compatibility) `packages/gambit/examples/respond_flow/` is kept as a legacy compatibility -example for historical transcript/grader behavior. New decks should return -schema-valid assistant output directly instead of calling `gambit_respond`. +example for historical transcript/grader behavior. New agent definitions should +return schema-valid assistant output directly instead of calling +`gambit_respond`. ``` cd packages/gambit diff --git a/deno.jsonc b/deno.jsonc index 25a6fdc46..b81414a46 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -2,7 +2,7 @@ "$schema": "https://deno.land/x/deno/cli/schemas/config-file.v1.json", "name": "@bolt-foundry/gambit", "description": "Agent harness framework for building, running, and verifying LLM workflows in Markdown and code.", - "version": "1.0.0-rc.1", + "version": "1.0.0-rc.2", "license": "Apache-2.0", "repository": { "type": "git", diff --git a/docs/external/README.md b/docs/external/README.md index 30ebc268c..b78a9eb54 100644 --- a/docs/external/README.md +++ b/docs/external/README.md @@ -1,8 +1,17 @@ # Gambit Documentation -Welcome to Gambit, the open-source, developer-first agent harness for building, -running, and verifying LLM workflows. These docs focus on helping you ship -assistants with confidence. +Agent frameworks help you build agents. Gambit helps you create the evidence +that they work. + +Gambit is the synthetic scenario and evaluation layer for agent systems. These +docs focus on helping you create realistic scenarios, validate scenario quality, +run agents against curated suites, grade behavior, capture trace evidence, and +turn failures into regression checks. + +Use Gambit as the native runtime for a fully integrated agent loop, or use it on +the other side of an existing framework such as Mastra, LangGraph, OpenAI, or +custom code: the test-data engine, grader loop, local reproduction harness, and +CI check for behavior that must not regress. Start here: diff --git a/docs/external/concepts/hourglass.md b/docs/external/concepts/hourglass.md index 63649b267..d08516c1b 100644 --- a/docs/external/concepts/hourglass.md +++ b/docs/external/concepts/hourglass.md @@ -21,7 +21,7 @@ the needed context and tone. ## Applying to Gambit - Use separate snippets for assistant persona, user persona, and behavior; embed - them into the deck body to keep concerns isolated. + them into the agent body to keep concerns isolated. - Keep the “pinch” (behavior/constraints) closest to the model call, and keep personas above it so they influence style without diluting instructions. - Make the behavior section concrete: steps, priorities, stopping conditions, @@ -29,12 +29,12 @@ the needed context and tone. - Keep schemas tight so actions/tools stay aligned with the narrow middle of the hourglass. -## Skeleton (deck + snippets) +## Skeleton (agent definition + snippets) ``` src/decks/ gambit-assistant.deck.md # root deck embeds the snippets below - cards/ + cards/ # legacy directory name for snippet files assistant_persona.card.md # who the assistant is user_persona.card.md # who the user is / goals / constraints behavior.card.md # steps, guardrails, outputs, when to ask vs act diff --git a/docs/external/concepts/runtime.md b/docs/external/concepts/runtime.md index 6cbcda0ba..c338ba935 100644 --- a/docs/external/concepts/runtime.md +++ b/docs/external/concepts/runtime.md @@ -1,14 +1,15 @@ # Runtime and guardrails -How Gambit, an agent harness framework, runs decks and keeps them -safe/observable. +How Gambit runs native agent definitions and keeps behavior safe, observable, +and reproducible. The runtime still uses `deck` in exact file names, CLI +arguments, config fields, and API names. ## Guardrails and flow - Defaults: `maxDepth=3`, `maxPasses=10`, `timeoutMs≈120s`; override per deck with `guardrails`. -- Compute vs LLM: a deck with `modelParams` runs as LLM; otherwise it must - expose `run`/`execute` for compute. +- Compute vs LLM: an agent definition with `modelParams` runs as LLM; otherwise + it must expose `run`/`execute` for compute. - Non-root decks must declare both `contextSchema` and `responseSchema`; roots allow looser IO but should still use schemas. - Child calls use `actionDecks` (LLM tool calls) or `spawnAndWait` in compute @@ -58,8 +59,8 @@ safe/observable. ## Streaming and tracing -- Streaming is supported for LLM decks; callbacks are invoked per chunk and - handler messages also stream. +- Streaming is supported for LLM agent definitions; callbacks are invoked per + chunk and handler messages also stream. - Tracing: `--verbose` prints trace events; `--trace ` writes JSONL. Useful events include `model.call/result`, `tool.call/result`, and handler triggers. diff --git a/docs/external/examples/README.md b/docs/external/examples/README.md index e479be013..ddb9c7afb 100644 --- a/docs/external/examples/README.md +++ b/docs/external/examples/README.md @@ -18,11 +18,11 @@ Notes: (advanced demos in `.../advanced/`, FAQ bot in `.../faq-bot-example/`). - When `gambit demo` seeds a workspace, generated copies live under `gambit/examples/`. -- Examples show how to use Gambit as an agent harness for workflow execution and - verification. +- Examples show how to author agents, run synthetic scenarios, grade behavior, + inspect traces, and keep regressions reproducible. - Routing examples rely on tight action descriptions and schemas so the model picks the right tool. - Handler examples show how busy/idle status streams and how handled errors - surface as envelopes. + return structured envelopes. - All examples default to `openai/gpt-4o-mini`; override with `--model` or `--model-force` to test other providers. diff --git a/docs/external/examples/agent_with_typescript.md b/docs/external/examples/agent_with_typescript.md index f100f20c9..6b0beee71 100644 --- a/docs/external/examples/agent_with_typescript.md +++ b/docs/external/examples/agent_with_typescript.md @@ -2,21 +2,22 @@ What it shows -- Mixing a Markdown LLM deck with a TypeScript compute action. -- Embedding a card to steer tool usage and reply style. +- Mixing a Markdown LLM agent definition with a TypeScript compute action. +- Embedding a reusable instruction snippet to steer tool usage and reply style. Key files - `init/examples/advanced/agent_with_typescript/agent_with_typescript.deck.md` — - root LLM deck and workflow steps. + root LLM agent definition and workflow steps. - `init/examples/advanced/agent_with_typescript/get_time.deck.ts` — TS compute action (returns ISO timestamp with Zod schemas). - `init/examples/advanced/agent_with_typescript/tooling.card.md` — card that - defines how/when to call the tool and how to respond. + defines how/when to call the tool and how to respond. The `.card.md` suffix is + a legacy compatibility filename for a prompt snippet. Why it’s structured this way -- The card keeps routing guidance separate from the deck body, making it +- The snippet keeps routing guidance separate from the agent body, making it reusable. - The TS action uses strict `contextSchema`/`responseSchema` to guarantee a deterministic payload from compute code. diff --git a/docs/external/examples/handlers_ts.md b/docs/external/examples/handlers_ts.md index 2f4c07030..138c15121 100644 --- a/docs/external/examples/handlers_ts.md +++ b/docs/external/examples/handlers_ts.md @@ -23,7 +23,7 @@ Why it’s structured this way rich envelope with code/status/meta/payload. - `onBusy`/`onIdle` return strings that stream as status updates, showing how handlers don’t block the main run. -- Root prompt nudges the model to prefer `flaky_action_ts` and surface handler +- Root prompt nudges the model to prefer `flaky_action_ts` and show handler messages clearly when things fail. How to run diff --git a/docs/external/getting-started/README.md b/docs/external/getting-started/README.md index b469daff8..062c5069c 100644 --- a/docs/external/getting-started/README.md +++ b/docs/external/getting-started/README.md @@ -1,18 +1,36 @@ # Getting Started -Use this page to get a fast, developer-first path from install to first run. +Use this page to get a fast path from install to your first native Gambit agent, +then to the scenario generation, scenario validation, and grader loop that +catches behavior regressions. ## Quickstart path -1. Run the demo to generate example decks and configs: +1. Run the demo to generate example agent definitions and configs: - `npx @bolt-foundry/gambit demo` -2. Run a deck in the terminal: +2. Run a native Gambit agent definition in the terminal: - `npx @bolt-foundry/gambit run gambit/hello.deck.md --context '"Gambit"'` 3. Open the debug UI to inspect traces: - - `npx @bolt-foundry/gambit serve gambit/hello.deck.md --port 8000` + - `npx @bolt-foundry/gambit-simulator serve gambit/hello.deck.md --port 8000` - Visit `http://localhost:8000/debug` +4. Add a scenario for behavior you care about, check that it is a useful test, + then grade the saved session: + - `npx @bolt-foundry/gambit scenario --test-deck --grade --state .gambit/scenario.json --trace .gambit/scenario.jsonl` + +## What you just set up + +- A native Gambit agent path: Gambit owns the agent definition, local run, + trace, scenario, and grader. +- A regression path: the same scenario and grader command can run in CI as a + behavior check. +- A scenario-quality path: generated or hand-authored scenarios can be reviewed + for realism, coverage, difficulty, grounding, duplication, and expected + outcome clarity before they become regression data. +- A bring-your-own-framework path: for Mastra, LangGraph, OpenAI, or custom + code, keep the production agent where it is and use Gambit scenarios, graders, + traces, and reproduction inputs around the behavior that matters. ## Learn by doing -- Author a simple deck: `../guides/authoring.md` +- Author a simple native Gambit agent: `../guides/authoring.md` - Explore working examples: `../examples/` diff --git a/docs/external/guides/authoring.md b/docs/external/guides/authoring.md index 5497a6bbd..08051bcb9 100644 --- a/docs/external/guides/authoring.md +++ b/docs/external/guides/authoring.md @@ -1,30 +1,40 @@ -# Authoring Gambit decks +# Authoring Native Gambit Agents -Audience: new deck authors who want to build runnable Gambit assistants quickly. -Gambit is an agent harness framework, so decks are the unit of execution and -verification. +Audience: authors who want to build runnable native Gambit agents, create +synthetic scenarios, validate scenario quality, and verify behavior with +graders, traces, and local reproduction. + +Gambit still uses `deck` in file names, CLI syntax, and TypeScript helper names. +Think of a deck as a Gambit agent definition. ## Mental model -- Decks are single units of work. They can be LLM-powered (via `modelParams`) or - compute-only (via `run`/`execute`). -- Snippets are reusable prompt fragments. Embedding snippets merges their deck - references (action/scenario/grader) and schema fragments into the parent deck. -- Action decks are child decks exposed as model tools. Names must match +- Agent definitions are single units of work. They can be LLM-powered (via + `modelParams`) or compute-only (via `run`/`execute`). +- Scenarios describe synthetic users, tasks, personas, workflows, policy + pressure, or edge cases that exercise the behavior you care about. +- Scenario-quality graders check whether generated or hand-authored scenarios + are realistic, novel, grounded, clear, and worth keeping. +- Behavior graders score transcripts or artifacts so scenario runs can become + release checks. +- Snippets are reusable prompt fragments. Embedding snippets merges their + action/scenario/grader references and schema fragments into the parent agent + definition. +- Actions are child agent definitions exposed as model tools. Names must match `^[A-Za-z_][A-Za-z0-9_]*$` and avoid the `gambit_` prefix (reserved). -- Persona/scenario decks may accept free-form user turns. Use the +- Persona/scenario agents may accept free-form user turns. Use the `acceptsUserTurns` flag to control this behavior: root decks default to `true`, while action decks default to `false`. Set it explicitly to `true` for persona/bot decks or to `false` for workflow-only decks. ## Pick a format -- Markdown deck/snippet: great for quick prompt-first flows. Front matter - declares label/model/actions/scenarios/graders/handlers; body is the prompt. - Embeds via image syntax pull in snippets or special markers. -- TypeScript deck/snippet: best when you need compute logic or co-locate - schemas. Export `defineDeck`/`defineCard` with Zod schemas and a - `run`/`execute` for compute decks. +- Markdown agent definition/snippet: great for quick prompt-first flows. Front + matter declares label/model/actions/scenarios/graders/handlers; body is the + prompt. Embeds via image syntax pull in snippets or special markers. +- TypeScript agent definition/snippet: best when you need compute logic or + co-locate schemas. Export `defineDeck`/`defineCard` with Zod schemas and a + `run`/`execute` for compute agent definitions. ## Minimal examples @@ -77,18 +87,19 @@ migrate a repository, run: deno run -A packages/gambit/scripts/migrate-schema-terms.ts ``` -## Action decks, scenario decks, grader decks +## Actions, scenarios, and graders -- Add action decks in front matter or TS definitions: +- Add action references in front matter or TS definitions: `actions = [{ name = "get_time", path = "./get_time.deck.ts" }]`. -- Action decks defined on embedded snippets are merged into the deck; duplicates - are overridden by the deck’s own entries. +- Actions defined on embedded snippets are merged into the agent definition; + duplicates are overridden by the agent definition's own entries. - In compute decks, call child decks with `ctx.spawnAndWait({ path, input })`. - In LLM decks, the model chooses action decks via tool calls. Provide clear descriptions so the model routes correctly. -- `scenarios` describe persona decks (synthetic users/bots). Each entry points - to a deck that produces user turns/scenarios—use them for automated QA, - persona-vs-workflow simulations, or even bot-vs-bot runs. +- `scenarios` describe persona agents (synthetic users/bots). Each entry points + to an agent definition that produces user turns, tasks, edge cases, or + workflow pressure; use them for automated QA, persona-vs-workflow simulations, + or even bot-vs-bot runs. - Example (see `init/examples/advanced/voice_front_desk/decks/root.deck.md`): ```toml [[scenarios]] @@ -101,7 +112,7 @@ deno run -A packages/gambit/scripts/migrate-schema-terms.ts should set `acceptsUserTurns = true` and may declare its own `contextSchema` (for example `contextSchema = "../schemas/my_persona_test.zod.ts"`) so the Test tab renders a schema-driven “Scenario” form for that persona. -- For persona/scenario decks, embed `![init](gambit://snippets/init.md)` to +- For persona/scenario agents, embed `![init](gambit://snippets/init.md)` to include the scenario init-fill contract instructions. - To keep personas in the participant role and standardize completion, embed `![scenario-participant](gambit://snippets/scenario-participant.md)`. @@ -123,8 +134,9 @@ deno run -A packages/gambit/scripts/migrate-schema-terms.ts The persona should respond with **JSON only**, returning values for the missing fields (nested by path). Explicit init values are never overwritten; invalid JSON or schema-violating output blocks the run with a clear error. -- `graderDecks` describe calibration decks that score transcripts/artifacts. The - simulator Calibrate page will run these decks against stored runs. +- `graders` describe calibration agents that score transcripts, artifacts, or + scenario data. The simulator Calibrate page runs these graders against stored + runs. - For graders that inspect assistant tool usage, set `contextSchema = "gambit://schemas/graders/contexts/turn_tools.zod.ts"` so `session.messages[*].tool_calls` is available in the grader input. @@ -134,7 +146,7 @@ deno run -A packages/gambit/scripts/migrate-schema-terms.ts - Markdown roots default to `true`; TypeScript decks default to `false` everywhere. Set it to `false` for any workflow deck that should never accept user turns (regardless of how it's run). - - Persona/scenario decks should set `acceptsUserTurns = true` so they can + - Persona/scenario agents should set `acceptsUserTurns = true` so they can receive messages even when invoked as non-root bots. ## Synthetic tools and handlers diff --git a/docs/external/reference/cli.md b/docs/external/reference/cli.md index 8eb620fe0..cc3c1069e 100644 --- a/docs/external/reference/cli.md +++ b/docs/external/reference/cli.md @@ -1,6 +1,7 @@ # CLI, REPL, and debug UI -How to run Gambit, the agent harness framework, locally and observe runs. +How to run Gambit locally, generate scenario runs, grade behavior, and preserve +the evidence needed to debug regressions. ## Commands @@ -48,7 +49,7 @@ How to run Gambit, the agent harness framework, locally and observe runs. - `GAMBIT_RESPONSES_MODE=1`: env alternative to `--responses` for runtime/state. - `GAMBIT_OPENROUTER_RESPONSES=1`: route OpenRouter calls through the Responses API (experimental; chat remains the default path). -- Worker execution defaults on for deck-executing surfaces. Use +- Worker execution defaults on for CLI commands that execute decks. Use `--no-worker-sandbox` (or `--legacy-exec`) to roll back to legacy in-process execution. `--sandbox/--no-sandbox` still work as deprecated aliases. - `gambit.toml` config equivalent: diff --git a/docs/external/reference/cli/commands/chat.md b/docs/external/reference/cli/commands/chat.md new file mode 100644 index 000000000..670962f4f --- /dev/null +++ b/docs/external/reference/cli/commands/chat.md @@ -0,0 +1,40 @@ ++++ +command = "chat" +summary = "Start a local deck chat server" +usage = "gambit chat [--context ] [--model ] [--model-force ] [--port ] [--state ] [--trace ] [--runtime-tools ...] [--repro-message ] [--responses] [--open] [--verbose] [-A|--allow-all|--allow-] [--worker-sandbox|--no-worker-sandbox|--legacy-exec]" +flags = [ + "--context Context payload (seeds gambit_context; legacy --init still works)", + "--model Default model id", + "--model-force Override model id", + "--port Port for chat (default: 8787)", + "--state Load/persist state", + "--trace Write trace events to file (JSONL)", + "--runtime-tools Markdown/TOML runtime-tool definitions (repeatable)", + "--repro-message Store a repeatable repro prompt in session state", + "--responses Run runtime/state in Responses mode", + "--open Open the localhost chat URL in the browser", + "--verbose Print trace events to console", + "-A, --allow-all Allow all session permissions (read/write/run/net/env)", + "--allow-read[=] Session read override (all when value omitted)", + "--allow-write[=] Session write override (all when value omitted)", + "--allow-run[=] Session run override (all when value omitted)", + "--allow-net[=] Session net override (all when value omitted)", + "--allow-env[=] Session env override (all when value omitted)", + "--worker-sandbox Force worker execution on", + "--no-worker-sandbox Force worker execution off", + "--legacy-exec Alias for --no-worker-sandbox", + "--sandbox Deprecated alias for --worker-sandbox", + "--no-sandbox Deprecated alias for --no-worker-sandbox", +] ++++ + +Starts a focused localhost browser chat for a deck. The command prints the chat +URL and serves a transcript, session panel, tool summaries, and trace events. +Use `--repro-message` to attach a known user ask to the session payload for +repeatable repro flows. It does not send the message automatically. + +Runtime tools are supplied by Markdown files with TOML frontmatter using +`[[tools]]`. Each tool may include `name`, `description`, `inputSchema`, and +`action`. Duplicate tool names fail fast. When a runtime tool is called and an +`action` is configured, the chat server runs that action deck with the tool +arguments as context and returns the action output as the tool result. diff --git a/docs/external/reference/handlers.md b/docs/external/reference/handlers.md index 34f50f2d1..7b74fcd55 100644 --- a/docs/external/reference/handlers.md +++ b/docs/external/reference/handlers.md @@ -1,6 +1,6 @@ # Handlers: error, busy, idle -Gambit decks can register optional handler decks to surface status or recover +Gambit decks can register optional handler decks to report status or recover from failures without blocking the main turn. They run as child decks with structured inputs. diff --git a/docs/external/reference/openai-compat.md b/docs/external/reference/openai-compat.md index 88582293c..eeaae5e35 100644 --- a/docs/external/reference/openai-compat.md +++ b/docs/external/reference/openai-compat.md @@ -1,8 +1,8 @@ # OpenAI Chat Completions compatibility -Gambit, an agent harness framework, can act as a drop-in-ish wrapper around the -OpenAI Chat Completions request/response shape, while still letting you point at -a Gambit deck for system prompt + deck-defined action decks. +Gambit can act as a drop-in-ish wrapper around the OpenAI Chat Completions +request/response shape, while still letting you point at a Gambit deck for +system prompt + deck-defined action decks. This is useful when you already have code that constructs Chat Completions requests and you want to: diff --git a/examples/dev/simpsons_explainer_notest/cards/grader_decks.card.md b/examples/dev/simpsons_explainer_notest/cards/grader_decks.card.md index 09a010dc2..2f5aa7e4f 100644 --- a/examples/dev/simpsons_explainer_notest/cards/grader_decks.card.md +++ b/examples/dev/simpsons_explainer_notest/cards/grader_decks.card.md @@ -1,12 +1,12 @@ +++ label = "demo_grader_decks" -[[graderDecks]] +[[graders]] label = "Brevity (turn)" description = "Checks that the assistant stays concise and avoids unnecessary length." path = "../graders/brevity_turn_grader.deck.md" -[[graderDecks]] +[[graders]] label = "Friendliness (turn)" description = "Checks that the assistant is warm, polite, and approachable." path = "../graders/friendliness_turn_grader.deck.md" diff --git a/examples/dev/simpsons_explainer_user/cards/grader_decks.card.md b/examples/dev/simpsons_explainer_user/cards/grader_decks.card.md index 09a010dc2..2f5aa7e4f 100644 --- a/examples/dev/simpsons_explainer_user/cards/grader_decks.card.md +++ b/examples/dev/simpsons_explainer_user/cards/grader_decks.card.md @@ -1,12 +1,12 @@ +++ label = "demo_grader_decks" -[[graderDecks]] +[[graders]] label = "Brevity (turn)" description = "Checks that the assistant stays concise and avoids unnecessary length." path = "../graders/brevity_turn_grader.deck.md" -[[graderDecks]] +[[graders]] label = "Friendliness (turn)" description = "Checks that the assistant is warm, polite, and approachable." path = "../graders/friendliness_turn_grader.deck.md" diff --git a/examples/dev/simpsons_explainer_user_notest/cards/grader_decks.card.md b/examples/dev/simpsons_explainer_user_notest/cards/grader_decks.card.md index 09a010dc2..2f5aa7e4f 100644 --- a/examples/dev/simpsons_explainer_user_notest/cards/grader_decks.card.md +++ b/examples/dev/simpsons_explainer_user_notest/cards/grader_decks.card.md @@ -1,12 +1,12 @@ +++ label = "demo_grader_decks" -[[graderDecks]] +[[graders]] label = "Brevity (turn)" description = "Checks that the assistant stays concise and avoids unnecessary length." path = "../graders/brevity_turn_grader.deck.md" -[[graderDecks]] +[[graders]] label = "Friendliness (turn)" description = "Checks that the assistant is warm, polite, and approachable." path = "../graders/friendliness_turn_grader.deck.md" diff --git a/examples/local-chat/actions/complete_task.mock.deck.ts b/examples/local-chat/actions/complete_task.mock.deck.ts new file mode 100644 index 000000000..66577c78b --- /dev/null +++ b/examples/local-chat/actions/complete_task.mock.deck.ts @@ -0,0 +1,25 @@ +import { defineDeck } from "../../../mod.ts"; +import { z } from "npm:zod"; + +export default defineDeck({ + label: "complete_task_mock", + contextSchema: z.object({ + summary: z.string().optional(), + }).passthrough(), + responseSchema: z.object({ + status: z.number(), + mocked: z.boolean(), + applied: z.boolean(), + tool: z.literal("complete_task"), + summary: z.string(), + }), + run(ctx) { + return { + status: 200, + mocked: true, + applied: false, + tool: "complete_task", + summary: ctx.input.summary ?? "Task completion captured by mock tool.", + }; + }, +}); diff --git a/examples/local-chat/actions/escalate_task.mock.deck.ts b/examples/local-chat/actions/escalate_task.mock.deck.ts new file mode 100644 index 000000000..8753dc63d --- /dev/null +++ b/examples/local-chat/actions/escalate_task.mock.deck.ts @@ -0,0 +1,27 @@ +import { defineDeck } from "../../../mod.ts"; +import { z } from "npm:zod"; + +export default defineDeck({ + label: "escalate_task_mock", + contextSchema: z.object({ + reason: z.string().optional(), + blocker: z.string().optional(), + }).passthrough(), + responseSchema: z.object({ + status: z.number(), + mocked: z.boolean(), + applied: z.boolean(), + tool: z.literal("escalate_task"), + reason: z.string(), + }), + run(ctx) { + return { + status: 200, + mocked: true, + applied: false, + tool: "escalate_task", + reason: ctx.input.reason ?? ctx.input.blocker ?? + "Escalation captured by mock tool.", + }; + }, +}); diff --git a/examples/local-chat/actions/taxo_lookup_account.mock.deck.ts b/examples/local-chat/actions/taxo_lookup_account.mock.deck.ts new file mode 100644 index 000000000..dda9f767a --- /dev/null +++ b/examples/local-chat/actions/taxo_lookup_account.mock.deck.ts @@ -0,0 +1,33 @@ +import { defineDeck } from "../../../mod.ts"; +import { z } from "npm:zod"; + +export default defineDeck({ + label: "taxo_lookup_account_mock", + contextSchema: z.object({ + customer: z.string().optional(), + domain: z.string().optional(), + }).passthrough(), + responseSchema: z.object({ + status: z.number(), + mocked: z.boolean(), + applied: z.boolean(), + account: z.object({ + id: z.string(), + plan: z.string(), + health: z.string(), + }), + }), + run(ctx) { + const key = ctx.input.customer ?? ctx.input.domain ?? "unknown"; + return { + status: 200, + mocked: true, + applied: false, + account: { + id: `taxo-${key}`, + plan: "representative", + health: "ready_for_local_debug", + }, + }; + }, +}); diff --git a/examples/local-chat/taxo-runtime-tools.mock.md b/examples/local-chat/taxo-runtime-tools.mock.md new file mode 100644 index 000000000..670013ddd --- /dev/null +++ b/examples/local-chat/taxo-runtime-tools.mock.md @@ -0,0 +1,11 @@ ++++ +label = "taxo_runtime_tools_mock" + +[[tools]] +name = "taxo_lookup_account" +description = "Look up a customer account in the Taxo-style support system." +action = "./actions/taxo_lookup_account.mock.deck.ts" ++++ + +Mock runtime tools for local customer-deck chat verification. The deck stays +portable; the launcher supplies this tool surface at runtime. diff --git a/examples/local-chat/taxo-support.deck.md b/examples/local-chat/taxo-support.deck.md new file mode 100644 index 000000000..23de07e77 --- /dev/null +++ b/examples/local-chat/taxo-support.deck.md @@ -0,0 +1,13 @@ ++++ +label = "taxo_support_chat_fixture" + +[modelParams] +model = "codex-cli/default" ++++ + +You are a customer support assistant for a representative external Gambit +customer. + +When the user asks about an account, call `taxo_lookup_account` with the +customer identifier or domain. Use the tool result to answer plainly, and say +when the runtime did not supply the tool or the tool returned an error. diff --git a/examples/local-chat/workloop-runtime-tools.mock.md b/examples/local-chat/workloop-runtime-tools.mock.md new file mode 100644 index 000000000..a1b0c36fc --- /dev/null +++ b/examples/local-chat/workloop-runtime-tools.mock.md @@ -0,0 +1,17 @@ ++++ +label = "workloop_runtime_tools_mock" + +[[tools]] +name = "complete_task" +description = "Mark the current Workloop task complete." +action = "./actions/complete_task.mock.deck.ts" + +[[tools]] +name = "escalate_task" +description = "Escalate or block the current Workloop task." +action = "./actions/escalate_task.mock.deck.ts" ++++ + +Mock Workloop runtime tools for local Chief debugging. These tools are supplied +by the chat launcher so the Chief deck does not need Workloop-specific tool +definitions embedded in the deck source. diff --git a/packages/gambit-core/README.md b/packages/gambit-core/README.md index e7ef1a3b6..3ceeae72c 100644 --- a/packages/gambit-core/README.md +++ b/packages/gambit-core/README.md @@ -24,7 +24,7 @@ can live in any host. [`defineCard`](src/definitions.ts) that enforce Zod `contextSchema`/`responseSchema`. - Loader that understands Markdown decks/cards, inline embeds, and companion - decks (`actionDecks`, `testDecks`, `graderDecks`). + deck references (`[[actions]]`, `[[scenarios]]`, `[[graders]]`). - Guardrail-aware runtime (`runDeck`) that can mix LLM actions and pure compute decks with structured tracing and execution context helpers. - Response-first runtime helpers that plug into any model provider implementing @@ -70,9 +70,9 @@ etc.). spawning child decks. - **Handlers**: Background decks triggered on busy/idle/error intervals. Paths are resolved relative to the parent deck file. -- **Companion decks**: `actionDecks` expose tools (function calls) to the model, - `testDecks` house personas or scripted tests, and `graderDecks` evaluate saved - transcripts. +- **Companion decks**: `[[actions]]` expose tools (function calls) to the model, + `[[scenarios]]` house personas or scripted tests, and `[[graders]]` evaluate + saved transcripts. All actual type definitions live under [`src/types.ts`](src/types.ts). Use them when scripting tooling or writing custom providers. @@ -193,12 +193,16 @@ Embedded cards or system hints can be referenced with markdown image syntax. label: Support Triage contextSchema: ./schemas/triage_input.ts responseSchema: ./schemas/triage_output.ts -actionDecks: - - name: escalate - description: Escalate to a manager - path: ./actions/escalate.deck.md -testDecks: - - path: ./personas/test_bot.deck.md +[[actions]] +name = "escalate" +description = "Escalate to a manager" +path = "./actions/escalate.deck.md" + +[[scenarios]] +path = "./personas/test_bot.deck.md" + +[[graders]] +path = "./graders/support_triage.deck.md" --- ![](gambit://snippets/context.md) @@ -209,11 +213,11 @@ clarifying questions before choosing an action. ``` `loadDeck` normalizes relative paths, merges card fragments, enforces unique -action names, and warns about deprecated fields (`actions`, -`handlers.onInterval`, `intervalMs`). The Markdown loader also injects helper -text for built-in tools like `gambit_context` when you add `gambit://` markers. -Legacy `gambit_respond` and `gambit_end` markers are migration-only and now -hard-fail in default runtime paths. +action names, and warns about deprecated fields (`actionDecks`, `testDecks`, +`graderDecks`, `handlers.onInterval`, `intervalMs`). The Markdown loader also +injects helper text for built-in tools like `gambit_context` when you add +`gambit://` markers. Legacy `gambit_respond` and `gambit_end` markers are +migration-only and now hard-fail in default runtime paths. ## Compatibility and utilities diff --git a/packages/gambit-core/deno.json b/packages/gambit-core/deno.json index 412eacf9c..fbbdc162f 100644 --- a/packages/gambit-core/deno.json +++ b/packages/gambit-core/deno.json @@ -1,6 +1,6 @@ { "name": "@bolt-foundry/gambit-core", - "version": "1.0.0-rc.1", + "version": "1.0.0-rc.2", "description": "Core runtime for Gambit decks.", "license": "Apache-2.0", "repository": { diff --git a/packages/gambit-simulator/deno.json b/packages/gambit-simulator/deno.json index fac1d6b29..7bf758e8f 100644 --- a/packages/gambit-simulator/deno.json +++ b/packages/gambit-simulator/deno.json @@ -1,6 +1,6 @@ { "name": "@bolt-foundry/gambit-simulator", - "version": "1.0.0-rc.1", + "version": "1.0.0-rc.2", "description": "Simulator app and local debug UI for Gambit.", "license": "Apache-2.0", "repository": { diff --git a/packages/gambit-simulator/simulator-ui/demo/gambit-ui-demo-script.md b/packages/gambit-simulator/simulator-ui/demo/gambit-ui-demo-script.md index 314816795..044b33def 100644 --- a/packages/gambit-simulator/simulator-ui/demo/gambit-ui-demo-script.md +++ b/packages/gambit-simulator/simulator-ui/demo/gambit-ui-demo-script.md @@ -20,8 +20,8 @@ Gambit answers one question: what happened, exactly, when this AI system ran. It is open source, deterministic, stateless, and code-first. It produces ground truth artifacts like runs, traces, and grades. -Gambit is not a system of record or a multi-user product. Bolt Foundry stores -and compares artifacts over time. Gambit executes and tells the truth. +Gambit is not a system of record or a multi-user product. Hosted products can +store and compare artifacts over time. Gambit executes and tells the truth. If you can run it, you can inspect it. That is the core promise. diff --git a/packages/gambit-simulator/simulator-ui/src/DocsPage.tsx b/packages/gambit-simulator/simulator-ui/src/DocsPage.tsx index 5004c9348..431f93c97 100644 --- a/packages/gambit-simulator/simulator-ui/src/DocsPage.tsx +++ b/packages/gambit-simulator/simulator-ui/src/DocsPage.tsx @@ -24,8 +24,8 @@ export default function DocsPage() {

Test an agent. See where it fails. Fix it fast.

- Gambit is an open-source agent harness framework. It's designed to - make it simple to create, debug, and fix AI workflows, agents and + Gambit runs synthetic scenarios, grades behavior, and preserves the + evidence you need to debug regressions in AI workflows, agents, and assistants.

@@ -60,7 +60,9 @@ export default function DocsPage() {

If something looks wrong

  • Leave a quick rating and a short note explaining why.
    • -
  • Failures get captured for evaluation.
    • +
  • + Failures get captured as evidence for evaluation and replay. +

Grade the agent

    diff --git a/scaffolds/demo/examples/advanced/agent_with_multi_actions/PROMPT.md b/scaffolds/demo/examples/advanced/agent_with_multi_actions/PROMPT.md index 0f3b0eb99..cbe6b9ef0 100644 --- a/scaffolds/demo/examples/advanced/agent_with_multi_actions/PROMPT.md +++ b/scaffolds/demo/examples/advanced/agent_with_multi_actions/PROMPT.md @@ -8,7 +8,7 @@ temperature = 0 label = "Multi-actions scenario" path = "./tests/agent_with_multi_actions_test.deck.md" description = "Synthetic user that requests a simple translation." -[[graderDecks]] +[[graders]] label = "Action routing guard (turn)" path = "./graders/action_routing_guard_llm.deck.md" description = "Grades each assistant turn for single-action routing and result use." diff --git a/scaffolds/demo/examples/advanced/agent_with_typescript/PROMPT.md b/scaffolds/demo/examples/advanced/agent_with_typescript/PROMPT.md index 04157f05f..d32353ca2 100644 --- a/scaffolds/demo/examples/advanced/agent_with_typescript/PROMPT.md +++ b/scaffolds/demo/examples/advanced/agent_with_typescript/PROMPT.md @@ -9,7 +9,7 @@ description = "Return the current ISO timestamp." label = "Typescript agent scenario" path = "./tests/agent_with_typescript_test.deck.md" description = "Synthetic user that asks for the current time." -[[graderDecks]] +[[graders]] label = "Time greeting guard" path = "./graders/time_greeting_guard_llm.deck.md" description = "Checks get_time usage, timestamp echo, and brief reply." diff --git a/scaffolds/demo/examples/advanced/arena_chatbot/PROMPT.md b/scaffolds/demo/examples/advanced/arena_chatbot/PROMPT.md index 17c795d65..a801f95bf 100644 --- a/scaffolds/demo/examples/advanced/arena_chatbot/PROMPT.md +++ b/scaffolds/demo/examples/advanced/arena_chatbot/PROMPT.md @@ -9,7 +9,7 @@ temperature = 0.7 label = "Arena challenger" path = "./tests/arena_challenger.deck.md" description = "Synthetic user that asks about a Pokemon and requests a tip." -[[graderDecks]] +[[graders]] label = "Pokemon response guard" path = "./graders/pokemon_response_guard_llm.deck.md" description = "Checks empty prompt handling and concise Pokemon tips." diff --git a/scaffolds/demo/examples/advanced/cli_cold_emailer/lookup_profile.deck.ts b/scaffolds/demo/examples/advanced/cli_cold_emailer/lookup_profile.deck.ts index 7f9695a84..9b3515461 100644 --- a/scaffolds/demo/examples/advanced/cli_cold_emailer/lookup_profile.deck.ts +++ b/scaffolds/demo/examples/advanced/cli_cold_emailer/lookup_profile.deck.ts @@ -17,9 +17,9 @@ type Profile = { const profiles: Record = { randall: { - summary: "Co-founder at Bolt Foundry building AI telemetry and analytics.", + summary: "Co-founder at Northstar AI Labs building eval data tooling.", role: "Co-founder", - company: "Bolt Foundry", + company: "Northstar AI Labs", industry: "AI tooling", challenges: ["Proving LLM quality", "Scaling AI reliability"], initiatives: ["Telemetry SDK rollout", "Developer adoption"], diff --git a/scaffolds/demo/examples/advanced/cli_cold_emailer/sample_input.json b/scaffolds/demo/examples/advanced/cli_cold_emailer/sample_input.json index 2a079ccd1..b01f0dae8 100644 --- a/scaffolds/demo/examples/advanced/cli_cold_emailer/sample_input.json +++ b/scaffolds/demo/examples/advanced/cli_cold_emailer/sample_input.json @@ -1,6 +1,6 @@ { "name": "Randall Bennett", - "details": "Co-founder at Bolt Foundry. Reaching out about PostHog and product analytics.", + "details": "Co-founder at Northstar AI Labs. Reaching out about PostHog and product analytics.", "sender": "James@PostHog", "products": [ "Product analytics", diff --git a/scaffolds/demo/examples/advanced/patient_swapper/PROMPT.md b/scaffolds/demo/examples/advanced/patient_swapper/PROMPT.md index 60683d27c..7b51d3c3a 100644 --- a/scaffolds/demo/examples/advanced/patient_swapper/PROMPT.md +++ b/scaffolds/demo/examples/advanced/patient_swapper/PROMPT.md @@ -3,7 +3,7 @@ label = "patient_swapper" contextSchema = "./schemas/patient_swapper_input.zod.ts" modelParams = { model = "openai/gpt-4o-mini", temperature = 0 } guardrails = { maxPasses = 20 } -[[graderDecks]] +[[graders]] label = "Tool sequence guard" path = "./graders/tool_sequence_guard_llm.deck.md" description = "Checks tool call order and confirmation details." diff --git a/scaffolds/demo/examples/advanced/policy_support_bot/PROMPT.md b/scaffolds/demo/examples/advanced/policy_support_bot/PROMPT.md index c1f578d87..effc24ee5 100644 --- a/scaffolds/demo/examples/advanced/policy_support_bot/PROMPT.md +++ b/scaffolds/demo/examples/advanced/policy_support_bot/PROMPT.md @@ -9,7 +9,7 @@ description = "Retrieve the most relevant AcmeFlow FAQ entries with confidence s label = "New account persona" path = "./tests/new_account_persona.deck.md" description = "Synthetic user that asks an in-scope FAQ question." -[[graderDecks]] +[[graders]] label = "FAQ grounding guard" path = "./graders/faq_grounding_guard_llm.deck.md" description = "Ensures answers are grounded to FAQ results or refuse when missing." diff --git a/scaffolds/demo/examples/advanced/simpsons_explainer/cards/grader_decks.card.md b/scaffolds/demo/examples/advanced/simpsons_explainer/cards/grader_decks.card.md index 09a010dc2..2f5aa7e4f 100644 --- a/scaffolds/demo/examples/advanced/simpsons_explainer/cards/grader_decks.card.md +++ b/scaffolds/demo/examples/advanced/simpsons_explainer/cards/grader_decks.card.md @@ -1,12 +1,12 @@ +++ label = "demo_grader_decks" -[[graderDecks]] +[[graders]] label = "Brevity (turn)" description = "Checks that the assistant stays concise and avoids unnecessary length." path = "../graders/brevity_turn_grader.deck.md" -[[graderDecks]] +[[graders]] label = "Friendliness (turn)" description = "Checks that the assistant is warm, polite, and approachable." path = "../graders/friendliness_turn_grader.deck.md" diff --git a/scaffolds/demo/examples/advanced/voice_front_desk/cards/grader_decks.card.md b/scaffolds/demo/examples/advanced/voice_front_desk/cards/grader_decks.card.md index 55a6a5c1f..825cb64fb 100644 --- a/scaffolds/demo/examples/advanced/voice_front_desk/cards/grader_decks.card.md +++ b/scaffolds/demo/examples/advanced/voice_front_desk/cards/grader_decks.card.md @@ -1,18 +1,18 @@ +++ label = "grader_decks" -[[graderDecks]] +[[graders]] label = "Fact verifier (conversation)" path = "../graders/fact_verifier_llm.deck.md" description = "Fails if any assistant fact in the conversation lacks explicit tool-call proof." -[[graderDecks]] +[[graders]] label = "Fact verifier (turn)" path = "../graders/fact_verifier_turn_llm.deck.md" description = "Fails if the latest assistant message states a fact without explicit tool-call proof." -[[graderDecks]] +[[graders]] label = "Tone human-likeness" path = "../graders/tone_human_likeness_llm.deck.md" description = "Scores how human and natural the assistant tone feels." -[[graderDecks]] +[[graders]] label = "Tone human-likeness (turn)" path = "../graders/tone_human_likeness_turn_llm.deck.md" description = "Scores how human and natural the graded assistant message feels." diff --git a/scaffolds/demo/examples/advanced/voice_front_desk/schemas/voice_call_input.zod.ts b/scaffolds/demo/examples/advanced/voice_front_desk/schemas/voice_call_input.zod.ts index e630cb16e..58baa7a8a 100644 --- a/scaffolds/demo/examples/advanced/voice_front_desk/schemas/voice_call_input.zod.ts +++ b/scaffolds/demo/examples/advanced/voice_front_desk/schemas/voice_call_input.zod.ts @@ -13,11 +13,11 @@ export default z.object({ "ISO date for 'today' so the assistant can reference it in speech", ).default(todayIso), clinicName: z.string().describe("The clinic or organization name").default( - "Bolt Foundry Clinic", + "Maple Grove Clinic", ), initialGreeting: z.string().describe( "First line the assistant should use when answering", ).default( - "Hi and thanks for calling the Bolt Foundry Clinic. You've reached our after hours line, and I'm an AI assistant. How can I help you today?", + "Hi and thanks for calling Maple Grove Clinic. You've reached our after hours line, and I'm an AI assistant. How can I help you today?", ), }); diff --git a/scaffolds/demo/examples/faq-bot-example/after/faq_bot_after.deck.md b/scaffolds/demo/examples/faq-bot-example/after/faq_bot_after.deck.md index 1e8d5108e..d4d3d82ab 100644 --- a/scaffolds/demo/examples/faq-bot-example/after/faq_bot_after.deck.md +++ b/scaffolds/demo/examples/faq-bot-example/after/faq_bot_after.deck.md @@ -15,27 +15,27 @@ label = "Voice modules persona" path = "../tests/modules_voice_persona.deck.md" description = "Voice caller asking about Gambit modules and demanding plain sentences." -[[graderDecks]] +[[graders]] label = "Voice assistant tone (conversation)" path = "../graders/voice_assistant_tone_llm.deck.md" description = "Checks for spoken, plain-text voice assistant tone." -[[graderDecks]] +[[graders]] label = "Voice assistant tone (turn)" path = "../graders/voice_assistant_tone_turn_llm.deck.md" description = "Checks the graded turn for spoken, plain-text voice assistant tone." -[[graderDecks]] +[[graders]] label = "Voice readability guard" path = "../graders/voice_readability_llm.deck.md" description = "Ensures responses can be read aloud (no screen-only formatting)." -[[graderDecks]] +[[graders]] label = "Fact verifier (conversation)" path = "../graders/fact_verifier_llm.deck.md" description = "Ensures all factual claims are grounded in FAQ tool output." -[[graderDecks]] +[[graders]] label = "Fact verifier (turn)" path = "../graders/fact_verifier_turn_llm.deck.md" description = "Ensures graded turn facts are grounded in FAQ tool output." diff --git a/scaffolds/demo/examples/faq-bot-example/before/faq_bot_before.deck.md b/scaffolds/demo/examples/faq-bot-example/before/faq_bot_before.deck.md index 2516a7bd2..07d74ff02 100644 --- a/scaffolds/demo/examples/faq-bot-example/before/faq_bot_before.deck.md +++ b/scaffolds/demo/examples/faq-bot-example/before/faq_bot_before.deck.md @@ -15,27 +15,27 @@ label = "Voice modules persona" path = "../tests/modules_voice_persona.deck.md" description = "Voice caller asking about Gambit modules and demanding plain sentences." -[[graderDecks]] +[[graders]] label = "Voice assistant tone (conversation)" path = "../graders/voice_assistant_tone_llm.deck.md" description = "Checks for spoken, plain-text voice assistant tone." -[[graderDecks]] +[[graders]] label = "Voice assistant tone (turn)" path = "../graders/voice_assistant_tone_turn_llm.deck.md" description = "Checks the graded turn for spoken, plain-text voice assistant tone." -[[graderDecks]] +[[graders]] label = "Voice readability guard" path = "../graders/voice_readability_llm.deck.md" description = "Ensures responses can be read aloud (no screen-only formatting)." -[[graderDecks]] +[[graders]] label = "Fact verifier (conversation)" path = "../graders/fact_verifier_llm.deck.md" description = "Ensures all factual claims are grounded in FAQ tool output." -[[graderDecks]] +[[graders]] label = "Fact verifier (turn)" path = "../graders/fact_verifier_turn_llm.deck.md" description = "Ensures graded turn facts are grounded in FAQ tool output." diff --git a/scaffolds/demo/hello.deck.md b/scaffolds/demo/hello.deck.md index c66e73d46..bce54ada5 100644 --- a/scaffolds/demo/hello.deck.md +++ b/scaffolds/demo/hello.deck.md @@ -5,7 +5,7 @@ modelParams = { model = "openai/gpt-4o-mini", temperature = 0 } label = "Hello scenario" path = "./hello.test.deck.md" description = "Synthetic user that sends a single greeting input." -[[graderDecks]] +[[graders]] label = "Hello echo guard" path = "./hello.grader.deck.md" description = "Enforces the exact hello echo responses." diff --git a/src/cli.ts b/src/cli.ts index 14bc39d4c..2a3c8ca34 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -11,6 +11,7 @@ import { makeConsoleTracer, makeJsonlTracer } from "./trace.ts"; import { startTui } from "./tui.ts"; import type { PermissionDeclarationInput } from "@bolt-foundry/gambit-core"; import { handleCheckCommand } from "./commands/check.ts"; +import { handleChatCommand } from "./commands/chat.ts"; import { handleRunCommand } from "./commands/run.ts"; import { runTestBotLoop } from "./commands/test_bot.ts"; import { runGraderAgainstState } from "./commands/grade.ts"; @@ -443,6 +444,30 @@ async function main() { return; } + if (args.cmd === "chat") { + await handleChatCommand({ + deckPath, + context: args.context, + contextProvided: args.contextProvided, + modelProvider: provider, + model: args.model, + modelForce: args.modelForce, + port: args.port, + statePath: args.statePath, + tracePath: args.trace, + reproMessage: args.reproMessage, + runtimeToolsPaths: args.runtimeToolsPaths, + responsesMode, + open: args.open, + workspacePermissions, + workspacePermissionsBaseDir: projectConfig?.root, + sessionPermissions, + sessionPermissionsBaseDir, + workerSandbox: args.workerSandbox ?? workerSandboxFromConfig, + }); + return; + } + if (args.cmd === "repl") { if (!Deno.stdin.isTerminal()) { if (args.message === undefined) { diff --git a/src/cli_args.test.ts b/src/cli_args.test.ts index 6f947301e..9a2cf0d8f 100644 --- a/src/cli_args.test.ts +++ b/src/cli_args.test.ts @@ -99,6 +99,35 @@ Deno.test("parseCliArgs parses yolo flag", () => { assertEquals(args.yolo, true); }); +Deno.test("parseCliArgs parses chat runtime tools and open flag", () => { + const args = parseCliArgs([ + "chat", + "root.deck.md", + "--runtime-tools", + "./workloop-tools.md", + "--runtime-tools", + "./taxo-tools.md", + "--repro-message", + "original user ask", + "--open", + "--port", + "9797", + ]); + assertEquals(args.cmd, "chat"); + assertEquals(args.deckPath, "root.deck.md"); + assertEquals(args.runtimeToolsPaths, [ + "./workloop-tools.md", + "./taxo-tools.md", + ]); + assertEquals(args.open, true); + assertEquals(args.port, 9797); + assertEquals(args.reproMessage, "original user ask"); +}); + +Deno.test("CLI command registry exposes chat", () => { + assertEquals(isKnownCommand("chat"), true); +}); + Deno.test("parseCliArgs rejects conflicting worker flags", () => { assertThrows( () => diff --git a/src/cli_args.ts b/src/cli_args.ts index 6d54adfe3..d66ea7f5e 100644 --- a/src/cli_args.ts +++ b/src/cli_args.ts @@ -9,6 +9,7 @@ let workerSandboxAliasWarningShown = false; const COMMANDS = [ "bot", + "chat", "check", "demo", "run", @@ -27,6 +28,7 @@ function isKnownCommand(cmd?: string): cmd is Command { const HELP_COMMANDS = [ "bot", + "chat", "check", "demo", "run", @@ -65,11 +67,14 @@ type Args = { statePath?: string; outPath?: string; artifactPath?: string; + reproMessage?: string; + runtimeToolsPaths?: Array; verbose?: boolean; online?: boolean; json?: boolean; port?: number; watch?: boolean; + open?: boolean; bundle?: boolean; sourcemap?: boolean; platform?: string; @@ -126,6 +131,8 @@ const STRING_OPTION_FLAGS = [ "state", "out", "artifact", + "repro-message", + "runtime-tools", "port", ] as const; const OPTION_VALUE_FLAGS = new Set( @@ -427,11 +434,13 @@ export function parseCliArgs(argv: Array): Args { "help", "version", "watch", + "open", "bundle", "no-bundle", "sourcemap", ], string: [...STRING_OPTION_FLAGS], + collect: ["grade", "runtime-tools"], alias: { help: "h", version: "V", @@ -522,11 +531,16 @@ export function parseCliArgs(argv: Array): Args { statePath: parsed.state as string | undefined, outPath: parsed.out as string | undefined, artifactPath: parsed.artifact as string | undefined, + reproMessage: parsed["repro-message"] as string | undefined, + runtimeToolsPaths: normalizeFlagList( + parsed["runtime-tools"] as string | Array | undefined, + ), verbose: Boolean(parsed.verbose), online: Boolean(parsed.online), json: Boolean(parsed.json), port: parsePortValue(parsed.port), watch: Boolean(parsed.watch), + open: Boolean(parsed.open), bundle: hasNoBundleFlag ? false : hasBundleFlag ? true : undefined, sourcemap: hasNoSourceMapFlag ? false : hasSourceMapFlag ? true : undefined, platform: parsed.platform as string | undefined, diff --git a/src/cli_help.test.ts b/src/cli_help.test.ts index 2f4f40791..48ec1426e 100644 --- a/src/cli_help.test.ts +++ b/src/cli_help.test.ts @@ -28,6 +28,7 @@ Deno.test({ assert(output.includes("gambit [options]")); assert(output.includes("gambit help ")); assert(output.includes("bot Run the Gambit bot assistant")); + assert(output.includes("chat Start a local deck chat server")); assert(output.includes("run Run a deck once")); assert(output.includes("scenario Run a scenario loop with a persona deck")); assert(!output.includes("init")); @@ -43,9 +44,21 @@ Deno.test({ }); assert(output.includes("Details:")); assert(output.includes("Usage:\n gambit run")); + assert(output.includes("Usage:\n gambit chat")); assert(output.includes("--state ")); }); +Deno.test({ + name: "CLI help output renders chat command usage", + permissions: { read: true }, +}, () => { + const output = captureConsoleLog(() => { + printCommandUsage("chat"); + }); + assert(output.includes("Usage:\n gambit chat")); + assert(output.includes("--runtime-tools ")); +}); + Deno.test({ name: "CLI help output renders command usage", permissions: { read: true }, diff --git a/src/cli_utils.test.ts b/src/cli_utils.test.ts index 47e2ce49a..401abc583 100644 --- a/src/cli_utils.test.ts +++ b/src/cli_utils.test.ts @@ -16,7 +16,7 @@ Deno.test("defaultSessionRoot anchors restored artifact decks to workspace root" "a337dcb7-6d06-4b01-adb9-7154366d1ba1", "deck", "apps", - "boltfoundry-com", + "getworkloop-com", "gambit", "faq", "decks", diff --git a/src/codex_app_server_debug.ts b/src/codex_app_server_debug.ts index f32e6f9a6..04d23c6b4 100644 --- a/src/codex_app_server_debug.ts +++ b/src/codex_app_server_debug.ts @@ -1,7 +1,4 @@ -const CODEX_APP_SERVER_DEBUG_ENV = - "WORKLOOP_CHIEF_RUNTIME_DEBUG_CODEX_APP_SERVER"; -const LEGACY_CODEX_APP_SERVER_DEBUG_ENV = - "BOLT_FOUNDRY_DESKTOP_CHIEF_RUNTIME_DEBUG_CODEX_APP_SERVER"; +const CODEX_APP_SERVER_DEBUG_ENV = "GAMBIT_CODEX_APP_SERVER_DEBUG"; const STRUCTURAL_STRING_KEYS = new Set([ "error", @@ -86,8 +83,7 @@ function summarizeDebugValue( } export function shouldDebugCodexAppServer(): boolean { - return parseTruthy(Deno.env.get(CODEX_APP_SERVER_DEBUG_ENV)) || - parseTruthy(Deno.env.get(LEGACY_CODEX_APP_SERVER_DEBUG_ENV)); + return parseTruthy(Deno.env.get(CODEX_APP_SERVER_DEBUG_ENV)); } export function logCodexAppServerDebug( @@ -103,4 +99,4 @@ export function summarizeCodexAppServerDebugValue(value: unknown): DebugValue { return summarizeDebugValue(value); } -export { CODEX_APP_SERVER_DEBUG_ENV, LEGACY_CODEX_APP_SERVER_DEBUG_ENV }; +export { CODEX_APP_SERVER_DEBUG_ENV }; diff --git a/src/codex_auth.test.ts b/src/codex_auth.test.ts index 1dbbe9e7d..ae5af3277 100644 --- a/src/codex_auth.test.ts +++ b/src/codex_auth.test.ts @@ -1,7 +1,6 @@ import { assertEquals } from "@std/assert"; import { CODEX_HOST_AUTH_BUNDLE_ENV, - LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV, readCodexAuthBundleFromEnv, } from "./codex_auth.ts"; @@ -16,10 +15,8 @@ const TEST_BUNDLE = JSON.stringify({ function withRestoredAuthBundleEnv(fn: () => void) { const priorBundle = Deno.env.get(CODEX_HOST_AUTH_BUNDLE_ENV); - const priorLegacyBundle = Deno.env.get(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV); try { Deno.env.delete(CODEX_HOST_AUTH_BUNDLE_ENV); - Deno.env.delete(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV); fn(); } finally { if (priorBundle === undefined) { @@ -27,15 +24,10 @@ function withRestoredAuthBundleEnv(fn: () => void) { } else { Deno.env.set(CODEX_HOST_AUTH_BUNDLE_ENV, priorBundle); } - if (priorLegacyBundle === undefined) { - Deno.env.delete(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV); - } else { - Deno.env.set(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV, priorLegacyBundle); - } } } -Deno.test("codex auth bundle env reads the workloop host bundle env", () => { +Deno.test("codex auth bundle env reads the Gambit auth bundle env", () => { withRestoredAuthBundleEnv(() => { Deno.env.set(CODEX_HOST_AUTH_BUNDLE_ENV, TEST_BUNDLE); @@ -45,14 +37,3 @@ Deno.test("codex auth bundle env reads the workloop host bundle env", () => { assertEquals(bundle?.accessToken, "access-token"); }); }); - -Deno.test("codex auth bundle env falls back to the legacy desktop bundle env", () => { - withRestoredAuthBundleEnv(() => { - Deno.env.set(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV, TEST_BUNDLE); - - const bundle = readCodexAuthBundleFromEnv(); - - assertEquals(bundle?.chatgptAccountId, "acct-auth-env"); - assertEquals(bundle?.accessToken, "access-token"); - }); -}); diff --git a/src/codex_auth.ts b/src/codex_auth.ts index 7d6cf113c..f616f9c8b 100644 --- a/src/codex_auth.ts +++ b/src/codex_auth.ts @@ -1,9 +1,7 @@ const CODEX_AUTH0_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann"; const CODEX_AUTH0_TOKEN_URL = "https://auth.openai.com/oauth/token"; -export const CODEX_HOST_AUTH_BUNDLE_ENV = "WORKLOOP_CODEX_AUTH_BUNDLE"; -export const LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV = - "BOLT_FOUNDRY_DESKTOP_CODEX_AUTH_BUNDLE"; +export const CODEX_HOST_AUTH_BUNDLE_ENV = "GAMBIT_CODEX_AUTH_BUNDLE"; export type CodexChatgptAuthTokens = { accessToken: string; @@ -175,8 +173,7 @@ export function serializeCodexAuthBundle( } export function readCodexAuthBundleFromEnv(): CodexChatgptAuthTokens | null { - const raw = Deno.env.get(CODEX_HOST_AUTH_BUNDLE_ENV)?.trim() || - Deno.env.get(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV)?.trim(); + const raw = Deno.env.get(CODEX_HOST_AUTH_BUNDLE_ENV)?.trim(); if (!raw) return null; return parseCodexAuthBundle(raw); } diff --git a/src/codex_preflight.test.ts b/src/codex_preflight.test.ts index 54e000fbc..72a684d7f 100644 --- a/src/codex_preflight.test.ts +++ b/src/codex_preflight.test.ts @@ -1,9 +1,6 @@ import { assertEquals, assertStringIncludes } from "@std/assert"; import { join } from "@std/path"; -import { - CODEX_HOST_AUTH_BUNDLE_ENV, - LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV, -} from "./codex_auth.ts"; +import { CODEX_HOST_AUTH_BUNDLE_ENV } from "./codex_auth.ts"; import { MINIMUM_SUPPORTED_CODEX_CLI_VERSION, readCodexLoginStatus, @@ -424,7 +421,6 @@ done Deno.test("codex preflight falls back to login status when no host auth bundle is present", async () => { const priorBin = Deno.env.get("GAMBIT_CODEX_BIN"); const priorBundle = Deno.env.get(CODEX_HOST_AUTH_BUNDLE_ENV); - const priorLegacyBundle = Deno.env.get(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV); const root = await Deno.makeTempDir({ prefix: "codex-preflight-legacy-" }); const fakeCodexPath = join(root, "fake-codex"); @@ -444,7 +440,6 @@ exit 64 Deno.env.set("GAMBIT_CODEX_BIN", fakeCodexPath); Deno.env.delete(CODEX_HOST_AUTH_BUNDLE_ENV); - Deno.env.delete(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV); try { const status = await readCodexLoginStatus(); @@ -461,11 +456,6 @@ exit 64 } else { Deno.env.set(CODEX_HOST_AUTH_BUNDLE_ENV, priorBundle); } - if (priorLegacyBundle == null) { - Deno.env.delete(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV); - } else { - Deno.env.set(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV, priorLegacyBundle); - } await Deno.remove(root, { recursive: true }).catch(() => undefined); } }); @@ -473,7 +463,6 @@ exit 64 Deno.test("codex preflight rejects unsupported codex cli versions before login checks", async () => { const priorBin = Deno.env.get("GAMBIT_CODEX_BIN"); const priorBundle = Deno.env.get(CODEX_HOST_AUTH_BUNDLE_ENV); - const priorLegacyBundle = Deno.env.get(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV); const root = await Deno.makeTempDir({ prefix: "codex-preflight-unsupported-version-", }); @@ -495,7 +484,6 @@ exit 64 Deno.env.set("GAMBIT_CODEX_BIN", fakeCodexPath); Deno.env.delete(CODEX_HOST_AUTH_BUNDLE_ENV); - Deno.env.delete(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV); try { const status = await readCodexLoginStatus(); @@ -517,11 +505,6 @@ exit 64 } else { Deno.env.set(CODEX_HOST_AUTH_BUNDLE_ENV, priorBundle); } - if (priorLegacyBundle == null) { - Deno.env.delete(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV); - } else { - Deno.env.set(LEGACY_CODEX_HOST_AUTH_BUNDLE_ENV, priorLegacyBundle); - } await Deno.remove(root, { recursive: true }).catch(() => undefined); } }); diff --git a/src/commands/chat.test.ts b/src/commands/chat.test.ts new file mode 100644 index 000000000..2f63180df --- /dev/null +++ b/src/commands/chat.test.ts @@ -0,0 +1,943 @@ +import { + assert, + assertEquals, + assertNotEquals, + assertStringIncludes, + assertThrows, +} from "@std/assert"; +import * as path from "@std/path"; +import type { ModelProvider, ResponseItem } from "@bolt-foundry/gambit-core"; +import { loadRuntimeTools, startLocalChatServer } from "./chat.ts"; +import { saveCanonicalWorkspaceState } from "../workspace_sqlite.ts"; + +async function writeTempFile( + dir: string, + relative: string, + contents: string, +): Promise { + const target = path.join(dir, relative); + await Deno.mkdir(path.dirname(target), { recursive: true }); + await Deno.writeTextFile(target, contents); + return target; +} + +function modImportPath(): string { + const here = path.dirname(path.fromFileUrl(import.meta.url)); + return path.toFileUrl(path.resolve(here, "..", "..", "mod.ts")).href; +} + +function modelResponse(output: Array) { + return Promise.resolve({ + id: crypto.randomUUID(), + object: "response" as const, + status: "completed" as const, + output, + }); +} + +function assistantText(text: string): ResponseItem { + return { + type: "message", + role: "assistant", + content: [{ type: "output_text", text }], + }; +} + +type ChatSessionTestPayload = { + runId?: string; + firstUserMessage?: string; + lastUserMessage?: string; + transcript: Array<{ role: string; content: string }>; + runtimeTools?: Array<{ name: string }>; + traceEvents?: Array<{ type: string; name?: string; ts?: number }>; + metrics?: { + turnStartedAt?: string; + modelCalledAt?: string; + firstTokenAt?: string; + ttftMs?: number; + modelTtftMs?: number; + }; + errors?: Array; + running?: boolean; +}; + +async function waitForSession( + port: number, + predicate: (session: ChatSessionTestPayload) => boolean, +) { + let lastSession: ChatSessionTestPayload | undefined; + for (let i = 0; i < 100; i++) { + const session = await (await fetch(`http://localhost:${port}/api/session`)) + .json() as ChatSessionTestPayload; + lastSession = session; + if (predicate(session)) return session; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error( + `timed out waiting for chat session; last session: ${ + JSON.stringify(lastSession, null, 2) + }`, + ); +} + +Deno.test("chat server rejects worker sandbox execution while stop requires cancellation", () => { + assertThrows( + () => + startLocalChatServer({ + deckPath: "root.deck.md", + modelProvider: {} as ModelProvider, + workerSandbox: true, + }), + Error, + "gambit chat does not support worker sandbox execution yet", + ); +}); + +Deno.test({ + name: + "chat server runs a deck turn, persists state, and exposes trace/tool details", + permissions: { read: true, write: true, net: true, env: true }, +}, async () => { + const dir = await Deno.makeTempDir({ prefix: "gambit-chat-test-" }); + const deckPath = await writeTempFile( + dir, + "root.deck.md", + `+++ +label = "chat test" + +[modelParams] +model = "test/model" ++++ + +Call runtime_echo once, then summarize the tool result.`, + ); + const runtimeToolsPath = await writeTempFile( + dir, + "runtime-tools.md", + `+++ +[[tools]] +name = "runtime_echo" +description = "Echo a value supplied by the runtime." +action = "./actions/runtime_echo.mock.deck.ts" ++++ + +Runtime tools for chat test.`, + ); + await writeTempFile( + dir, + "actions/runtime_echo.mock.deck.ts", + `import { defineDeck } from "${modImportPath()}"; +import { z } from "npm:zod"; + +export default defineDeck({ + label: "runtime_echo_mock", + contextSchema: z.object({ value: z.string() }), + responseSchema: z.object({ + status: z.number(), + mocked: z.boolean(), + applied: z.boolean(), + value: z.string(), + }), + run(ctx) { + return { + status: 200, + mocked: true, + applied: false, + value: ctx.input.value, + }; + }, +}); +`, + ); + const runtimeTools = await loadRuntimeTools([runtimeToolsPath]); + const tracePath = path.join(dir, "trace.jsonl"); + const statePath = path.join(dir, "workspace.sqlite"); + let callCount = 0; + const provider: ModelProvider = { + responses(input) { + callCount++; + if (callCount === 1) { + assertEquals( + input.request.tools?.some((tool) => + tool.type === "function" && tool.function.name === "runtime_echo" + ), + true, + ); + return modelResponse([{ + type: "function_call", + call_id: "call-runtime-echo", + name: "runtime_echo", + arguments: '{"value":"hello"}', + } as ResponseItem]); + } + assertEquals( + input.request.input.some((item) => + item.type === "function_call_output" && + item.call_id === "call-runtime-echo" && + String(item.output).includes("mocked") + ), + true, + ); + return modelResponse([assistantText("tool result was visible")]); + }, + }; + const abort = new AbortController(); + const port = 19000 + Math.floor(Math.random() * 1000); + const server = startLocalChatServer({ + deckPath, + modelProvider: provider, + port, + statePath, + tracePath, + runtimeTools, + responsesMode: true, + signal: abort.signal, + }); + + try { + const response = await fetch(`http://localhost:${port}/api/message`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ message: "please use the runtime tool" }), + }); + assertEquals(response.status, 202); + await response.text(); + const session = await waitForSession( + port, + (value) => + value.transcript.some((row) => + row.role === "assistant" && row.content.includes("tool result") + ), + ) as { + transcript: Array<{ role: string; content: string }>; + runtimeTools: Array<{ name: string }>; + traceEvents: Array<{ type: string; name?: string }>; + }; + assertEquals(session.runtimeTools[0]?.name, "runtime_echo"); + assertEquals( + session.transcript.some((row) => + row.role === "assistant" && row.content.includes("tool result") + ), + true, + ); + assertEquals( + session.transcript.some((row) => + row.role === "tool" && + row.content.includes("Tool result: runtime_echo") && + row.content.includes("mocked") + ), + true, + ); + assertEquals( + session.traceEvents.some((event) => + event.type === "tool.call" && event.name === "runtime_echo" + ), + true, + ); + const traceText = await Deno.readTextFile(tracePath); + assertStringIncludes(traceText, "runtime_echo"); + const stateInfo = await Deno.stat(statePath); + assert(stateInfo.isFile); + } finally { + abort.abort(); + await server.finished.catch(() => {}); + } +}); + +Deno.test({ + name: + "chat runtime tools preserve JSON object payloads without truthy status", + permissions: { read: true, write: true, net: true, env: true }, +}, async () => { + const dir = await Deno.makeTempDir({ + prefix: "gambit-chat-tool-json-test-", + }); + const deckPath = await writeTempFile( + dir, + "root.deck.md", + `+++ +label = "chat runtime tool json test" + +[modelParams] +model = "test/model" ++++ + +Call both supplied runtime tools.`, + ); + const runtimeToolsPath = await writeTempFile( + dir, + "runtime-tools.md", + `+++ +[[tools]] +name = "runtime_zero" +description = "Return structured JSON with status 0." +action = "./actions/runtime_zero.mock.deck.ts" + +[[tools]] +name = "runtime_plain" +description = "Return structured JSON without status." +action = "./actions/runtime_plain.mock.deck.ts" ++++ + +Runtime tools for JSON payload preservation.`, + ); + await writeTempFile( + dir, + "actions/runtime_zero.mock.deck.ts", + `import { defineDeck } from "${modImportPath()}"; + +export default defineDeck({ + label: "runtime_zero_mock", + run() { + return { + status: 0, + ok: true, + value: "zero", + }; + }, +}); +`, + ); + await writeTempFile( + dir, + "actions/runtime_plain.mock.deck.ts", + `import { defineDeck } from "${modImportPath()}"; + +export default defineDeck({ + label: "runtime_plain_mock", + run() { + return { + ok: true, + value: "plain", + }; + }, +}); +`, + ); + const runtimeTools = await loadRuntimeTools([runtimeToolsPath]); + let callCount = 0; + const provider: ModelProvider = { + responses(input) { + callCount++; + if (callCount === 1) { + return modelResponse([ + { + type: "function_call", + call_id: "call-runtime-zero", + name: "runtime_zero", + arguments: "{}", + } as ResponseItem, + { + type: "function_call", + call_id: "call-runtime-plain", + name: "runtime_plain", + arguments: "{}", + } as ResponseItem, + ]); + } + const outputByCallId = new Map(); + for (const item of input.request.input) { + if (item.type === "function_call_output") { + outputByCallId.set(item.call_id, String(item.output)); + } + } + const zero = JSON.parse(outputByCallId.get("call-runtime-zero") ?? "{}"); + const plain = JSON.parse( + outputByCallId.get("call-runtime-plain") ?? "{}", + ); + assertEquals(zero.status, 0); + assertEquals(zero.payload?.value, "zero"); + assertEquals(plain.payload?.value, "plain"); + return modelResponse([assistantText("structured tool payloads visible")]); + }, + }; + const abort = new AbortController(); + const port = 19000 + Math.floor(Math.random() * 1000); + const server = startLocalChatServer({ + deckPath, + modelProvider: provider, + port, + runtimeTools, + responsesMode: true, + signal: abort.signal, + }); + + try { + const response = await fetch(`http://localhost:${port}/api/message`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ message: "please use the runtime tools" }), + }); + assertEquals(response.status, 202); + await response.text(); + await waitForSession( + port, + (value) => + value.transcript.some((row) => + row.role === "assistant" && + row.content.includes("structured tool payloads visible") + ), + ); + } finally { + abort.abort(); + await server.finished.catch(() => {}); + } +}); + +Deno.test({ + name: "chat server exposes repro message and shell affordances", + permissions: { read: true, write: true, net: true, env: true }, +}, async () => { + const dir = await Deno.makeTempDir({ prefix: "gambit-chat-shell-test-" }); + const deckPath = await writeTempFile( + dir, + "root.deck.md", + `+++ +label = "chat shell test" ++++ + +Echo the user.`, + ); + const provider: ModelProvider = { + responses() { + return modelResponse([assistantText("ok")]); + }, + }; + const abort = new AbortController(); + const port = 20000 + Math.floor(Math.random() * 1000); + const reproMessage = "original user ask"; + const server = startLocalChatServer({ + deckPath, + modelProvider: provider, + port, + reproMessage, + signal: abort.signal, + }); + + try { + const html = await (await fetch(`http://localhost:${port}/`)).text(); + assertStringIncludes(html, "Deck Chat Repro"); + assertStringIncludes(html, 'data-tab="run"'); + assertStringIncludes(html, "New run"); + assertStringIncludes(html, "Run again"); + assertStringIncludes(html, "Stop"); + assertStringIncludes(html, "/api/session/stop"); + assertStringIncludes(html, "requestSubmit"); + assertStringIncludes(html, "EventSource"); + assertStringIncludes(html, "/api/session/events"); + assertStringIncludes(html, "liveTurnRows"); + assertStringIncludes(html, "Tool call:"); + assertStringIncludes(html, "Tool result:"); + assertStringIncludes(html, "response.output_text.delta"); + assertStringIncludes(html, "formatMs"); + assertStringIncludes(html, "Model TTFT"); + assertStringIncludes(html, "curatedEvents"); + assertStringIncludes(html, "messageText(event.message)"); + assertStringIncludes(html, "shouldShowTimelineEvent"); + assertStringIncludes(html, "pendingTranscript"); + assertStringIncludes(html, " - sending"); + assertStringIncludes(html, ".transcript::before"); + assertStringIncludes(html, "margin-top: auto"); + assertStringIncludes(html, 'class="gds-panel chat"'); + assertEquals(html.includes("[...data.transcript].reverse()"), false); + assertStringIncludes( + html, + "$('transcript').scrollTop = $('transcript').scrollHeight", + ); + assertEquals(html.includes("Use repro message"), false); + const session = await (await fetch(`http://localhost:${port}/api/session`)) + .json() as { reproMessage?: string }; + assertEquals(session.reproMessage, reproMessage); + } finally { + abort.abort(); + await server.finished.catch(() => {}); + } +}); + +Deno.test({ + name: "chat server hydrates transcript from persisted workspace state", + permissions: { read: true, write: true, net: true, env: true }, +}, async () => { + const dir = await Deno.makeTempDir({ + prefix: "gambit-chat-hydrate-test-", + }); + const deckPath = await writeTempFile( + dir, + "root.deck.md", + `+++ +label = "chat hydrate test" + +[modelParams] +model = "test/model" ++++ + +Continue the conversation.`, + ); + const statePath = path.join(dir, "workspace.sqlite"); + saveCanonicalWorkspaceState(statePath, { + runId: "persisted-run", + messages: [ + { role: "user", content: "visible previous ask" }, + { role: "assistant", content: "visible previous answer" }, + ], + }); + let callCount = 0; + const provider: ModelProvider = { + responses(input) { + callCount++; + assertEquals( + input.request.input.some((item) => + item.type === "message" && + item.role === "user" && + item.content.some((content) => + content.type === "input_text" && + content.text.includes("visible previous ask") + ) + ), + true, + ); + return modelResponse([ + assistantText(callCount === 1 ? "continued answer" : "rerun answer"), + ]); + }, + }; + const abort = new AbortController(); + const port = 20450 + Math.floor(Math.random() * 1000); + const server = startLocalChatServer({ + deckPath, + modelProvider: provider, + port, + statePath, + responsesMode: true, + signal: abort.signal, + }); + + try { + const initial = await (await fetch(`http://localhost:${port}/api/session`)) + .json() as ChatSessionTestPayload; + assertEquals(initial.runId, "persisted-run"); + assertEquals(initial.firstUserMessage, "visible previous ask"); + assertEquals(initial.lastUserMessage, "visible previous ask"); + assertEquals( + initial.transcript.map((row) => `${row.role}:${row.content}`), + [ + "user:visible previous ask", + "assistant:visible previous answer", + ], + ); + + const response = await fetch(`http://localhost:${port}/api/message`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ message: "continue" }), + }); + assertEquals(response.status, 202); + await response.text(); + const session = await waitForSession( + port, + (value) => + value.transcript.some((row) => row.content === "continued answer"), + ); + assertEquals( + session.transcript.map((row) => `${row.role}:${row.content}`), + [ + "user:visible previous ask", + "assistant:visible previous answer", + "user:continue", + "assistant:continued answer", + ], + ); + + const reset = await fetch(`http://localhost:${port}/api/session/reset`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ rerun: true }), + }); + assertEquals(reset.status, 202); + await reset.text(); + const rerun = await waitForSession( + port, + (value) => value.transcript.some((row) => row.content === "rerun answer"), + ); + assertEquals( + rerun.transcript.map((row) => `${row.role}:${row.content}`), + [ + "user:visible previous ask", + "assistant:rerun answer", + ], + ); + } finally { + abort.abort(); + await server.finished.catch(() => {}); + } +}); + +Deno.test({ + name: "chat server can stop a running deck turn", + permissions: { read: true, write: true, net: true, env: true }, +}, async () => { + const dir = await Deno.makeTempDir({ prefix: "gambit-chat-stop-test-" }); + const deckPath = await writeTempFile( + dir, + "root.deck.md", + `+++ +label = "chat stop test" + +[modelParams] +model = "test/model" ++++ + +Echo the user slowly.`, + ); + let seenSignal: AbortSignal | undefined; + const provider: ModelProvider = { + responses(input) { + seenSignal = input.signal; + return new Promise((_resolve, reject) => { + input.signal?.addEventListener("abort", () => { + reject(new DOMException("stopped", "AbortError")); + }, { once: true }); + }); + }, + }; + const abort = new AbortController(); + const port = 20350 + Math.floor(Math.random() * 1000); + const server = startLocalChatServer({ + deckPath, + modelProvider: provider, + port, + signal: abort.signal, + }); + + try { + const response = await fetch(`http://localhost:${port}/api/message`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ message: "slow prompt" }), + }); + assertEquals(response.status, 202); + await response.text(); + await waitForSession(port, (value) => value.running === true); + assert(seenSignal); + const stop = await fetch(`http://localhost:${port}/api/session/stop`, { + method: "POST", + }); + assertEquals(stop.status, 202); + await stop.text(); + const session = await waitForSession( + port, + (value) => + value.running === false && + value.transcript.some((row) => row.content === "Stopped."), + ); + assertEquals(seenSignal.aborted, true); + assertEquals(session.errors ?? [], []); + assertEquals( + session.transcript.map((row) => `${row.role}:${row.content}`), + ["user:slow prompt", "system:Stopped."], + ); + } finally { + abort.abort(); + await server.finished.catch(() => {}); + } +}); + +Deno.test({ + name: "chat server preserves separate assistant message items", + permissions: { read: true, write: true, net: true, env: true }, +}, async () => { + const dir = await Deno.makeTempDir({ prefix: "gambit-chat-items-test-" }); + const deckPath = await writeTempFile( + dir, + "root.deck.md", + `+++ +label = "chat message items test" + +[modelParams] +model = "test/model" ++++ + +Return two assistant messages.`, + ); + const provider: ModelProvider = { + responses() { + return modelResponse([ + assistantText("first assistant item"), + assistantText("second assistant item"), + ]); + }, + }; + const abort = new AbortController(); + const port = 20250 + Math.floor(Math.random() * 1000); + const server = startLocalChatServer({ + deckPath, + modelProvider: provider, + port, + signal: abort.signal, + }); + + try { + const response = await fetch(`http://localhost:${port}/api/message`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ message: "split please" }), + }); + assertEquals(response.status, 202); + await response.text(); + const session = await waitForSession( + port, + (value) => + value.transcript.some((row) => + row.role === "assistant" && row.content === "second assistant item" + ), + ); + assertEquals( + session.transcript.map((row) => `${row.role}:${row.content}`), + [ + "user:split please", + "assistant:first assistant item", + "assistant:second assistant item", + ], + ); + } finally { + abort.abort(); + await server.finished.catch(() => {}); + } +}); + +Deno.test({ + name: "chat server streams session updates while a deck turn runs", + permissions: { read: true, write: true, net: true, env: true }, +}, async () => { + const dir = await Deno.makeTempDir({ prefix: "gambit-chat-sse-test-" }); + const deckPath = await writeTempFile( + dir, + "root.deck.md", + `+++ +label = "chat sse test" + +[modelParams] +model = "test/model" ++++ + +Echo the user.`, + ); + let releaseResponse: (() => void) | undefined; + const provider: ModelProvider = { + responses(input) { + input.onStreamEvent?.({ + type: "response.output_text.delta", + output_index: 0, + item_id: "msg_stream", + delta: "slow ", + }); + return new Promise((resolve) => { + releaseResponse = () => + resolve(modelResponse([assistantText("slow ok")])); + }); + }, + }; + const abort = new AbortController(); + const port = 20500 + Math.floor(Math.random() * 1000); + const server = startLocalChatServer({ + deckPath, + modelProvider: provider, + port, + signal: abort.signal, + }); + + try { + const events = await fetch(`http://localhost:${port}/api/session/events`); + assertEquals(events.status, 200); + assertStringIncludes( + events.headers.get("content-type") ?? "", + "text/event-stream", + ); + const reader = events.body?.getReader(); + assert(reader); + const initial = await reader.read(); + assertEquals(initial.done, false); + assertStringIncludes(new TextDecoder().decode(initial.value), "session"); + + const response = await fetch(`http://localhost:${port}/api/message`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ message: "slow prompt" }), + }); + assertEquals(response.status, 202); + await response.text(); + const running = await (await fetch(`http://localhost:${port}/api/session`)) + .json() as ChatSessionTestPayload; + assertEquals(running.running, true); + assertEquals( + running.transcript.map((row) => `${row.role}:${row.content}`), + ["user:slow prompt"], + ); + + const update = await reader.read(); + assertEquals(update.done, false); + assertStringIncludes(new TextDecoder().decode(update.value), "slow prompt"); + + releaseResponse?.(); + const completed = await waitForSession( + port, + (value) => + value.running === false && + value.transcript.some((row) => row.content.includes("slow ok")), + ); + assertEquals(completed.running, false); + assert(typeof completed.metrics?.ttftMs === "number"); + assert(completed.metrics.ttftMs >= 0); + assert(typeof completed.metrics?.modelTtftMs === "number"); + assert(completed.metrics.modelTtftMs >= 0); + assert(completed.metrics.firstTokenAt); + assert( + completed.traceEvents?.some((event) => + event.type === "response.output_text.delta" && + typeof event.ts === "number" + ), + ); + assertEquals( + completed.transcript.map((row) => row.role), + ["user", "assistant"], + ); + await reader.cancel(); + } finally { + abort.abort(); + await server.finished.catch(() => {}); + } +}); + +Deno.test({ + name: "chat server can start a fresh run and replay the first prompt", + permissions: { read: true, write: true, net: true, env: true }, +}, async () => { + const dir = await Deno.makeTempDir({ prefix: "gambit-chat-rerun-test-" }); + const deckPath = await writeTempFile( + dir, + "root.deck.md", + `+++ +label = "chat rerun test" + +[modelParams] +model = "test/model" ++++ + +Echo the user.`, + ); + let callCount = 0; + const provider: ModelProvider = { + responses() { + callCount++; + return modelResponse([assistantText(`reply ${callCount}`)]); + }, + }; + const abort = new AbortController(); + const port = 21000 + Math.floor(Math.random() * 1000); + const statePath = path.join(dir, "workspace.sqlite"); + const server = startLocalChatServer({ + deckPath, + modelProvider: provider, + port, + statePath, + signal: abort.signal, + }); + + try { + const initial = await (await fetch(`http://localhost:${port}/api/session`)) + .json() as { runId: string }; + const first = await fetch(`http://localhost:${port}/api/message`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ message: "same prompt" }), + }); + assertEquals(first.status, 202); + await first.text(); + await waitForSession( + port, + (value) => + value.transcript.some((row) => row.content.includes("reply 1")) && + value.running === false, + ); + const followUp = await fetch(`http://localhost:${port}/api/message`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ message: "follow-up prompt" }), + }); + assertEquals(followUp.status, 202); + await followUp.text(); + await waitForSession( + port, + (value) => + value.transcript.some((row) => row.content.includes("reply 2")) && + value.running === false, + ); + const reset = await fetch(`http://localhost:${port}/api/session/reset`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ rerun: true }), + }); + assertEquals(reset.status, 202); + await reset.text(); + const session = await waitForSession( + port, + (value) => + value.transcript.some((row) => row.content.includes("reply 3")) && + value.running === false, + ) as { + runId: string; + firstUserMessage?: string; + lastUserMessage?: string; + transcript: Array<{ role: string; content: string }>; + }; + assertNotEquals(session.runId, initial.runId); + assertEquals(session.firstUserMessage, "same prompt"); + assertEquals(session.lastUserMessage, "same prompt"); + assertEquals(callCount, 3); + assertEquals( + session.transcript.filter((row) => row.role === "user").map((row) => + row.content + ), + ["same prompt"], + ); + assertEquals( + session.transcript.some((row) => row.content.includes("reply 3")), + true, + ); + } finally { + abort.abort(); + await server.finished.catch(() => {}); + } +}); + +Deno.test({ + name: "runtime tool loader rejects duplicate tool names", + permissions: { read: true, write: true }, +}, async () => { + const dir = await Deno.makeTempDir({ prefix: "gambit-chat-tools-test-" }); + const one = await writeTempFile( + dir, + "one.md", + `+++ +[[tools]] +name = "same" ++++ +`, + ); + const two = await writeTempFile( + dir, + "two.md", + `+++ +[[tools]] +name = "same" ++++ +`, + ); + + try { + await loadRuntimeTools([one, two]); + throw new Error("expected duplicate tool rejection"); + } catch (err) { + assertStringIncludes(String(err), "Duplicate runtime tool"); + } +}); diff --git a/src/commands/chat.ts b/src/commands/chat.ts new file mode 100644 index 000000000..0bb451fa9 --- /dev/null +++ b/src/commands/chat.ts @@ -0,0 +1,1194 @@ +import * as path from "@std/path"; +import { + runDeckResponses, + stringifyResponseOutput, +} from "@bolt-foundry/gambit-core"; +import type { + ModelMessage, + ModelProvider, + PermissionDeclarationInput, + ResponseItem, + SavedState, + TraceEvent, +} from "@bolt-foundry/gambit-core"; +import { parseContext } from "../cli_utils.ts"; +import { makeJsonlTracer } from "../trace.ts"; +import { enrichStateMeta } from "../cli_utils.ts"; +import { + loadCanonicalWorkspaceState, + saveCanonicalWorkspaceState, +} from "../workspace_sqlite.ts"; +import { + loadRuntimeTools, + type RuntimeToolBinding, +} from "./chat_runtime_tools.ts"; + +export { loadRuntimeTools }; + +const logger = console; +const DEFAULT_CHAT_PORT = 8787; +const CHAT_WORKER_SANDBOX_UNSUPPORTED_MESSAGE = + "gambit chat does not support worker sandbox execution yet because the Stop control requires root run cancellation; use --no-worker-sandbox or remove the worker sandbox setting for chat."; + +type ChatTranscriptEntry = { + id: string; + role: "user" | "assistant" | "system" | "tool"; + content: string; + at: string; +}; + +type ChatMetrics = { + turnStartedAt?: string; + modelCalledAt?: string; + firstTokenAt?: string; + ttftMs?: number; + modelTtftMs?: number; +}; + +type ChatTimingState = { + turnStartedMs?: number; + modelCallMs?: number; + firstTokenMs?: number; +}; + +type ChatSession = { + deckPath: string; + statePath?: string; + tracePath?: string; + reproMessage?: string; + runId: string; + runStartedAt: string; + firstUserMessage?: string; + lastUserMessage?: string; + runtimeTools: Array; + state?: SavedState; + transcript: Array; + traceEvents: Array; + metrics: ChatMetrics; + timing: ChatTimingState; + errors: Array; + running: boolean; +}; + +function nowIso(): string { + return new Date().toISOString(); +} + +function isoFromMs(ms: number): string { + return new Date(ms).toISOString(); +} + +function createChatRunState(): SavedState { + return { runId: crypto.randomUUID(), messages: [] }; +} + +function transcriptContentFromMessage(message: ModelMessage): string { + if (typeof message.content === "string") return message.content; + if (message.tool_calls?.length) return compactJson(message.tool_calls); + return ""; +} + +function transcriptFromSavedState( + state: SavedState, + at: string, +): Array { + return (state.messages ?? []) + .map((message): ChatTranscriptEntry | undefined => { + const content = transcriptContentFromMessage(message); + if (!content) return undefined; + return { + id: crypto.randomUUID(), + role: message.role, + content, + at, + }; + }) + .filter((entry): entry is ChatTranscriptEntry => entry !== undefined); +} + +function userMessagesFromTranscript( + transcript: Array, +): Array { + return transcript + .filter((entry) => entry.role === "user" && entry.content.trim()) + .map((entry) => entry.content); +} + +function jsonResponse(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value, null, 2), { + status, + headers: { "content-type": "application/json; charset=utf-8" }, + }); +} + +function safeJsonParseObject( + text: string, +): Record | undefined { + try { + const parsed = JSON.parse(text); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + return parsed as Record; + } + if (typeof parsed === "string") { + return safeJsonParseObject(parsed); + } + } catch { + // fall through + } + return undefined; +} + +function isPlainRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function structuredToolResult(value: unknown): unknown { + if (isPlainRecord(value)) return value; + if (typeof value === "string") { + return safeJsonParseObject(value) ?? value; + } + return value; +} + +function responseItemsText(output: Array): string { + const text = stringifyResponseOutput(output); + if (text) return text; + return JSON.stringify(output, null, 2); +} + +function compactJson(value: unknown): string { + if (value === undefined || value === null) return ""; + if (typeof value === "string") return value; + return JSON.stringify(value, null, 2); +} + +function traceToolCallKey(event: TraceEvent, fallback: number): string { + if (event.type !== "tool.call" && event.type !== "tool.result") { + return String(fallback); + } + return event.actionCallId || event.name || String(fallback); +} + +function traceToolActivityContent(input: { + call?: Extract; + result?: Extract; +}): string { + const name = input.call?.name || input.result?.name || "tool"; + const resultValue = input.result?.result; + const status = resultValue && typeof resultValue === "object" && + "status" in resultValue + ? String(resultValue.status) + : undefined; + const lines = [ + input.result ? `Tool result: ${name}` : `Tool call: ${name}`, + ]; + if (status) lines.push(`status: ${status}`); + if (input.call && !input.result) lines.push(compactJson(input.call.args)); + if (input.result) lines.push(compactJson(resultValue)); + return lines.filter(Boolean).join("\n"); +} + +function toolTranscriptEntriesFromTrace( + traceEvents: Array, + startedAt: string, +): Array { + const rows: Array = []; + const rowIndexesByKey = new Map(); + + for (const event of traceEvents) { + if (event.type === "tool.call") { + const key = traceToolCallKey(event, rowIndexesByKey.size); + rowIndexesByKey.set(key, rows.length); + rows.push({ + id: crypto.randomUUID(), + role: "tool", + content: traceToolActivityContent({ call: event }), + at: startedAt, + }); + } + if (event.type === "tool.result") { + const key = traceToolCallKey(event, rowIndexesByKey.size); + const existingIndex = rowIndexesByKey.get(key); + if (existingIndex === undefined) { + rowIndexesByKey.set(key, rows.length); + rows.push({ + id: crypto.randomUUID(), + role: "tool", + content: traceToolActivityContent({ result: event }), + at: startedAt, + }); + continue; + } + rows[existingIndex] = { + ...rows[existingIndex], + content: traceToolActivityContent({ result: event }), + }; + } + } + + return rows; +} + +function assistantResponseTexts(output: Array): Array { + const texts: Array = []; + for (const item of output) { + if (item.type !== "message" || item.role !== "assistant") continue; + const text = item.content + .filter((content) => content.type === "output_text") + .map((content) => content.text) + .join(""); + if (text) texts.push(text); + } + return texts; +} + +function stampTraceEvent(event: TraceEvent): TraceEvent { + if (event.ts !== undefined) return event; + return { ...event, ts: Date.now() } as TraceEvent; +} + +function beginTurnTiming(session: ChatSession): void { + const startedMs = Date.now(); + session.timing = { turnStartedMs: startedMs }; + session.metrics = { turnStartedAt: isoFromMs(startedMs) }; +} + +function recordTraceTiming(session: ChatSession, event: TraceEvent): void { + const eventMs = event.ts ?? Date.now(); + if (event.type === "model.call" && session.timing.modelCallMs === undefined) { + session.timing.modelCallMs = eventMs; + session.metrics.modelCalledAt = isoFromMs(eventMs); + } + if ( + event.type === "response.output_text.delta" && + session.timing.firstTokenMs === undefined + ) { + session.timing.firstTokenMs = eventMs; + session.metrics.firstTokenAt = isoFromMs(eventMs); + if (session.timing.turnStartedMs !== undefined) { + session.metrics.ttftMs = eventMs - session.timing.turnStartedMs; + } + if (session.timing.modelCallMs !== undefined) { + session.metrics.modelTtftMs = eventMs - session.timing.modelCallMs; + } + } +} + +function createTraceCollector( + session: ChatSession, + tracePath?: string, + onTrace?: (event: TraceEvent) => void, +): (event: TraceEvent) => void { + const jsonlTracer = tracePath ? makeJsonlTracer(tracePath) : undefined; + return (event) => { + const stamped = stampTraceEvent(event); + recordTraceTiming(session, stamped); + session.traceEvents.push(stamped); + jsonlTracer?.(stamped); + onTrace?.(stamped); + }; +} + +function runtimeToolSummary( + binding: RuntimeToolBinding, +): Record { + return { + name: binding.name, + description: binding.description, + sourcePath: binding.sourcePath, + inputSchemaPath: binding.inputSchemaPath, + actionPath: binding.actionPath, + }; +} + +async function runRuntimeToolAction(input: { + binding: RuntimeToolBinding; + args: Record; + modelProvider: ModelProvider; + model?: string; + modelForce?: string; + responsesMode?: boolean; + trace: (event: TraceEvent) => void; + signal?: AbortSignal; + workerSandbox?: boolean; + workspacePermissions?: PermissionDeclarationInput; + workspacePermissionsBaseDir?: string; + sessionPermissions?: PermissionDeclarationInput; + sessionPermissionsBaseDir?: string; +}): Promise { + if (!input.binding.actionPath) { + return { + status: 501, + code: "missing_runtime_tool_action", + message: `Runtime tool ${input.binding.name} has no action binding.`, + tool: input.binding.name, + }; + } + const result = await runDeckResponses({ + path: input.binding.actionPath, + input: input.args, + inputProvided: true, + initialUserMessage: undefined, + modelProvider: input.modelProvider, + isRoot: true, + defaultModel: input.model, + modelOverride: input.modelForce, + responsesMode: input.responsesMode, + trace: input.trace, + signal: input.signal, + workerSandbox: input.workerSandbox, + workspacePermissions: input.workspacePermissions, + workspacePermissionsBaseDir: input.workspacePermissionsBaseDir, + sessionPermissions: input.sessionPermissions, + sessionPermissionsBaseDir: input.sessionPermissionsBaseDir, + }); + if (result.output.length === 0 && result.legacyResult !== undefined) { + return structuredToolResult(result.legacyResult); + } + const text = responseItemsText(result.output); + const parsed = safeJsonParseObject(text); + if (parsed) return parsed; + const legacyResult = structuredToolResult(result.legacyResult); + if (isPlainRecord(legacyResult)) return legacyResult; + return text; +} + +function renderChatHtml(): string { + return ` + + + + +Deck Chat Repro + + + +
    +
    +
    +

    Deck Chat Repro

    +

    +
    +
    + + + +
    +
    +
    +
    +
    +
    + + +
    +
    + +
    +
    + + +`; +} + +export function startLocalChatServer(opts: { + deckPath: string; + context?: unknown; + contextProvided?: boolean; + modelProvider: ModelProvider; + model?: string; + modelForce?: string; + port?: number; + statePath?: string; + tracePath?: string; + reproMessage?: string; + runtimeTools?: Array; + responsesMode?: boolean; + workerSandbox?: boolean; + workspacePermissions?: PermissionDeclarationInput; + workspacePermissionsBaseDir?: string; + sessionPermissions?: PermissionDeclarationInput; + sessionPermissionsBaseDir?: string; + signal?: AbortSignal; +}): ReturnType { + if (opts.workerSandbox === true) { + throw new Error(CHAT_WORKER_SANDBOX_UNSUPPORTED_MESSAGE); + } + const chatWorkerSandbox = opts.workerSandbox ?? false; + const state = (() => { + if (!opts.statePath) return undefined; + try { + return loadCanonicalWorkspaceState(opts.statePath).state; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if ( + message.includes("Workspace sqlite not found") || + message.includes("Workspace state not found") + ) { + return createChatRunState(); + } + throw err; + } + })(); + const initialState = state ?? createChatRunState(); + const runStartedAt = nowIso(); + const initialTranscript = transcriptFromSavedState( + initialState, + runStartedAt, + ); + const restoredUserMessages = userMessagesFromTranscript(initialTranscript); + const session: ChatSession = { + deckPath: path.resolve(opts.deckPath), + statePath: opts.statePath ? path.resolve(opts.statePath) : undefined, + tracePath: opts.tracePath ? path.resolve(opts.tracePath) : undefined, + reproMessage: opts.reproMessage, + runId: initialState.runId, + runStartedAt, + firstUserMessage: restoredUserMessages[0], + lastUserMessage: restoredUserMessages.at(-1), + runtimeTools: opts.runtimeTools ?? [], + state: initialState, + transcript: initialTranscript, + traceEvents: [], + metrics: {}, + timing: {}, + errors: [], + running: false, + }; + const port = opts.port ?? DEFAULT_CHAT_PORT; + const encoder = new TextEncoder(); + const sseClients = new Set>(); + let currentRunAbortController: AbortController | undefined; + const sessionPayload = () => ({ + deckPath: session.deckPath, + statePath: session.statePath, + tracePath: session.tracePath, + reproMessage: session.reproMessage, + runId: session.runId, + runStartedAt: session.runStartedAt, + firstUserMessage: session.firstUserMessage, + lastUserMessage: session.lastUserMessage, + runtimeTools: session.runtimeTools.map(runtimeToolSummary), + transcript: session.transcript, + traceEvents: session.traceEvents, + metrics: session.metrics, + errors: session.errors, + running: session.running, + }); + const encodeSse = (value: unknown) => + encoder.encode(`event: session\ndata: ${JSON.stringify(value)}\n\n`); + const broadcastSession = () => { + const message = encodeSse(sessionPayload()); + for (const client of sseClients) { + try { + client.enqueue(message); + } catch { + sseClients.delete(client); + } + } + }; + const trace = createTraceCollector(session, opts.tracePath, broadcastSession); + const resetSessionRun = () => { + const freshState = createChatRunState(); + session.runId = freshState.runId; + session.runStartedAt = nowIso(); + session.state = freshState; + session.transcript = []; + session.traceEvents = []; + session.metrics = {}; + session.timing = {}; + session.errors = []; + session.firstUserMessage = undefined; + session.lastUserMessage = undefined; + session.running = false; + if (opts.statePath) { + saveCanonicalWorkspaceState( + opts.statePath, + enrichStateMeta(freshState, opts.deckPath), + ); + } + broadcastSession(); + }; + + const runTurn = async (message: string) => { + const turnAbortController = new AbortController(); + currentRunAbortController = turnAbortController; + const turnTraceStartIndex = session.traceEvents.length; + beginTurnTiming(session); + session.running = true; + session.firstUserMessage ??= message; + session.lastUserMessage = message; + session.transcript.push({ + id: crypto.randomUUID(), + role: "user", + content: message, + at: nowIso(), + }); + broadcastSession(); + const appendToolActivity = () => { + const rows = toolTranscriptEntriesFromTrace( + session.traceEvents.slice(turnTraceStartIndex), + nowIso(), + ); + session.transcript.push(...rows); + }; + try { + const bindingsByName = new Map( + session.runtimeTools.map((binding) => [binding.name, binding]), + ); + const result = await runDeckResponses({ + path: opts.deckPath, + input: opts.context, + inputProvided: Boolean(opts.contextProvided), + initialUserMessage: message, + modelProvider: opts.modelProvider, + isRoot: true, + defaultModel: opts.model, + modelOverride: opts.modelForce, + trace, + stream: false, + signal: turnAbortController.signal, + state: session.state, + onStateUpdate: (s) => { + session.state = s; + if (opts.statePath) { + saveCanonicalWorkspaceState( + opts.statePath, + enrichStateMeta(s, opts.deckPath), + ); + } + broadcastSession(); + }, + responsesMode: opts.responsesMode, + runtimeTools: session.runtimeTools.map((binding) => binding.tool), + onTool: async (toolInput) => { + const binding = bindingsByName.get(toolInput.name); + if (!binding) { + return { + status: 404, + code: "missing_runtime_tool", + message: `Runtime tool ${toolInput.name} was not supplied.`, + }; + } + return await runRuntimeToolAction({ + binding, + args: toolInput.args, + modelProvider: opts.modelProvider, + model: opts.model, + modelForce: opts.modelForce, + responsesMode: opts.responsesMode, + trace, + signal: turnAbortController.signal, + workerSandbox: chatWorkerSandbox, + workspacePermissions: opts.workspacePermissions, + workspacePermissionsBaseDir: opts.workspacePermissionsBaseDir, + sessionPermissions: opts.sessionPermissions, + sessionPermissionsBaseDir: opts.sessionPermissionsBaseDir, + }); + }, + workerSandbox: chatWorkerSandbox, + workspacePermissions: opts.workspacePermissions, + workspacePermissionsBaseDir: opts.workspacePermissionsBaseDir, + sessionPermissions: opts.sessionPermissions, + sessionPermissionsBaseDir: opts.sessionPermissionsBaseDir, + }); + if (turnAbortController.signal.aborted) { + appendToolActivity(); + session.transcript.push({ + id: crypto.randomUUID(), + role: "system", + content: "Stopped.", + at: nowIso(), + }); + return; + } + if (result.state) { + session.state = result.state; + session.runId = result.state.runId; + if (opts.statePath) { + saveCanonicalWorkspaceState( + opts.statePath, + enrichStateMeta(result.state, opts.deckPath), + ); + } + } + appendToolActivity(); + const assistantTexts = assistantResponseTexts(result.output); + if (assistantTexts.length > 0) { + for (const content of assistantTexts) { + session.transcript.push({ + id: crypto.randomUUID(), + role: "assistant", + content, + at: nowIso(), + }); + } + } else { + session.transcript.push({ + id: crypto.randomUUID(), + role: "assistant", + content: responseItemsText(result.output), + at: nowIso(), + }); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (turnAbortController.signal.aborted) { + appendToolActivity(); + session.transcript.push({ + id: crypto.randomUUID(), + role: "system", + content: "Stopped.", + at: nowIso(), + }); + return; + } + session.errors.push(message); + session.transcript.push({ + id: crypto.randomUUID(), + role: "system", + content: `Error: ${message}`, + at: nowIso(), + }); + } finally { + if (currentRunAbortController === turnAbortController) { + currentRunAbortController = undefined; + } + session.running = false; + broadcastSession(); + } + }; + + return Deno.serve({ + hostname: "127.0.0.1", + port, + signal: opts.signal, + }, async (request) => { + const url = new URL(request.url); + if (request.method === "GET" && url.pathname === "/") { + return new Response(renderChatHtml(), { + headers: { "content-type": "text/html; charset=utf-8" }, + }); + } + if (request.method === "GET" && url.pathname === "/api/session") { + return jsonResponse(sessionPayload()); + } + if (request.method === "GET" && url.pathname === "/api/session/events") { + let keepAlive: number | undefined; + let streamController: + | ReadableStreamDefaultController< + Uint8Array + > + | undefined; + const stream = new ReadableStream({ + start(controller) { + streamController = controller; + sseClients.add(controller); + controller.enqueue(encoder.encode(": connected\n\n")); + controller.enqueue(encodeSse(sessionPayload())); + keepAlive = setInterval(() => { + try { + controller.enqueue(encoder.encode(": keep-alive\n\n")); + } catch { + sseClients.delete(controller); + if (keepAlive !== undefined) clearInterval(keepAlive); + } + }, 15_000); + }, + cancel() { + if (streamController) sseClients.delete(streamController); + if (keepAlive !== undefined) clearInterval(keepAlive); + }, + }); + return new Response(stream, { + headers: { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-cache, no-transform", + "connection": "keep-alive", + }, + }); + } + if (request.method === "POST" && url.pathname === "/api/session/reset") { + if (session.running) { + return jsonResponse({ error: "run already in progress" }, 409); + } + const body = await request.json().catch(() => ({})) as { + rerun?: unknown; + message?: unknown; + }; + const message = typeof body.message === "string" && body.message.trim() + ? body.message + : session.firstUserMessage; + if (body.rerun === true && !message) { + return jsonResponse({ error: "no message available to rerun" }, 400); + } + resetSessionRun(); + if (body.rerun === true && message) { + void runTurn(message); + return jsonResponse({ ok: true, session: sessionPayload() }, 202); + } + return jsonResponse({ ok: true, session: sessionPayload() }); + } + if (request.method === "POST" && url.pathname === "/api/session/stop") { + if (!session.running || !currentRunAbortController) { + return jsonResponse({ + ok: true, + stopped: false, + session: sessionPayload(), + }); + } + currentRunAbortController.abort(); + broadcastSession(); + return jsonResponse({ + ok: true, + stopped: true, + session: sessionPayload(), + }, 202); + } + if (request.method === "POST" && url.pathname === "/api/message") { + if (session.running) { + return jsonResponse({ error: "run already in progress" }, 409); + } + const body = await request.json().catch(() => ({})) as { + message?: unknown; + }; + if (typeof body.message !== "string" || !body.message.trim()) { + return jsonResponse({ error: "message is required" }, 400); + } + void runTurn(body.message); + return jsonResponse({ ok: true, session: sessionPayload() }, 202); + } + return new Response("Not found", { status: 404 }); + }); +} + +async function maybeOpenBrowser(url: string): Promise { + const command = Deno.build.os === "darwin" + ? new Deno.Command("open", { args: [url] }) + : Deno.build.os === "windows" + ? new Deno.Command("cmd", { args: ["/c", "start", url] }) + : new Deno.Command("xdg-open", { args: [url] }); + await command.output().catch((err) => { + logger.warn( + `[chat] unable to open browser: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }); +} + +export async function handleChatCommand(opts: { + deckPath: string; + context?: string; + contextProvided?: boolean; + modelProvider: ModelProvider; + model?: string; + modelForce?: string; + port?: number; + statePath?: string; + tracePath?: string; + reproMessage?: string; + runtimeToolsPaths?: Array; + responsesMode?: boolean; + open?: boolean; + workerSandbox?: boolean; + workspacePermissions?: PermissionDeclarationInput; + workspacePermissionsBaseDir?: string; + sessionPermissions?: PermissionDeclarationInput; + sessionPermissionsBaseDir?: string; +}) { + const runtimeTools = await loadRuntimeTools(opts.runtimeToolsPaths ?? []); + const port = opts.port ?? DEFAULT_CHAT_PORT; + const url = `http://localhost:${port}/`; + const server = startLocalChatServer({ + deckPath: opts.deckPath, + context: opts.context !== undefined + ? parseContext(opts.context) + : undefined, + contextProvided: opts.contextProvided, + modelProvider: opts.modelProvider, + model: opts.model, + modelForce: opts.modelForce, + port, + statePath: opts.statePath, + tracePath: opts.tracePath, + reproMessage: opts.reproMessage, + runtimeTools, + responsesMode: opts.responsesMode, + workerSandbox: opts.workerSandbox, + workspacePermissions: opts.workspacePermissions, + workspacePermissionsBaseDir: opts.workspacePermissionsBaseDir, + sessionPermissions: opts.sessionPermissions, + sessionPermissionsBaseDir: opts.sessionPermissionsBaseDir, + }); + logger.log(`[chat] ${url}`); + if (opts.open) { + await maybeOpenBrowser(url); + } + await server.finished; +} diff --git a/src/commands/chat_runtime_tools.ts b/src/commands/chat_runtime_tools.ts new file mode 100644 index 000000000..f015eb27e --- /dev/null +++ b/src/commands/chat_runtime_tools.ts @@ -0,0 +1,109 @@ +import { parse as parseToml } from "@std/toml"; +import * as path from "@std/path"; +import type { ExternalToolDefinition } from "@bolt-foundry/gambit-core"; + +export type RuntimeToolBinding = { + sourcePath: string; + name: string; + description?: string; + inputSchemaPath?: string; + actionPath?: string; + tool: ExternalToolDefinition; +}; + +function parseTomlFrontMatter( + text: string, +): { data: Record; body: string } { + const lines = text.split(/\r?\n/); + if (lines[0] !== "+++") { + throw new Error( + "runtime-tools file must start with TOML frontmatter (+++)", + ); + } + const endIndex = lines.indexOf("+++", 1); + if (endIndex === -1) { + throw new Error("runtime-tools file is missing closing +++ frontmatter"); + } + const tomlText = lines.slice(1, endIndex).join("\n"); + return { + data: parseToml(tomlText) as Record, + body: lines.slice(endIndex + 1).join("\n"), + }; +} + +async function loadSchema(schemaPath: string): Promise { + const imported = await import(path.toFileUrl(schemaPath).href); + return imported.default ?? imported.schema; +} + +type RuntimeToolInputSchema = ExternalToolDefinition["inputSchema"]; + +function normalizeToolRows(value: unknown): Array> { + if (!Array.isArray(value)) return []; + return value.filter((item): item is Record => + Boolean(item) && typeof item === "object" && !Array.isArray(item) + ); +} + +async function loadRuntimeToolsFile( + filePath: string, +): Promise> { + const resolved = path.resolve(filePath); + const text = await Deno.readTextFile(resolved); + const parsed = parseTomlFrontMatter(text); + const rows = normalizeToolRows(parsed.data.tools); + const baseDir = path.dirname(resolved); + const bindings: Array = []; + + for (const row of rows) { + const rawName = row.name; + if (typeof rawName !== "string" || !rawName.trim()) { + throw new Error(`runtime tool in ${resolved} is missing name`); + } + const name = rawName.trim(); + const description = typeof row.description === "string" + ? row.description + : undefined; + const inputSchemaPath = typeof row.inputSchema === "string" + ? path.resolve(baseDir, row.inputSchema) + : undefined; + const actionPath = typeof row.action === "string" + ? path.resolve(baseDir, row.action) + : undefined; + const tool: ExternalToolDefinition = { + name, + description, + inputSchema: inputSchemaPath + ? await loadSchema(inputSchemaPath) as RuntimeToolInputSchema + : undefined, + }; + bindings.push({ + sourcePath: resolved, + name, + description, + inputSchemaPath, + actionPath, + tool, + }); + } + + return bindings; +} + +export async function loadRuntimeTools( + filePaths: Array, +): Promise> { + const bindings = (await Promise.all(filePaths.map(loadRuntimeToolsFile))) + .flat(); + const seen = new Map(); + for (const binding of bindings) { + const previous = seen.get(binding.name); + if (previous) { + throw new Error( + `Duplicate runtime tool "${binding.name}" in ${binding.sourcePath}; already declared in ${previous}`, + ); + } + seen.set(binding.name, binding.sourcePath); + } + return bindings; +} diff --git a/src/decks/gambit-bot/policy/deck-format-1.0.md b/src/decks/gambit-bot/policy/deck-format-1.0.md index c09050c57..4ea16600d 100644 --- a/src/decks/gambit-bot/policy/deck-format-1.0.md +++ b/src/decks/gambit-bot/policy/deck-format-1.0.md @@ -432,9 +432,9 @@ Reserved (future, not currently executable): - Build tab scaffolding and deck discovery treat `PROMPT.md` as the canonical entrypoint. `root.deck.md` is legacy-only during the pre-1.0 window. -- Test tab discovery uses `[[scenarios]]` (or legacy `[[testDecks]]`) on the - root deck. Grade tab discovery uses `[[graders]]` (or legacy - `[[graderDecks]]`). +- Test tab discovery uses `[[scenarios]]` on the root deck. Grade tab discovery + uses `[[graders]]`. Legacy `testDecks` and `graderDecks` remain accepted only + for pre-1.0 compatibility. - Scenario and grader references SHOULD point to `.../PROMPT.md` to keep the 1.0 contract intact, even if those decks are executed directly. diff --git a/src/decks/gambit-bot/scenarios/faq_bot_build_flow/PROMPT.md b/src/decks/gambit-bot/scenarios/faq_bot_build_flow/PROMPT.md index 2f84e44be..30e8ea24f 100644 --- a/src/decks/gambit-bot/scenarios/faq_bot_build_flow/PROMPT.md +++ b/src/decks/gambit-bot/scenarios/faq_bot_build_flow/PROMPT.md @@ -13,7 +13,7 @@ effort = "medium" ![scenario-participant](gambit://snippets/scenario-participant.md) -You are Dan Sisco, cofounder of Bolt Foundry, speaking as the user. +You are Dan Sisco, speaking as the user. Important framing: diff --git a/src/mcp_server.test.ts b/src/mcp_server.test.ts index 9641230f6..17af75ef2 100644 --- a/src/mcp_server.test.ts +++ b/src/mcp_server.test.ts @@ -146,11 +146,11 @@ leakTolerantTest( const logPath = path.join(fixture.dir, "gambit-mcp-debug.log"); const previousRootDeck = Deno.env.get("GAMBIT_MCP_ROOT_DECK_PATH"); const previousDebug = Deno.env.get( - "WORKLOOP_CHIEF_RUNTIME_DEBUG_MCP", + "GAMBIT_MCP_DEBUG", ); const previousLogPath = Deno.env.get("GAMBIT_MCP_DEBUG_LOG_PATH"); Deno.env.set("GAMBIT_MCP_ROOT_DECK_PATH", fixture.rootDeckPath); - Deno.env.set("WORKLOOP_CHIEF_RUNTIME_DEBUG_MCP", "1"); + Deno.env.set("GAMBIT_MCP_DEBUG", "1"); Deno.env.set("GAMBIT_MCP_DEBUG_LOG_PATH", logPath); try { const response = await handleMcpRequest({ @@ -177,10 +177,10 @@ leakTolerantTest( Deno.env.set("GAMBIT_MCP_ROOT_DECK_PATH", previousRootDeck); } if (previousDebug === undefined) { - Deno.env.delete("WORKLOOP_CHIEF_RUNTIME_DEBUG_MCP"); + Deno.env.delete("GAMBIT_MCP_DEBUG"); } else { Deno.env.set( - "WORKLOOP_CHIEF_RUNTIME_DEBUG_MCP", + "GAMBIT_MCP_DEBUG", previousDebug, ); } diff --git a/src/mcp_server.ts b/src/mcp_server.ts index 6f915b8d4..a1fc6c5f5 100644 --- a/src/mcp_server.ts +++ b/src/mcp_server.ts @@ -40,8 +40,7 @@ const encoder = new TextEncoder(); const MCP_ALLOW_MODELS_ENV = "GAMBIT_MCP_ALLOW_MODELS"; const MCP_ROOT_DECK_PATH_ENV = "GAMBIT_MCP_ROOT_DECK_PATH"; const EXTERNAL_TOOL_BRIDGE_ENV = "GAMBIT_EXTERNAL_TOOL_BRIDGE"; -const DEBUG_MCP_ENV = "WORKLOOP_CHIEF_RUNTIME_DEBUG_MCP"; -const LEGACY_DEBUG_MCP_ENV = "BOLT_FOUNDRY_DESKTOP_CHIEF_RUNTIME_DEBUG_MCP"; +const DEBUG_MCP_ENV = "GAMBIT_MCP_DEBUG"; const DEBUG_MCP_LOG_PATH_ENV = "GAMBIT_MCP_DEBUG_LOG_PATH"; const SUPPORTED_PROTOCOL_VERSIONS = new Set([ "2025-06-18", @@ -56,11 +55,9 @@ type ToolCatalog = { }; function shouldDebugMcpBridge(): boolean { - for (const envName of [DEBUG_MCP_ENV, LEGACY_DEBUG_MCP_ENV]) { - const raw = Deno.env.get(envName)?.trim().toLowerCase(); - if (raw === "1" || raw === "true" || raw === "yes") return true; - } - return Boolean(debugMcpLogPath()); + const raw = Deno.env.get(DEBUG_MCP_ENV)?.trim().toLowerCase(); + return raw === "1" || raw === "true" || raw === "yes" || + Boolean(debugMcpLogPath()); } function debugMcpLogPath(): string | null { diff --git a/src/providers/codex.ts b/src/providers/codex.ts index 882f03ee8..6e4a27ced 100644 --- a/src/providers/codex.ts +++ b/src/providers/codex.ts @@ -36,8 +36,7 @@ const MCP_ROOT_DECK_PATH_ENV = "GAMBIT_MCP_ROOT_DECK_PATH"; const EXTERNAL_TOOL_BRIDGE_ENV = "GAMBIT_EXTERNAL_TOOL_BRIDGE"; const MCP_DEBUG_LOG_PATH_ENV = "GAMBIT_MCP_DEBUG_LOG_PATH"; const DENO_DIR_ENV = "DENO_DIR"; -const DEBUG_MCP_ENV = "WORKLOOP_CHIEF_RUNTIME_DEBUG_MCP"; -const LEGACY_DEBUG_MCP_ENV = "BOLT_FOUNDRY_DESKTOP_CHIEF_RUNTIME_DEBUG_MCP"; +const DEBUG_MCP_ENV = "GAMBIT_MCP_DEBUG"; const MCP_SERVER_PATH = (() => { try { const moduleUrl = new URL(import.meta.url); @@ -189,10 +188,8 @@ function shouldEnableMcpBridge(): boolean { } function shouldDebugMcpBridge(): boolean { - return [DEBUG_MCP_ENV, LEGACY_DEBUG_MCP_ENV].some((envName) => { - const raw = Deno.env.get(envName)?.trim().toLowerCase(); - return raw === "1" || raw === "true" || raw === "yes"; - }); + const raw = Deno.env.get(DEBUG_MCP_ENV)?.trim().toLowerCase(); + return raw === "1" || raw === "true" || raw === "yes"; } function isCodexNativeOrGambitBuiltinTool(name: string): boolean { @@ -218,7 +215,7 @@ function logCodexMcpDebug( function codexMcpDebugLogPath(cwd: string): string { return path.join( cwd, - ".boltfoundry", + ".workloop", "runtime", "chief-runtime", "gambit-mcp-debug.log", diff --git a/src/providers/ollama.test.ts b/src/providers/ollama.test.ts index 88e6290d2..02df213e6 100644 --- a/src/providers/ollama.test.ts +++ b/src/providers/ollama.test.ts @@ -86,13 +86,13 @@ Deno.test("fetchOllamaTags preserves baseURL path prefix", async () => { }); }); try { - await fetchOllamaTags("https://host.boltfoundry.bflocal:8017/ollama/v1"); + await fetchOllamaTags("https://host.workloop.bflocal:8017/ollama/v1"); } finally { restore(); } assertEquals( seen, - "https://host.boltfoundry.bflocal:8017/ollama/api/tags", + "https://host.workloop.bflocal:8017/ollama/api/tags", ); }); diff --git a/src/runtime_host_service.ts b/src/runtime_host_service.ts index 1f33096c7..8682151ec 100644 --- a/src/runtime_host_service.ts +++ b/src/runtime_host_service.ts @@ -1,7 +1,7 @@ export const RUNTIME_HOST_SERVICE_SOCKET_ENV = - "WORKLOOP_RUNTIME_HOST_SERVICE_SOCKET"; + "GAMBIT_RUNTIME_HOST_SERVICE_SOCKET"; export const RUNTIME_HOST_SERVICE_TOKEN_ENV = - "WORKLOOP_RUNTIME_HOST_SERVICE_TOKEN"; + "GAMBIT_RUNTIME_HOST_SERVICE_TOKEN"; export const CODEX_REFRESH_HOST_SERVICE_METHOD = "providerAuth.codex.refreshChatgptTokens"; @@ -170,7 +170,7 @@ export async function callRuntimeHostService(input: { const token = input.token?.trim() || Deno.env.get(RUNTIME_HOST_SERVICE_TOKEN_ENV)?.trim(); if (!socketPath || !token) { - throw new Error("Workloop host service bridge is not configured."); + throw new Error("Runtime host service bridge is not configured."); } const request: RuntimeHostServiceRequest = { id: crypto.randomUUID(), @@ -185,7 +185,7 @@ export async function callRuntimeHostService(input: { const line = await readFirstLine(conn.readable); if (!line) { throw new Error( - "Workloop host service bridge closed without a response.", + "Runtime host service bridge closed without a response.", ); } const response = JSON.parse(line) as RuntimeHostServiceResponse;