From 0290dd76cf95b00d472e92d129867cd8e6c8861e Mon Sep 17 00:00:00 2001 From: Daniel Kurzynski Date: Thu, 8 Oct 2026 18:01:27 +0200 Subject: [PATCH 01/14] fix(gpui): show project folders before the first session exists The main screen hands the session listing to the sidebar only when it differs from the last one it saw. Its initial value is an empty list, so an empty listing (a fresh install, or a new data directory) never reached the sidebar. The sidebar then never counted its sessions as loaded and never laid out the folders of the projects saved in projects.json: the Projects view stayed empty. Gpui now records when the first listing arrived, and the main screen hands that first listing over even when it is empty. --- crates/ui_gpui/src/app/commands.rs | 2 +- crates/ui_gpui/src/app/event_loop.rs | 2 +- crates/ui_gpui/src/lib.rs | 16 ++++++++++++++ crates/ui_gpui/src/main_screen/mod.rs | 11 ++++++++++ crates/ui_gpui/src/main_screen/tests/mod.rs | 1 + .../ui_gpui/src/main_screen/tests/sidebar.rs | 22 +++++++++++++++++++ crates/ui_gpui/src/sidebar/projects_view.rs | 9 ++++++++ 7 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 crates/ui_gpui/src/main_screen/tests/sidebar.rs diff --git a/crates/ui_gpui/src/app/commands.rs b/crates/ui_gpui/src/app/commands.rs index b6dcc7dc..fbbca071 100644 --- a/crates/ui_gpui/src/app/commands.rs +++ b/crates/ui_gpui/src/app/commands.rs @@ -76,7 +76,7 @@ impl Gpui { match service.list_sessions().await { Ok(sessions) => { *gpui.session_lifecycles.lock().unwrap() = lifecycles; - *gpui.chat_sessions.lock().unwrap() = sessions.clone(); + gpui.set_chat_sessions(sessions.clone()); gpui.push_event(UiEvent::UpdateChatList { sessions }); } Err(e) => gpui.display_error(format!("Failed to list sessions: {e:#}")), diff --git a/crates/ui_gpui/src/app/event_loop.rs b/crates/ui_gpui/src/app/event_loop.rs index 86fd51a8..2e975feb 100644 --- a/crates/ui_gpui/src/app/event_loop.rs +++ b/crates/ui_gpui/src/app/event_loop.rs @@ -555,7 +555,7 @@ impl Gpui { sessions.len() ); // Update local cache - *self.chat_sessions.lock().unwrap() = sessions.clone(); + self.set_chat_sessions(sessions.clone()); let _current_session_id = self.current_session_id.lock().unwrap().clone(); // Refresh all windows to trigger re-render with new chat data diff --git a/crates/ui_gpui/src/lib.rs b/crates/ui_gpui/src/lib.rs index fff3c082..1ea9e0b0 100644 --- a/crates/ui_gpui/src/lib.rs +++ b/crates/ui_gpui/src/lib.rs @@ -226,6 +226,9 @@ pub struct Gpui { // Current chat state current_session_id: Arc>>, chat_sessions: Arc>>, + /// Set once the first session listing arrived, so an empty `chat_sessions` + /// can be told apart from one not loaded yet. + chat_sessions_loaded: Arc, /// Visits and settlement per session, mirrored from the core for the /// sidebar. Sessions absent here have the default lifecycle. session_lifecycles: @@ -591,6 +594,7 @@ impl Gpui { current_session_id: Arc::new(Mutex::new(None)), chat_sessions: Arc::new(Mutex::new(Vec::new())), + chat_sessions_loaded: Arc::new(std::sync::atomic::AtomicBool::new(false)), session_lifecycles: Arc::new(Mutex::new(HashMap::new())), current_session_activity_state: Arc::new(Mutex::new(None)), session_stop_requests: Arc::new(Mutex::new(std::collections::HashSet::new())), @@ -889,6 +893,18 @@ impl Gpui { self.chat_sessions.lock().unwrap().clone() } + /// Whether the session listing arrived at least once (it may be empty). + pub fn chat_sessions_loaded(&self) -> bool { + self.chat_sessions_loaded + .load(std::sync::atomic::Ordering::Acquire) + } + + fn set_chat_sessions(&self, sessions: Vec) { + *self.chat_sessions.lock().unwrap() = sessions; + self.chat_sessions_loaded + .store(true, std::sync::atomic::Ordering::Release); + } + pub fn get_session_lifecycles( &self, ) -> HashMap { diff --git a/crates/ui_gpui/src/main_screen/mod.rs b/crates/ui_gpui/src/main_screen/mod.rs index cc090a5d..016506ce 100644 --- a/crates/ui_gpui/src/main_screen/mod.rs +++ b/crates/ui_gpui/src/main_screen/mod.rs @@ -208,6 +208,11 @@ pub struct MainScreen { /// Form for the viewed session's oldest open `ask_question` request. question_prompt: Option>, + /// Whether the sidebar got the first session listing. Without it, an + /// empty listing equals the initial `sessions` and never reaches the + /// sidebar, which then shows no project folders. + sidebar_sessions_synced: bool, + // Sidebar animation left_animator: SidebarAnimator, right_animator: SidebarAnimator, @@ -345,6 +350,7 @@ impl MainScreen { ui_scale: initial_scale, right_sidebar_width: px(initial_sidebar_width), right_sidebar_resizing: false, + sidebar_sessions_synced: false, resize_start_x: 0.0, resize_start_width: 0.0, context_limit_cache: None, @@ -1697,11 +1703,16 @@ impl Render for MainScreen { sidebar.set_persisted_projects(persisted_projects, cx); }); + let sessions_loaded = cx + .try_global::() + .is_some_and(|g| g.chat_sessions_loaded()); // Update project sidebar if needed if self.sessions != sessions || self.session_lifecycles != session_lifecycles || self.current_session_id != current_session_id + || (sessions_loaded && !self.sidebar_sessions_synced) { + self.sidebar_sessions_synced = sessions_loaded; let previous_session_id = self.current_session_id.clone(); self.sessions = sessions.clone(); self.session_lifecycles = session_lifecycles.clone(); diff --git a/crates/ui_gpui/src/main_screen/tests/mod.rs b/crates/ui_gpui/src/main_screen/tests/mod.rs index 5f49e9d5..f1290f7d 100644 --- a/crates/ui_gpui/src/main_screen/tests/mod.rs +++ b/crates/ui_gpui/src/main_screen/tests/mod.rs @@ -4,4 +4,5 @@ mod comments; mod composer; mod end_to_end; +mod sidebar; mod stored_outputs; diff --git a/crates/ui_gpui/src/main_screen/tests/sidebar.rs b/crates/ui_gpui/src/main_screen/tests/sidebar.rs new file mode 100644 index 00000000..fe52f791 --- /dev/null +++ b/crates/ui_gpui/src/main_screen/tests/sidebar.rs @@ -0,0 +1,22 @@ +//! The session sidebar as the main screen feeds it. + +use crate::test_support::MainScreenTest; +use code_assistant_core::ui::UiEvent; +use gpui_kit::TestAppContext; + +#[gpui_kit::test] +fn saved_projects_show_before_the_first_session_exists(cx: &mut TestAppContext) { + let mut test = MainScreenTest::new(cx); + // Replace what the machine's projects.json holds. + *test.gpui.persisted_projects.lock().unwrap() = ["proj".to_string()].into(); + + // A fresh store: the listing arrives, and it is empty. + test.push(UiEvent::UpdateChatList { + sessions: Vec::new(), + }); + + let folders = test.main_screen.read_with(&test.cx, |screen, cx| { + screen.project_sidebar.read(cx).folder_projects() + }); + assert_eq!(folders, vec![Some("proj".to_string())]); +} diff --git a/crates/ui_gpui/src/sidebar/projects_view.rs b/crates/ui_gpui/src/sidebar/projects_view.rs index 494fb050..5d66f702 100644 --- a/crates/ui_gpui/src/sidebar/projects_view.rs +++ b/crates/ui_gpui/src/sidebar/projects_view.rs @@ -161,6 +161,15 @@ impl SessionSidebar { .max() } + /// The projects of the displayed folders, in order (`None`: no project). + #[cfg(test)] + pub(crate) fn folder_projects(&self) -> Vec> { + self.folders + .iter() + .map(|folder| folder.project.clone()) + .collect() + } + pub(super) fn render_folders(&self, cx: &mut Context) -> Vec { if self.folders.is_empty() { return vec![render_hint("No projects yet", cx).into_any_element()]; From ebd8213492f5fc1b9a4e060353baa4ec9d7f37d5 Mon Sep 17 00:00:00 2001 From: Daniel Kurzynski Date: Thu, 8 Oct 2026 18:05:05 +0200 Subject: [PATCH 02/14] fix(gpui): keep a typed AI Core deployment mapping on save A model-to-deployment mapping becomes part of the AI Core provider form only when its add button is clicked. A mapping typed into the add row and saved right away was dropped without notice, and the form closes on save, so the loss showed only when the provider was opened again. Saving now includes a fully typed add row (model ID and deployment ID) unless a mapping for that model already exists. --- .../provider_forms/ai_core_form.rs | 96 ++++++++++++++++++- 1 file changed, 94 insertions(+), 2 deletions(-) diff --git a/crates/ui_gpui/src/settings_screen/provider_forms/ai_core_form.rs b/crates/ui_gpui/src/settings_screen/provider_forms/ai_core_form.rs index 574704eb..7302627f 100644 --- a/crates/ui_gpui/src/settings_screen/provider_forms/ai_core_form.rs +++ b/crates/ui_gpui/src/settings_screen/provider_forms/ai_core_form.rs @@ -257,9 +257,35 @@ impl ProviderForm for AiCoreProviderForm { config.insert("api_base_url".to_string(), Value::String(api_base_url)); } - if !self.deployments.is_empty() { + // A mapping typed into the add row but not added yet is saved too, + // rather than dropped with the form. + let mut deployments = self.deployments.clone(); + let pending_model = self + .new_deployment_model_input + .read(cx) + .value() + .trim() + .to_string(); + let pending_id = self + .new_deployment_id_input + .read(cx) + .value() + .trim() + .to_string(); + if !pending_model.is_empty() + && !pending_id.is_empty() + && !deployments.iter().any(|d| d.model_id == pending_model) + { + deployments.push(DeploymentEntry { + model_id: pending_model, + deployment_id: pending_id, + api_type: self.new_deployment_api_type.clone(), + }); + } + + if !deployments.is_empty() { let mut models = serde_json::Map::new(); - for entry in &self.deployments { + for entry in &deployments { if entry.api_type == "anthropic" { models.insert( entry.model_id.clone(), @@ -621,3 +647,69 @@ impl Render for AiCoreProviderForm { ) } } + +#[cfg(test)] +mod tests { + use super::*; + use gpui_kit::{TestAppContext, VisualTestContext}; + + fn form(cx: &mut TestAppContext) -> (Entity, &mut VisualTestContext) { + cx.update(gpui_kit::init); + cx.add_window_view(AiCoreProviderForm::new) + } + + fn type_into( + form: &Entity, + cx: &mut VisualTestContext, + pick: fn(&AiCoreProviderForm) -> Entity, + text: &str, + ) { + form.update_in(cx, |form, window, cx| { + pick(form).update(cx, |state, cx| { + state.set_value(SharedString::from(text.to_string()), window, cx) + }); + }); + } + + #[gpui_kit::test] + fn a_mapping_typed_but_not_added_is_saved(cx: &mut TestAppContext) { + let (form, cx) = form(cx); + type_into( + &form, + cx, + |f| f.new_deployment_model_input.clone(), + "model-a", + ); + type_into(&form, cx, |f| f.new_deployment_id_input.clone(), " dep-1 "); + form.update(cx, |form, _| form.new_deployment_api_type = "openai".into()); + + let config = form.read_with(cx, |form, cx| form.to_config_json(cx)); + assert_eq!( + config["models"]["model-a"], + serde_json::json!({"deployment": "dep-1", "api_type": "openai"}) + ); + } + + #[gpui_kit::test] + fn an_added_mapping_wins_over_the_add_row(cx: &mut TestAppContext) { + let (form, cx) = form(cx); + form.update(cx, |form, _| { + form.deployments.push(DeploymentEntry { + model_id: "model-a".into(), + deployment_id: "dep-1".into(), + api_type: "anthropic".into(), + }) + }); + type_into( + &form, + cx, + |f| f.new_deployment_model_input.clone(), + "model-a", + ); + type_into(&form, cx, |f| f.new_deployment_id_input.clone(), "dep-2"); + + let config = form.read_with(cx, |form, cx| form.to_config_json(cx)); + assert_eq!(config["models"]["model-a"], "dep-1"); + assert_eq!(config["models"].as_object().unwrap().len(), 1); + } +} From 0903dc58d3d68674ee06b831eb7453a71909a305 Mon Sep 17 00:00:00 2001 From: Daniel Kurzynski Date: Thu, 8 Oct 2026 18:09:18 +0200 Subject: [PATCH 03/14] feat: voice mode, a global voice assistant over all conversations A realtime speech model you talk to about your conversations: what is running, what an agent answered, start a conversation or send one a message. It runs next to the sessions, not inside one, and acts only through conversation tools. - llm::realtime: OpenAI Realtime protocol and WebSocket transport; connectors for OpenAI-type providers and AI Core deployments. - audio crate: microphone and speakers at 24 kHz; macOS uses the voice-processing I/O unit (echo cancellation), elsewhere cpal with half-duplex gating. - code_assistant_core::voice: VoiceService, the agent loop, the tools (list_conversations, list_projects, message_conversation, get_conversation) and notifications when a conversation finishes or waits. The pure Floor state machine keeps them from talking over the user or the model: they wait for playback to drain and a quiet period. - SessionService: session_content and session_activity_states. - GPUI: microphone, mute and status in the title bar, a transcript popover, and Settings -> Voice (voice.json). - macOS bundle: microphone usage description and audio-input entitlement. See docs/voice-mode.md. --- AGENTS.md | 16 +- Cargo.lock | 340 ++++++++- Cargo.toml | 2 +- crates/audio/Cargo.toml | 16 + crates/audio/examples/devices.rs | 56 ++ crates/audio/src/cpal_backend.rs | 146 ++++ crates/audio/src/lib.rs | 144 ++++ crates/audio/src/macos.rs | 125 +++ crates/audio/src/playback.rs | 119 +++ crates/audio/src/resample.rs | 65 ++ crates/code_assistant/Cargo.toml | 4 + .../code_assistant/assets/Entitlements.plist | 4 + crates/code_assistant/assets/Info.plist | 2 + crates/code_assistant/src/app/gpui.rs | 18 + crates/code_assistant/src/app/mod.rs | 2 + crates/code_assistant/src/app/voice.rs | 39 + crates/code_assistant_core/src/lib.rs | 1 + .../src/session/manager.rs | 21 + .../src/session/service.rs | 27 + .../code_assistant_core/src/ui/ui_events.rs | 9 + crates/code_assistant_core/src/voice/agent.rs | 468 ++++++++++++ .../code_assistant_core/src/voice/config.rs | 186 +++++ crates/code_assistant_core/src/voice/floor.rs | 719 ++++++++++++++++++ crates/code_assistant_core/src/voice/mod.rs | 106 +++ .../src/voice/notifications.rs | 164 ++++ .../code_assistant_core/src/voice/service.rs | 174 +++++ .../code_assistant_core/src/voice/source.rs | 282 +++++++ crates/code_assistant_core/src/voice/tests.rs | 436 +++++++++++ crates/code_assistant_core/src/voice/tools.rs | 496 ++++++++++++ crates/llm/src/factory.rs | 8 +- crates/llm/src/lib.rs | 1 + crates/llm/src/realtime/aicore.rs | 175 +++++ crates/llm/src/realtime/events.rs | 304 ++++++++ crates/llm/src/realtime/mod.rs | 128 ++++ crates/llm/src/realtime/transport.rs | 159 ++++ crates/ui_acp/src/ui.rs | 3 + crates/ui_gpui/assets/icons/audio_lines.svg | 1 + crates/ui_gpui/assets/icons/mic.svg | 1 + crates/ui_gpui/assets/icons/mic_off.svg | 1 + crates/ui_gpui/src/app/event_loop.rs | 14 + crates/ui_gpui/src/app/mod.rs | 1 + crates/ui_gpui/src/app/voice.rs | 81 ++ crates/ui_gpui/src/lib.rs | 6 + crates/ui_gpui/src/main_screen/mod.rs | 8 + .../ui_gpui/src/main_screen/voice_control.rs | 224 ++++++ crates/ui_gpui/src/settings_screen/mod.rs | 15 + .../src/settings_screen/voice_section.rs | 366 +++++++++ docs/voice-mode.md | 323 ++++++++ 48 files changed, 5970 insertions(+), 36 deletions(-) create mode 100644 crates/audio/Cargo.toml create mode 100644 crates/audio/examples/devices.rs create mode 100644 crates/audio/src/cpal_backend.rs create mode 100644 crates/audio/src/lib.rs create mode 100644 crates/audio/src/macos.rs create mode 100644 crates/audio/src/playback.rs create mode 100644 crates/audio/src/resample.rs create mode 100644 crates/code_assistant/src/app/voice.rs create mode 100644 crates/code_assistant_core/src/voice/agent.rs create mode 100644 crates/code_assistant_core/src/voice/config.rs create mode 100644 crates/code_assistant_core/src/voice/floor.rs create mode 100644 crates/code_assistant_core/src/voice/mod.rs create mode 100644 crates/code_assistant_core/src/voice/notifications.rs create mode 100644 crates/code_assistant_core/src/voice/service.rs create mode 100644 crates/code_assistant_core/src/voice/source.rs create mode 100644 crates/code_assistant_core/src/voice/tests.rs create mode 100644 crates/code_assistant_core/src/voice/tools.rs create mode 100644 crates/llm/src/realtime/aicore.rs create mode 100644 crates/llm/src/realtime/events.rs create mode 100644 crates/llm/src/realtime/mod.rs create mode 100644 crates/llm/src/realtime/transport.rs create mode 100644 crates/ui_gpui/assets/icons/audio_lines.svg create mode 100644 crates/ui_gpui/assets/icons/mic.svg create mode 100644 crates/ui_gpui/assets/icons/mic_off.svg create mode 100644 crates/ui_gpui/src/app/voice.rs create mode 100644 crates/ui_gpui/src/main_screen/voice_control.rs create mode 100644 crates/ui_gpui/src/settings_screen/voice_section.rs create mode 100644 docs/voice-mode.md diff --git a/AGENTS.md b/AGENTS.md index 3d1f5b43..3211d67b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ This is a Rust-based AI coding agent harness with multiple operational modes. ### Crate Layers ``` -Layer 0 (generic): llm command_executor pty_session fs_explorer sandbox web git terminal terminal_output +Layer 0 (generic): llm command_executor pty_session fs_explorer sandbox web git terminal terminal_output audio Layer 1 (generic): tools_core — tool trait, registry, render, spec, permissions mcp_client — wraps MCP server tools as registry tools Layer 2 (generic): agent_core — agent loop, hook traits, dialect trait, AgentUi trait @@ -74,6 +74,20 @@ Layer 5 (binary): code_assistant — CLI, config, feature-gated frontend through `UserInterface::ask_questions` and `SessionService::answer_questions`; see `docs/ask-question-tool.md` +### Voice Mode +- A global voice agent next to the sessions (`code_assistant_core::voice`): + a realtime speech model (`llm::realtime`, OpenAI Realtime protocol) with + tools only for conversations (list, read, message/create) +- Configured in `/voice.json` (*Settings → Voice*), started from + the GPUI title bar; driven through `VoiceService`, observed via app-scoped + `UiEvent::VoiceStatusChanged` / `VoiceTranscript` +- `voice::floor::Floor` is a pure state machine deciding when background + notifications (a conversation finished / waits) may speak: never over the + user or the model, only after playback drained plus a quiet period +- Audio through the `VoiceAudio` seam; the binary's `voice` feature plugs in + the `audio` crate (macOS voice-processing I/O with echo cancellation, + `cpal` half-duplex elsewhere); see `docs/voice-mode.md` + ### Session Storage - One folder per session below `sessions/`, the ID is its path: `/-` (`persistence::layout`) diff --git a/Cargo.lock b/Cargo.lock index 2417e21e..8a33d5f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -249,7 +249,7 @@ dependencies = [ "home", "libc", "log", - "mach2", + "mach2 0.5.0", "miow", "parking_lot", "piper", @@ -288,6 +288,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.11.1", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -793,6 +815,16 @@ dependencies = [ "zbus 5.15.0", ] +[[package]] +name = "audio" +version = "0.1.0" +dependencies = [ + "anyhow", + "coreaudio-rs 0.14.2", + "cpal", + "tracing", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -1035,7 +1067,7 @@ dependencies = [ "bitflags 2.11.1", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.13.0", "log", "prettyplease", "proc-macro2", @@ -1370,6 +1402,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -1668,6 +1706,7 @@ version = "0.2.17" dependencies = [ "anyhow", "async-channel", + "audio", "clap", "code_assistant_core", "command_executor", @@ -2045,6 +2084,40 @@ dependencies = [ "libm", ] +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-rs" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58" +dependencies = [ + "bitflags 2.11.1", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4739a805a62757a83e5654fa3faabec0442666b263bb2287d5a8185bfd953" +dependencies = [ + "bindgen", +] + [[package]] name = "cosmic-text" version = "0.19.0" @@ -2069,6 +2142,29 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs 0.11.3", + "dasp_sample", + "jni 0.21.1", + "js-sys", + "libc", + "mach2 0.4.3", + "ndk", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -2213,7 +2309,7 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa", - "phf 0.10.1", + "phf 0.11.3", "smallvec", ] @@ -2339,6 +2435,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.11.0" @@ -4800,7 +4902,7 @@ dependencies = [ "itertools 0.14.0", "libc", "log", - "mach2", + "mach2 0.5.0", "objc", "objc2 0.6.4", "objc2-app-kit 0.3.2", @@ -6041,6 +6143,22 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -6584,6 +6702,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "mach2" version = "0.5.0" @@ -6951,6 +7078,35 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" +dependencies = [ + "bitflags 2.11.1", + "jni-sys 0.3.1", + "log", + "ndk-sys 0.5.0+25.2.9519653", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "ndk-sys" version = "0.6.0+11769913" @@ -7246,6 +7402,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate 3.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "oauth2" version = "4.4.2" @@ -7272,7 +7450,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "chrono", "getrandom 0.2.17", "http 1.4.0", @@ -7368,6 +7546,21 @@ dependencies = [ "objc2-quartz-core 0.3.2", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.11.1", + "libc", + "objc2 0.6.4", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.3.2" @@ -7682,6 +7875,29 @@ dependencies = [ "memchr", ] +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni 0.21.1", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -7998,9 +8214,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" dependencies = [ - "phf_macros 0.10.0", "phf_shared 0.10.0", - "proc-macro-hack", ] [[package]] @@ -8074,20 +8288,6 @@ dependencies = [ "phf_shared 0.13.1", ] -[[package]] -name = "phf_macros" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0" -dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", - "proc-macro-hack", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "phf_macros" version = "0.11.3" @@ -8426,12 +8626,6 @@ dependencies = [ "toml_edit 0.25.11+spec-1.1.0", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.106" @@ -9490,7 +9684,7 @@ checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ "core-foundation 0.10.0", "core-foundation-sys", - "jni", + "jni 0.22.4", "log", "once_cell", "rustls 0.23.40", @@ -9858,7 +10052,7 @@ checksum = "7bd22781911de0ca6debda95f073c8f18bec65d1a94f1fa9573f3102e514cea4" dependencies = [ "ahash", "annotate-snippets", - "base64 0.21.7", + "base64 0.22.1", "encoding_rs_io", "getrandom 0.3.4", "granit-parser", @@ -12885,7 +13079,7 @@ dependencies = [ "libloading", "log", "naga", - "ndk-sys", + "ndk-sys 0.6.0+11769913", "objc2 0.6.4", "objc2-core-foundation", "objc2-foundation 0.3.2", @@ -12999,6 +13193,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.57.0" @@ -13075,6 +13279,16 @@ dependencies = [ "windows-core 0.62.2", ] +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.57.0" @@ -13341,6 +13555,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -13377,6 +13600,21 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -13452,6 +13690,12 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -13470,6 +13714,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -13488,6 +13738,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -13518,6 +13774,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -13536,6 +13798,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -13554,6 +13822,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -13572,6 +13846,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/Cargo.toml b/Cargo.toml index 5877900e..888d9d02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["crates/agent_core", "crates/agent_orchestration", "crates/code_assistant", "crates/code_assistant_core", "crates/ui_acp", "crates/ui_gpui", "crates/ui_terminal", "crates/command_executor", "crates/fs_explorer", "crates/git", "crates/llm", "crates/mcp_client", "crates/mcp_server", "crates/pty_session", "crates/sandbox", "crates/terminal", "crates/terminal_output", "crates/terminal_test_app", "crates/terminal_view", "crates/tools_core", "crates/web"] +members = ["crates/agent_core", "crates/agent_orchestration", "crates/code_assistant", "crates/code_assistant_core", "crates/audio", "crates/ui_acp", "crates/ui_gpui", "crates/ui_terminal", "crates/command_executor", "crates/fs_explorer", "crates/git", "crates/llm", "crates/mcp_client", "crates/mcp_server", "crates/pty_session", "crates/sandbox", "crates/terminal", "crates/terminal_output", "crates/terminal_test_app", "crates/terminal_view", "crates/tools_core", "crates/web"] resolver = "2" diff --git a/crates/audio/Cargo.toml b/crates/audio/Cargo.toml new file mode 100644 index 00000000..627de3a9 --- /dev/null +++ b/crates/audio/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "audio" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1.0" +tracing = "0.1" + +# macOS: the voice-processing I/O unit (echo cancellation, as in FaceTime). +[target.'cfg(target_os = "macos")'.dependencies] +coreaudio-rs = { version = "0.14", default-features = false, features = ["audio_unit", "core_audio"] } + +# Elsewhere: plain device streams with half-duplex gating. +[target.'cfg(not(target_os = "macos"))'.dependencies] +cpal = "0.15" diff --git a/crates/audio/examples/devices.rs b/crates/audio/examples/devices.rs new file mode 100644 index 00000000..0c879be5 --- /dev/null +++ b/crates/audio/examples/devices.rs @@ -0,0 +1,56 @@ +//! Open the voice devices and report what they deliver. +//! +//! `cargo run -p audio --example devices` listens for two seconds without +//! playing anything; `-- --tone` also plays a one-second tone and reports +//! when the speakers drained (`-- --silence` the same with silent samples). + +use audio::{AudioIo, AudioReport, SAMPLE_RATE}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +fn main() -> anyhow::Result<()> { + let tone = std::env::args().any(|a| a == "--tone"); + let silence = std::env::args().any(|a| a == "--silence"); + let chunks = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(Mutex::new(0i16)); + let drained = Arc::new(AtomicUsize::new(0)); + let (c, p, d) = (chunks.clone(), peak.clone(), drained.clone()); + let io = AudioIo::open(Arc::new(move |report| match report { + AudioReport::Captured(samples) => { + c.fetch_add(1, Ordering::Relaxed); + let max = samples + .iter() + .map(|s| s.saturating_abs()) + .max() + .unwrap_or(0); + let mut peak = p.lock().unwrap(); + *peak = (*peak).max(max); + } + AudioReport::Drained => { + d.fetch_add(1, Ordering::Relaxed); + } + AudioReport::Failed(e) => eprintln!("device failed: {e}"), + }))?; + if silence { + io.play(&vec![0; SAMPLE_RATE as usize]); + } + if tone { + let samples: Vec = (0..SAMPLE_RATE) + .map(|i| { + let t = i as f32 / SAMPLE_RATE as f32; + ((t * 440.0 * std::f32::consts::TAU).sin() * 3000.0) as i16 + }) + .collect(); + io.play(&samples); + } + std::thread::sleep(Duration::from_secs(2)); + println!( + "captured chunks: {} (expected ~40), peak level: {}, played samples: {}, drained: {}", + chunks.load(Ordering::Relaxed), + peak.lock().unwrap(), + io.played_samples(), + drained.load(Ordering::Relaxed) + ); + Ok(()) +} diff --git a/crates/audio/src/cpal_backend.rs b/crates/audio/src/cpal_backend.rs new file mode 100644 index 00000000..ec67d2dc --- /dev/null +++ b/crates/audio/src/cpal_backend.rs @@ -0,0 +1,146 @@ +//! Platforms without system echo cancellation: plain `cpal` streams. +//! +//! Without echo cancellation the microphone would hear the model, so it is +//! gated while audio plays and for a short tail after (half duplex). + +use crate::resample::Resampler; +use crate::{AudioReport, SAMPLE_RATE, Shared}; +use anyhow::{Context, Result, anyhow}; +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use cpal::{SampleFormat, Stream, StreamConfig}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +/// How long after playback the microphone stays gated (room echo). +const ECHO_TAIL: Duration = Duration::from_millis(300); + +pub struct Devices { + _input: Stream, + _output: Stream, +} + +impl Devices { + pub fn start(shared: Arc) -> Result { + let host = cpal::default_host(); + let input_device = host.default_input_device().context("No microphone found")?; + let output_device = host.default_output_device().context("No speakers found")?; + let input = build_input(&input_device, shared.clone())?; + let output = build_output(&output_device, shared)?; + input.play()?; + output.play()?; + Ok(Self { + _input: input, + _output: output, + }) + } +} + +fn report_error(shared: &Arc) -> impl FnMut(cpal::StreamError) + Send + 'static { + let shared = shared.clone(); + move |e| (shared.sink)(AudioReport::Failed(e.to_string())) +} + +fn build_input(device: &cpal::Device, shared: Arc) -> Result { + let supported = device.default_input_config()?; + let format = supported.sample_format(); + let config: StreamConfig = supported.into(); + let channels = config.channels as usize; + let mut resampler = Resampler::new(config.sample_rate.0, SAMPLE_RATE); + let mut mono = Vec::new(); + let mut resampled = Vec::new(); + let mut pending = Vec::new(); + let mut gated_until: Option = None; + let errors = report_error(&shared); + + let mut on_samples = move |frames: &mut dyn Iterator| { + mono.clear(); + let mut frame_sum = 0.0; + for (i, sample) in frames.enumerate() { + frame_sum += sample; + if (i + 1) % channels == 0 { + mono.push(frame_sum / channels as f32); + frame_sum = 0.0; + } + } + if shared.playback.is_playing() { + gated_until = Some(Instant::now() + ECHO_TAIL); + } + if gated_until.is_some_and(|until| Instant::now() < until) { + pending.clear(); + return; + } + resampled.clear(); + resampler.process(&mono, &mut resampled); + shared.capture(&mut pending, resampled.iter().copied()); + }; + + let stream = match format { + SampleFormat::F32 => device.build_input_stream( + &config, + move |data: &[f32], _| on_samples(&mut data.iter().copied()), + errors, + None, + )?, + SampleFormat::I16 => device.build_input_stream( + &config, + move |data: &[i16], _| { + on_samples(&mut data.iter().map(|s| *s as f32 / i16::MAX as f32)) + }, + errors, + None, + )?, + other => return Err(anyhow!("Unsupported microphone sample format {other:?}")), + }; + Ok(stream) +} + +fn build_output(device: &cpal::Device, shared: Arc) -> Result { + let supported = device.default_output_config()?; + let format = supported.sample_format(); + let config: StreamConfig = supported.into(); + let channels = config.channels as usize; + let device_rate = config.sample_rate.0; + let mut resampler = Resampler::new(SAMPLE_RATE, device_rate); + let mut source = Vec::new(); + let mut converted: Vec = Vec::new(); + let errors = report_error(&shared); + + // Fill `frames` device frames (at the device rate) from the 24 kHz queue. + let mut fill = move |frames: usize| -> Vec { + while converted.len() < frames { + let need = ((frames - converted.len()) as u64 * SAMPLE_RATE as u64 / device_rate as u64) + .max(1) as usize; + source.resize(need, 0.0); + shared.playback.render(&mut source); + resampler.process(&source, &mut converted); + } + converted.drain(..frames).collect() + }; + + let stream = match format { + SampleFormat::F32 => device.build_output_stream( + &config, + move |data: &mut [f32], _| { + let mono = fill(data.len() / channels); + for (frame, sample) in data.chunks_mut(channels).zip(mono) { + frame.fill(sample); + } + }, + errors, + None, + )?, + SampleFormat::I16 => device.build_output_stream( + &config, + move |data: &mut [i16], _| { + let mono = fill(data.len() / channels); + for (frame, sample) in data.chunks_mut(channels).zip(mono) { + frame.fill((sample * i16::MAX as f32) as i16); + } + }, + errors, + None, + )?, + other => return Err(anyhow!("Unsupported speaker sample format {other:?}")), + }; + Ok(stream) +} diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs new file mode 100644 index 00000000..d35988c6 --- /dev/null +++ b/crates/audio/src/lib.rs @@ -0,0 +1,144 @@ +//! Microphone capture and speaker playback for voice conversations. +//! +//! Audio is PCM16 mono at 24 kHz in both directions, the format of realtime +//! speech models. [`AudioIo::open`] starts the default devices on a thread of +//! their own; the handle queues playback and reports through a callback: +//! captured microphone chunks, the moment the speakers drained, and device +//! failures. +//! +//! On macOS the devices run through the voice-processing I/O unit, whose +//! echo cancellation keeps the speakers out of the microphone, so the user +//! can talk over the model. Elsewhere `cpal` streams are used and the +//! microphone is gated while audio plays (half duplex: no barge-in). + +#[cfg(not(target_os = "macos"))] +mod cpal_backend; +#[cfg(target_os = "macos")] +mod macos; +mod playback; +#[cfg(any(test, not(target_os = "macos")))] +mod resample; + +use anyhow::Result; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc; +use std::thread::JoinHandle; + +pub use playback::Playback; + +/// Sample rate of the audio this crate exchanges. +pub const SAMPLE_RATE: u32 = 24_000; +/// Samples per captured chunk (50 ms). +const CAPTURE_CHUNK: usize = 1_200; + +/// What the devices report. +#[derive(Debug, Clone, PartialEq)] +pub enum AudioReport { + /// Microphone samples. + Captured(Vec), + /// The speakers played everything queued. + Drained, + /// The devices stopped working. + Failed(String), +} + +pub type ReportSink = Arc; + +/// State shared between the handle and the device callbacks. +pub(crate) struct Shared { + pub playback: Playback, + pub muted: AtomicBool, + pub sink: ReportSink, +} + +impl Shared { + /// Collects microphone samples into chunks for the sink. + pub fn capture(&self, pending: &mut Vec, samples: impl Iterator) { + if self.muted.load(Ordering::Relaxed) { + pending.clear(); + return; + } + for sample in samples { + pending.push((sample.clamp(-1.0, 1.0) * i16::MAX as f32) as i16); + if pending.len() >= CAPTURE_CHUNK { + (self.sink)(AudioReport::Captured(std::mem::take(pending))); + } + } + } +} + +/// The open devices. Dropping the handle stops them. +pub struct AudioIo { + shared: Arc, + stop: Option>, + thread: Option>, +} + +impl AudioIo { + /// Open the default input and output devices. + pub fn open(sink: ReportSink) -> Result { + let shared = Arc::new(Shared { + playback: Playback::new(sink.clone()), + muted: AtomicBool::new(false), + sink, + }); + let (stop_tx, stop_rx) = mpsc::channel::<()>(); + let (ready_tx, ready_rx) = mpsc::channel::>(); + let thread_shared = shared.clone(); + // Device handles are not `Send` on every platform: they live and die + // on this thread. + let thread = std::thread::Builder::new() + .name("voice-audio".into()) + .spawn(move || { + #[cfg(target_os = "macos")] + let devices = macos::Devices::start(thread_shared); + #[cfg(not(target_os = "macos"))] + let devices = cpal_backend::Devices::start(thread_shared); + match devices { + Ok(devices) => { + let _ = ready_tx.send(Ok(())); + // Wait for the handle to drop (or the sender to go). + let _ = stop_rx.recv(); + drop(devices); + } + Err(e) => { + let _ = ready_tx.send(Err(e)); + } + } + })?; + ready_rx + .recv() + .map_err(|_| anyhow::anyhow!("The audio thread ended before starting"))??; + Ok(Self { + shared, + stop: Some(stop_tx), + thread: Some(thread), + }) + } + + pub fn play(&self, samples: &[i16]) { + self.shared.playback.push(samples); + } + + pub fn clear_playback(&self) { + self.shared.playback.clear(); + } + + pub fn played_samples(&self) -> u64 { + self.shared.playback.played() + } + + pub fn set_muted(&self, muted: bool) { + self.shared.muted.store(muted, Ordering::Relaxed); + } +} + +impl Drop for AudioIo { + fn drop(&mut self) { + drop(self.stop.take()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} diff --git a/crates/audio/src/macos.rs b/crates/audio/src/macos.rs new file mode 100644 index 00000000..8575a106 --- /dev/null +++ b/crates/audio/src/macos.rs @@ -0,0 +1,125 @@ +//! macOS: one voice-processing I/O unit for microphone and speakers. +//! +//! The unit cancels the echo of what it plays from what it records, so the +//! model's voice does not reach the realtime server as user speech. It also +//! converts between the device formats and the 24 kHz mono stream we ask for. + +use crate::{SAMPLE_RATE, Shared}; +use anyhow::{Context, Result}; +use coreaudio::audio_unit::audio_format::LinearPcmFlags; +use coreaudio::audio_unit::render_callback::{self, data}; +use coreaudio::audio_unit::{AudioUnit, Element, IOType, SampleFormat, Scope, StreamFormat}; +use std::sync::Arc; +use tracing::debug; + +// AudioUnitProperties.h +const PROPERTY_STREAM_FORMAT: u32 = 8; +const PROPERTY_ENABLE_IO: u32 = 2003; +const PROPERTY_OTHER_AUDIO_DUCKING: u32 = 2108; +const DUCKING_LEVEL_MIN: u32 = 10; + +/// `AUVoiceIOOtherAudioDuckingConfiguration` (macOS 14+). +#[repr(C)] +struct DuckingConfiguration { + enable_advanced_ducking: u8, + ducking_level: u32, +} + +type Args = render_callback::Args>; + +pub struct Devices { + unit: AudioUnit, +} + +impl Devices { + pub fn start(shared: Arc) -> Result { + let mut unit = AudioUnit::new_uninitialized(IOType::VoiceProcessingIO) + .context("No voice-processing audio unit")?; + let enable: u32 = 1; + unit.set_property( + PROPERTY_ENABLE_IO, + Scope::Input, + Element::Input, + Some(&enable), + ) + .context("Failed to enable the microphone")?; + unit.set_property( + PROPERTY_ENABLE_IO, + Scope::Output, + Element::Output, + Some(&enable), + ) + .context("Failed to enable the speakers")?; + + let format = StreamFormat { + sample_rate: SAMPLE_RATE as f64, + sample_format: SampleFormat::F32, + flags: LinearPcmFlags::IS_FLOAT + | LinearPcmFlags::IS_PACKED + | LinearPcmFlags::IS_NON_INTERLEAVED, + channels: 1, + } + .to_asbd(); + // What the microphone side hands us, and what we hand the speakers. + unit.set_property( + PROPERTY_STREAM_FORMAT, + Scope::Output, + Element::Input, + Some(&format), + ) + .context("Failed to set the capture format")?; + unit.set_property( + PROPERTY_STREAM_FORMAT, + Scope::Input, + Element::Output, + Some(&format), + ) + .context("Failed to set the playback format")?; + + // Voice processing ducks other apps' audio hard by default; keep the + // user's music audible. Not available before macOS 14. + let ducking = DuckingConfiguration { + enable_advanced_ducking: 1, + ducking_level: DUCKING_LEVEL_MIN, + }; + if let Err(e) = unit.set_property( + PROPERTY_OTHER_AUDIO_DUCKING, + Scope::Global, + Element::Output, + Some(&ducking), + ) { + debug!("Ducking configuration not applied: {e}"); + } + + unit.initialize() + .context("Failed to initialize the voice-processing unit")?; + + let capture = shared.clone(); + let mut pending = Vec::with_capacity(crate::CAPTURE_CHUNK); + unit.set_input_callback(move |args: Args| { + if let Some(channel) = args.data.channels().next() { + capture.capture(&mut pending, channel.iter().copied()); + } + Ok(()) + }) + .context("Failed to install the capture callback")?; + + let render = shared; + unit.set_render_callback(move |mut args: Args| { + if let Some(channel) = args.data.channels_mut().next() { + render.playback.render(channel); + } + Ok(()) + }) + .context("Failed to install the playback callback")?; + + unit.start().context("Failed to start the audio devices")?; + Ok(Self { unit }) + } +} + +impl Drop for Devices { + fn drop(&mut self) { + let _ = self.unit.stop(); + } +} diff --git a/crates/audio/src/playback.rs b/crates/audio/src/playback.rs new file mode 100644 index 00000000..ed867363 --- /dev/null +++ b/crates/audio/src/playback.rs @@ -0,0 +1,119 @@ +//! The playback queue between the handle and the output callback. + +use crate::{AudioReport, ReportSink}; +use std::collections::VecDeque; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; + +pub struct Playback { + state: Mutex, + /// Samples handed to the device since the queue was created. + played: AtomicU64, + sink: ReportSink, +} + +#[derive(Default)] +struct State { + queue: VecDeque, + /// Something was queued since the last drain report. + playing: bool, +} + +impl Playback { + pub fn new(sink: ReportSink) -> Self { + Self { + state: Mutex::new(State::default()), + played: AtomicU64::new(0), + sink, + } + } + + pub fn push(&self, samples: &[i16]) { + if samples.is_empty() { + return; + } + let mut state = self.state.lock().unwrap(); + state + .queue + .extend(samples.iter().map(|s| *s as f32 / i16::MAX as f32)); + state.playing = true; + } + + /// Drop everything queued. No drain report follows: the caller stopped + /// playback on purpose. + pub fn clear(&self) { + let mut state = self.state.lock().unwrap(); + state.queue.clear(); + state.playing = false; + } + + pub fn played(&self) -> u64 { + self.played.load(Ordering::Relaxed) + } + + /// Whether audio is queued or was queued since the last drain. + pub fn is_playing(&self) -> bool { + self.state.lock().unwrap().playing + } + + /// Fill a device buffer (24 kHz mono); silence once the queue is empty. + /// Reports the drain when the last queued sample went out. + pub fn render(&self, out: &mut [f32]) { + let mut state = self.state.lock().unwrap(); + let available = state.queue.len().min(out.len()); + for (slot, sample) in out.iter_mut().zip(state.queue.drain(..available)) { + *slot = sample; + } + out[available..].fill(0.0); + self.played.fetch_add(available as u64, Ordering::Relaxed); + let drained = state.playing && state.queue.is_empty(); + if drained { + state.playing = false; + } + drop(state); + if drained { + (self.sink)(AudioReport::Drained); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + fn recording() -> (Playback, Arc>>) { + let reports = Arc::new(Mutex::new(Vec::new())); + let sink_reports = reports.clone(); + let playback = Playback::new(Arc::new(move |r| sink_reports.lock().unwrap().push(r))); + (playback, reports) + } + + #[test] + fn counts_played_samples_and_reports_the_drain_once() { + let (playback, reports) = recording(); + playback.push(&[i16::MAX; 300]); + let mut buffer = [0.0; 256]; + playback.render(&mut buffer); + assert_eq!(playback.played(), 256); + assert!(reports.lock().unwrap().is_empty()); + playback.render(&mut buffer); + assert_eq!(playback.played(), 300); + assert!((buffer[43] - 1.0).abs() < 1e-6 && buffer[44] == 0.0); + playback.render(&mut buffer); + assert_eq!(*reports.lock().unwrap(), vec![AudioReport::Drained]); + } + + #[test] + fn clearing_drops_the_queue_without_a_drain_report() { + let (playback, reports) = recording(); + playback.push(&[1; 1000]); + playback.clear(); + let mut buffer = [1.0; 64]; + playback.render(&mut buffer); + assert!(buffer.iter().all(|s| *s == 0.0)); + assert_eq!(playback.played(), 0); + assert!(reports.lock().unwrap().is_empty()); + assert!(!playback.is_playing()); + } +} diff --git a/crates/audio/src/resample.rs b/crates/audio/src/resample.rs new file mode 100644 index 00000000..63372817 --- /dev/null +++ b/crates/audio/src/resample.rs @@ -0,0 +1,65 @@ +//! Linear resampling between device rates and 24 kHz. Plenty for speech. + +/// Streaming linear resampler; keeps its phase across calls. +pub struct Resampler { + /// Input samples per output sample. + step: f64, + /// Position of the next output sample, relative to `last`. + position: f64, + last: f32, +} + +impl Resampler { + pub fn new(from_rate: u32, to_rate: u32) -> Self { + Self { + step: from_rate as f64 / to_rate as f64, + position: 0.0, + last: 0.0, + } + } + + /// Resample a chunk of mono input, appending to `out`. + pub fn process(&mut self, input: &[f32], out: &mut Vec) { + for &sample in input { + // Emit every output position between `last` (0.0) and `sample` + // (1.0). + while self.position <= 1.0 { + let t = self.position as f32; + out.push(self.last + (sample - self.last) * t); + self.position += self.step; + } + self.position -= 1.0; + self.last = sample; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rate_ratio_sets_the_output_length() { + let input = vec![0.5; 4800]; + let mut out = Vec::new(); + Resampler::new(48_000, 24_000).process(&input, &mut out); + assert!((out.len() as i64 - 2400).abs() <= 2, "{}", out.len()); + + let mut up = Vec::new(); + Resampler::new(24_000, 48_000).process(&input, &mut up); + assert!((up.len() as i64 - 9600).abs() <= 2, "{}", up.len()); + } + + #[test] + fn phase_carries_across_chunks() { + let input: Vec = (0..441).map(|i| i as f32).collect(); + let mut whole = Vec::new(); + Resampler::new(44_100, 24_000).process(&input, &mut whole); + let mut chunked = Vec::new(); + let mut resampler = Resampler::new(44_100, 24_000); + for chunk in input.chunks(37) { + resampler.process(chunk, &mut chunked); + } + assert_eq!(whole, chunked); + } +} diff --git a/crates/code_assistant/Cargo.toml b/crates/code_assistant/Cargo.toml index c9e46579..9ea2ddd8 100644 --- a/crates/code_assistant/Cargo.toml +++ b/crates/code_assistant/Cargo.toml @@ -22,6 +22,7 @@ default = [ "mcp-server", "terminal-frontend", "document-conversion", + "voice", ] document-conversion = ["code_assistant_core/document-conversion"] # Each frontend and protocol mode is an optional compile unit; a headless @@ -32,12 +33,15 @@ gpui-frontend = ["dep:ui_gpui", "dep:async-channel"] browser-panel = ["gpui-frontend", "ui_gpui/browser-panel"] mcp-server = ["dep:mcp_server"] terminal-frontend = ["dep:ui_terminal"] +# Voice mode in the GPUI frontend: microphone and speakers via the audio crate. +voice = ["gpui-frontend", "dep:audio"] [dependencies] code_assistant_core = { path = "../code_assistant_core", default-features = false } mcp_server = { path = "../mcp_server", optional = true } ui_acp = { path = "../ui_acp", optional = true } ui_gpui = { path = "../ui_gpui", optional = true } +audio = { path = "../audio", optional = true } # Migration progress from the backend thread to the GPUI startup window async-channel = { version = "2.5.0", optional = true } ui_terminal = { path = "../ui_terminal", optional = true } diff --git a/crates/code_assistant/assets/Entitlements.plist b/crates/code_assistant/assets/Entitlements.plist index f81a8ce3..1d6391a5 100644 --- a/crates/code_assistant/assets/Entitlements.plist +++ b/crates/code_assistant/assets/Entitlements.plist @@ -21,5 +21,9 @@ com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.device.audio-input + diff --git a/crates/code_assistant/assets/Info.plist b/crates/code_assistant/assets/Info.plist index d3ee9fc7..54cce2eb 100644 --- a/crates/code_assistant/assets/Info.plist +++ b/crates/code_assistant/assets/Info.plist @@ -30,5 +30,7 @@ Code Assistant contributors LSApplicationCategoryType public.app-category.developer-tools + NSMicrophoneUsageDescription + Voice mode listens to you so you can talk to your coding conversations. diff --git a/crates/code_assistant/src/app/gpui.rs b/crates/code_assistant/src/app/gpui.rs index c278199d..70d12f8e 100644 --- a/crates/code_assistant/src/app/gpui.rs +++ b/crates/code_assistant/src/app/gpui.rs @@ -63,6 +63,8 @@ pub fn run(config: AgentRunConfig) -> Result<()> { // Create the session command service. The GUI gets the handle; the // worker runs on the backend tokio runtime below. The GUI consumes the // broadcast stream (see ui_gpui's event bridge). + #[cfg(feature = "voice")] + let voice_events = events.clone(); let (service, service_worker) = SessionService::new( multi_session_manager, Arc::new(AgentRuntimeOptions { @@ -78,6 +80,20 @@ pub fn run(config: AgentRunConfig) -> Result<()> { ); gui.set_session_service(service.clone()); + // Voice mode: a global voice agent over all sessions. Its worker runs + // on the backend runtime next to the session service's. + #[cfg(feature = "voice")] + let voice_worker = { + let (voice, worker) = code_assistant_core::voice::VoiceService::new( + service.clone(), + voice_events, + super::voice::audio_factory(), + code_assistant_core::voice::default_connector_factory(), + ); + gui.set_voice_service(voice); + worker + }; + let gui_for_thread = gui.clone(); let task = config.task.clone(); @@ -126,6 +142,8 @@ pub fn run(config: AgentRunConfig) -> Result<()> { tracing::warn!("Startup settlement failed: {e:#}"); } let worker = tokio::spawn(service_worker); + #[cfg(feature = "voice")] + tokio::spawn(voice_worker); // Pre-warm the registry (connects configured MCP servers) in the // background: remote servers take seconds, and the sidebar's diff --git a/crates/code_assistant/src/app/mod.rs b/crates/code_assistant/src/app/mod.rs index b5923ef5..cbd14919 100644 --- a/crates/code_assistant/src/app/mod.rs +++ b/crates/code_assistant/src/app/mod.rs @@ -6,6 +6,8 @@ pub mod gpui; pub mod server; #[cfg(feature = "terminal-frontend")] pub mod terminal; +#[cfg(feature = "voice")] +pub mod voice; pub use code_assistant_core::config::AgentRunConfig; use code_assistant_core::persistence::{FileSessionPersistence, MigrationProgress}; diff --git a/crates/code_assistant/src/app/voice.rs b/crates/code_assistant/src/app/voice.rs new file mode 100644 index 00000000..d87db855 --- /dev/null +++ b/crates/code_assistant/src/app/voice.rs @@ -0,0 +1,39 @@ +//! The audio devices of voice mode, from the `audio` crate. + +use code_assistant_core::voice::{AudioEvent, AudioFactory, VoiceAudio}; +use std::sync::Arc; + +struct Devices(audio::AudioIo); + +impl VoiceAudio for Devices { + fn play(&mut self, samples: Vec) { + self.0.play(&samples); + } + + fn clear_playback(&mut self) { + self.0.clear_playback(); + } + + fn played_samples(&self) -> u64 { + self.0.played_samples() + } + + fn set_capture_muted(&mut self, muted: bool) { + self.0.set_muted(muted); + } +} + +/// Opens the default microphone and speakers for each voice session. +pub fn audio_factory() -> AudioFactory { + Arc::new(|events| { + let sink: audio::ReportSink = Arc::new(move |report| { + let event = match report { + audio::AudioReport::Captured(samples) => AudioEvent::Captured(samples), + audio::AudioReport::Drained => AudioEvent::Drained, + audio::AudioReport::Failed(message) => AudioEvent::Failed(message), + }; + let _ = events.send(event); + }); + Ok(Box::new(Devices(audio::AudioIo::open(sink)?)) as Box) + }) +} diff --git a/crates/code_assistant_core/src/lib.rs b/crates/code_assistant_core/src/lib.rs index 2532f445..f06b66a5 100644 --- a/crates/code_assistant_core/src/lib.rs +++ b/crates/code_assistant_core/src/lib.rs @@ -29,6 +29,7 @@ pub mod types; pub mod ui; pub mod utils; pub mod version; +pub mod voice; // Mock building blocks (LLM provider, UI, project manager, tool fixtures). // Compiled for our own tests and, behind the `test-utils` feature, for diff --git a/crates/code_assistant_core/src/session/manager.rs b/crates/code_assistant_core/src/session/manager.rs index d19178a5..289fb1ed 100644 --- a/crates/code_assistant_core/src/session/manager.rs +++ b/crates/code_assistant_core/src/session/manager.rs @@ -1526,6 +1526,27 @@ impl SessionManager { self.persistence.list_chat_sessions() } + /// A filtered projection of a stored session, read without loading it + /// into the manager (see [`crate::session_query`]). + pub fn session_content( + &self, + session_id: &str, + projection: &crate::session_query::ContentProjection, + ) -> Result { + crate::session_query::get_session_content(&self.persistence, session_id, projection) + } + + /// The activity state of every loaded session. Sessions not loaded are + /// idle. + pub fn activity_states( + &self, + ) -> HashMap { + self.active_sessions + .iter() + .map(|(id, instance)| (id.clone(), instance.get_activity_state())) + .collect() + } + /// Delete a session pub fn delete_session(&mut self, session_id: &str) -> Result<()> { // Remove from active sessions diff --git a/crates/code_assistant_core/src/session/service.rs b/crates/code_assistant_core/src/session/service.rs index 0262ce2d..36a714c7 100644 --- a/crates/code_assistant_core/src/session/service.rs +++ b/crates/code_assistant_core/src/session/service.rs @@ -582,6 +582,33 @@ impl SessionService { .await } + /// A filtered projection of a stored session's content, read without + /// making it the active session. + pub async fn session_content( + &self, + session_id: String, + projection: crate::session_query::ContentProjection, + ) -> Result { + self.call(move |ctx| async move { + let manager = ctx.manager.lock().await; + manager.session_content(&session_id, &projection) + }) + .await + } + + /// The activity state of every loaded session; sessions absent from + /// the map are idle. + pub async fn session_activity_states( + &self, + ) -> Result> + { + self.call(move |ctx| async move { + let manager = ctx.manager.lock().await; + Ok(manager.activity_states()) + }) + .await + } + /// Incremental session refresh triggered by the file watcher. Compares /// the on-disk state with the in-memory state and emits only the delta /// as [`UiEvent`]s; falls back to a full reload if that fails. diff --git a/crates/code_assistant_core/src/ui/ui_events.rs b/crates/code_assistant_core/src/ui/ui_events.rs index 2764702b..aba4ca1c 100644 --- a/crates/code_assistant_core/src/ui/ui_events.rs +++ b/crates/code_assistant_core/src/ui/ui_events.rs @@ -389,4 +389,13 @@ pub enum UiEvent { /// Configuration files (providers.json / models.json) were changed on disk. /// The UI should reload model lists, settings sections, etc. ConfigChanged, + + // === Voice mode (app-scoped) === + /// The voice agent's state changed. + VoiceStatusChanged { status: crate::voice::VoiceStatus }, + /// A line of the voice conversation: something said, a tool call, or a + /// notification delivered to the voice model. + VoiceTranscript { + entry: crate::voice::TranscriptEntry, + }, } diff --git a/crates/code_assistant_core/src/voice/agent.rs b/crates/code_assistant_core/src/voice/agent.rs new file mode 100644 index 00000000..625cf983 --- /dev/null +++ b/crates/code_assistant_core/src/voice/agent.rs @@ -0,0 +1,468 @@ +//! The voice agent's event loop. +//! +//! One task per voice session. It owns the realtime connection, the audio +//! devices and the [`Floor`], and turns everything that happens — server +//! events, audio reports, tool completions, conversation events, timers — +//! into floor inputs, then carries out the floor's commands. + +use super::floor::{Floor, FloorCommand, FloorConfig, FloorInput, FloorState, Timer}; +use super::source::{NotificationSource, OpenRequests}; +use super::tools::{ToolEffect, ToolOutcome, VoiceTools}; +use super::{ + AudioEvent, AudioFactory, TranscriptEntry, TranscriptRole, VoiceActivity, VoiceAudio, + VoiceConfig, VoiceStatus, +}; +use crate::session::SessionService; +use crate::session::event_stream::{EventStream, StreamError, Subscription}; +use crate::ui::UiEvent; +use anyhow::{Context, Result, anyhow}; +use llm::realtime::{ + ClientEvent, Incoming, RealtimeConnection, RealtimeConnector, SAMPLE_RATE, ServerEvent, + SessionSettings, decode_pcm16, +}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc; +use tokio::time::Instant; +use tracing::{debug, info, warn}; + +/// Commands from the [`super::VoiceService`] to a running agent. +#[derive(Debug)] +pub(super) enum Control { + Stop, + SetMuted(bool), +} + +const RECONNECT_ATTEMPTS: u32 = 3; +/// Transcript entries replayed into a renewed realtime session. +const RESEED_ENTRIES: usize = 30; + +const INSTRUCTIONS: &str = "\ +You are the voice assistant of a coding-agent app. The user talks to you hands-free about their \ +conversations with coding agents: what is running, what an agent found or changed, what an agent \ +should do next. You cannot read or change code yourself. You act only through your tools: list \ +conversations and projects, read a conversation, and send a message to a conversation (which starts \ +a new one when you give no conversation id). + +Speak briefly: one to three sentences unless the user asks for detail. Refer to conversations by \ +title, never read ids or paths aloud, and summarise agent answers instead of reading code or long \ +lists. When you start or message a conversation, confirm in a few words and move on; the agent \ +works in the background. When you write a message for an agent, phrase it as a clear instruction \ +with all the context the user gave you. If it is unclear which conversation the user means, ask. + +Messages marked [background notification] come from the app, not the user. They tell you that a \ +conversation finished or waits for the user. Mention them briefly when they are useful, after \ +calling get_conversation for results. Answer in the language the user speaks."; + +pub(super) struct VoiceAgent { + config: VoiceConfig, + connector: Arc, + events: EventStream, + tools: VoiceTools, + source: NotificationSource, + subscription: Subscription, + floor: Floor, + connection: RealtimeConnection, + audio: Box, + audio_rx: mpsc::UnboundedReceiver, + tool_tx: mpsc::UnboundedSender<(u64, String, ToolOutcome)>, + tool_rx: mpsc::UnboundedReceiver<(u64, String, ToolOutcome)>, + /// Bumped per realtime connection; tool results of an older one are + /// dropped (their call ids mean nothing to the new session). + generation: u64, + timer: Option<(Timer, Instant)>, + muted: bool, + status: VoiceStatus, + transcript: Vec, + /// Samples handed to the speakers since the devices opened. + queued_samples: u64, + /// The assistant item whose audio plays, with the sample offset its + /// audio starts at. + audio_item: Option<(String, u64)>, + /// The item playback was stopped for; its late audio deltas are dropped. + discarded_item: Option, +} + +impl VoiceAgent { + /// Open audio and the realtime session. Fails without side effects + /// left running. + pub(super) async fn start( + config: VoiceConfig, + connector: Arc, + service: SessionService, + events: EventStream, + audio_factory: AudioFactory, + ) -> Result { + let open_requests = OpenRequests::default(); + // Subscribe before reading the initial states, so no transition + // falls in between. + let subscription = service.subscribe(); + let source = + NotificationSource::new(service.clone(), config.notify, open_requests.clone()).await?; + let tools = VoiceTools::new(service, open_requests); + + let connection = connector.connect().await?; + let (audio_tx, audio_rx) = mpsc::unbounded_channel(); + let audio = audio_factory(audio_tx).context("Failed to open the audio devices")?; + let (tool_tx, tool_rx) = mpsc::unbounded_channel(); + let floor = Floor::new(FloorConfig { + cooling: config.cooling(), + ..FloorConfig::default() + }); + + let agent = Self { + config, + connector, + events, + tools, + source, + subscription, + floor, + connection, + audio, + audio_rx, + tool_tx, + tool_rx, + generation: 0, + timer: None, + muted: false, + status: VoiceStatus { + activity: VoiceActivity::Connecting, + ..VoiceStatus::default() + }, + transcript: Vec::new(), + queued_samples: 0, + audio_item: None, + discarded_item: None, + }; + agent.send(ClientEvent::SessionUpdate(agent.session_settings())); + Ok(agent) + } + + /// Run until stopped. Returns the error that ended the session, if any. + pub(super) async fn run(mut self, mut control: mpsc::UnboundedReceiver) -> Result<()> { + self.publish_status(); + loop { + let timer = self.timer; + tokio::select! { + command = control.recv() => match command { + Some(Control::SetMuted(muted)) => { + self.muted = muted; + self.audio.set_capture_muted(muted); + self.publish_status(); + } + Some(Control::Stop) | None => return Ok(()), + }, + incoming = self.connection.incoming.recv() => match incoming { + Some(Incoming::Event(event)) => self.on_server_event(event), + Some(Incoming::Closed(reason)) => self.reconnect(reason).await?, + None => self.reconnect(None).await?, + }, + audio = self.audio_rx.recv() => match audio { + Some(AudioEvent::Captured(samples)) => { + if !self.muted { + self.send(ClientEvent::AppendAudio(samples)); + } + } + Some(AudioEvent::Drained) => self.floor_input(FloorInput::PlaybackDrained), + Some(AudioEvent::Failed(message)) => { + return Err(anyhow!("Audio device failed: {message}")); + } + None => return Err(anyhow!("The audio devices closed")), + }, + Some((generation, call_id, outcome)) = self.tool_rx.recv() => { + if generation == self.generation { + self.on_tool_completed(call_id, outcome); + } + } + event = self.subscription.recv() => { + let inputs = match event { + Ok(event) => self.source.on_event(&event).await, + Err(StreamError::Lagged { .. }) => self.source.resync().await, + Err(StreamError::Closed) => return Err(anyhow!("The app is shutting down")), + }; + for input in inputs { + self.floor_input(input); + } + } + _ = sleep_until(timer) => { + if let Some((fired, _)) = self.timer.take() { + self.floor_input(FloorInput::TimerFired(fired)); + } + } + } + } + } + + fn on_server_event(&mut self, event: ServerEvent) { + match event { + ServerEvent::SessionCreated | ServerEvent::SessionUpdated => { + if self.status.activity == VoiceActivity::Connecting { + self.status.activity = VoiceActivity::Listening; + self.publish_status(); + } + } + ServerEvent::ResponseCreated { .. } => self.floor_input(FloorInput::ResponseCreated), + ServerEvent::OutputItemAdded { .. } => {} + ServerEvent::AudioDelta { item_id, delta } => self.on_audio_delta(item_id, &delta), + ServerEvent::AudioTranscriptDone { transcript, .. } => { + self.record(TranscriptRole::Assistant, transcript) + } + ServerEvent::InputTranscription { transcript, .. } => { + self.record(TranscriptRole::User, transcript) + } + ServerEvent::SpeechStarted => self.floor_input(FloorInput::SpeechStarted), + ServerEvent::SpeechStopped => self.floor_input(FloorInput::SpeechStopped), + ServerEvent::OutputItemDone { item } => { + if item.kind == "function_call" { + let name = item.name.unwrap_or_default(); + let call_id = item.call_id.unwrap_or_default(); + let arguments = item.arguments.unwrap_or_default(); + self.start_tool(name, call_id, arguments); + } + } + ServerEvent::ResponseDone { response } => { + debug!( + "Realtime response {} done: {:?}", + response.id, response.status + ); + let pending = self.pending_playback(); + self.floor_input(FloorInput::ResponseDone { + pending_playback: pending, + }); + } + ServerEvent::Error { error } => { + warn!( + "Realtime error ({}): {}", + error.code.as_deref().unwrap_or("-"), + error.message + ); + self.floor_input(FloorInput::ServerError); + } + ServerEvent::Other => {} + } + } + + fn on_audio_delta(&mut self, item_id: String, delta: &str) { + if self.discarded_item.as_deref() == Some(item_id.as_str()) { + return; + } + let samples = match decode_pcm16(delta) { + Ok(samples) => samples, + Err(e) => { + warn!("Undecodable realtime audio: {e}"); + return; + } + }; + if self + .audio_item + .as_ref() + .is_none_or(|(id, _)| *id != item_id) + { + self.audio_item = Some((item_id, self.queued_samples)); + } + self.queued_samples += samples.len() as u64; + self.audio.play(samples); + } + + fn start_tool(&mut self, name: String, call_id: String, arguments: String) { + info!("Voice tool call {name} {arguments}"); + self.record(TranscriptRole::Tool, format!("{name} {arguments}")); + self.floor_input(FloorInput::ToolStarted); + let tools = self.tools.clone(); + let tx = self.tool_tx.clone(); + let generation = self.generation; + tokio::spawn(async move { + let outcome = tools.call(&name, &arguments).await; + let _ = tx.send((generation, call_id, outcome)); + }); + } + + fn on_tool_completed(&mut self, call_id: String, outcome: ToolOutcome) { + for effect in outcome.effects { + match effect { + ToolEffect::Read(conversation_id) => { + self.floor_input(FloorInput::ConversationRead { conversation_id }) + } + ToolEffect::Touched(conversation_id) => self.source.touch(&conversation_id), + } + } + self.floor_input(FloorInput::ToolCompleted { + call_id, + output: outcome.output, + }); + } + + fn floor_input(&mut self, input: FloorInput) { + let commands = self.floor.handle(input); + self.execute(commands); + self.sync_status(); + } + + fn execute(&mut self, commands: Vec) { + for command in commands { + match command { + FloorCommand::CancelResponse => self.send(ClientEvent::CancelResponse), + FloorCommand::StopPlayback => self.stop_playback(), + FloorCommand::SendToolOutput { call_id, output } => { + self.send(ClientEvent::FunctionCallOutput { call_id, output }) + } + FloorCommand::SendNotification(text) => { + self.record(TranscriptRole::Notification, text.clone()); + self.send(ClientEvent::SystemMessage(text)); + } + FloorCommand::CreateResponse => self.send(ClientEvent::CreateResponse), + FloorCommand::StartTimer(timer, after) => { + self.timer = Some((timer, Instant::now() + after)) + } + FloorCommand::CancelTimer => self.timer = None, + } + } + } + + /// Barge-in: silence the speakers and cut the server's copy of the + /// item to what the user heard, so the model does not believe it said + /// the rest. + fn stop_playback(&mut self) { + let played = self.audio.played_samples(); + if let Some((item_id, start)) = self.audio_item.take() { + if played < self.queued_samples { + let heard = played.saturating_sub(start); + self.send(ClientEvent::TruncateItem { + item_id: item_id.clone(), + audio_end_ms: samples_to_ms(heard), + }); + } + self.discarded_item = Some(item_id); + } + self.audio.clear_playback(); + self.queued_samples = played; + } + + fn pending_playback(&self) -> Duration { + let pending = self + .queued_samples + .saturating_sub(self.audio.played_samples()); + Duration::from_millis(samples_to_ms(pending)) + } + + /// Replace a closed realtime session with a fresh one that knows the + /// conversation so far. + async fn reconnect(&mut self, reason: Option) -> Result<()> { + info!("Realtime session closed ({reason:?}); reconnecting"); + self.status.activity = VoiceActivity::Connecting; + self.publish_status(); + self.audio.clear_playback(); + self.queued_samples = self.audio.played_samples(); + self.audio_item = None; + + let mut last_error = anyhow!("connection closed: {}", reason.unwrap_or_default()); + for attempt in 0..RECONNECT_ATTEMPTS { + if attempt > 0 { + tokio::time::sleep(Duration::from_secs(1 << attempt)).await; + } + match self.connector.connect().await { + Ok(connection) => { + self.connection = connection; + self.generation += 1; + self.send(ClientEvent::SessionUpdate(self.session_settings())); + if let Some(summary) = self.transcript_summary() { + self.send(ClientEvent::SystemMessage(summary)); + } + let commands = self.floor.reconnected(); + self.execute(commands); + return Ok(()); + } + Err(e) => { + warn!("Realtime reconnect attempt {} failed: {e:#}", attempt + 1); + last_error = e; + } + } + } + Err(last_error.context("The realtime session could not be renewed")) + } + + fn transcript_summary(&self) -> Option { + let start = self.transcript.len().saturating_sub(RESEED_ENTRIES); + let lines: Vec = self.transcript[start..] + .iter() + .filter_map(|entry| match entry.role { + TranscriptRole::User => Some(format!("User: {}", entry.text)), + TranscriptRole::Assistant => Some(format!("You: {}", entry.text)), + TranscriptRole::Tool => Some(format!("(tool call: {})", entry.text)), + TranscriptRole::Notification => None, + }) + .collect(); + (!lines.is_empty()).then(|| { + format!( + "The connection was renewed. The voice conversation so far:\n{}", + lines.join("\n") + ) + }) + } + + fn session_settings(&self) -> SessionSettings { + SessionSettings { + model: self + .connector + .declares_model() + .then(|| self.config.model.clone()), + instructions: INSTRUCTIONS.to_string(), + voice: self.config.voice.clone(), + tools: VoiceTools::definitions(), + turn_detection: self.config.turn_detection(), + transcription_model: self.config.transcription(), + } + } + + fn send(&self, event: ClientEvent) { + // A closed connection shows up on the incoming side. + let _ = self.connection.outgoing.send(event); + } + + fn record(&mut self, role: TranscriptRole, text: String) { + if text.trim().is_empty() { + return; + } + let entry = TranscriptEntry { role, text }; + self.events.publish_app(UiEvent::VoiceTranscript { + entry: entry.clone(), + }); + self.transcript.push(entry); + } + + fn sync_status(&mut self) { + if self.status.activity == VoiceActivity::Connecting { + return; + } + let activity = match self.floor.state() { + FloorState::Speaking => VoiceActivity::Speaking, + FloorState::UserTurn => VoiceActivity::UserSpeaking, + FloorState::Idle | FloorState::Cooling => VoiceActivity::Listening, + }; + let queued = self.floor.queued_notifications(); + if activity != self.status.activity || queued != self.status.queued_notifications { + self.status.activity = activity; + self.status.queued_notifications = queued; + self.publish_status(); + } + } + + fn publish_status(&mut self) { + self.status.muted = self.muted; + self.events.publish_app(UiEvent::VoiceStatusChanged { + status: self.status.clone(), + }); + } +} + +fn samples_to_ms(samples: u64) -> u64 { + samples * 1000 / SAMPLE_RATE as u64 +} + +async fn sleep_until(timer: Option<(Timer, Instant)>) { + match timer { + Some((_, deadline)) => tokio::time::sleep_until(deadline).await, + None => std::future::pending().await, + } +} diff --git a/crates/code_assistant_core/src/voice/config.rs b/crates/code_assistant_core/src/voice/config.rs new file mode 100644 index 00000000..25ec7b67 --- /dev/null +++ b/crates/code_assistant_core/src/voice/config.rs @@ -0,0 +1,186 @@ +//! Voice mode configuration, persisted at `/voice.json`. + +use crate::utils::file_utils::atomic_write_json; +use anyhow::{Result, anyhow, bail}; +use llm::provider_config::ConfigurationSystem; +use llm::realtime::RealtimeConnector; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; +use tracing::warn; + +/// Which conversations report to the voice agent. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub enum NotifyScope { + /// Every conversation of this process. + #[default] + All, + /// Only conversations the voice agent created or messaged. + Touched, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct VoiceConfig { + /// Provider id from `providers.json`; empty while unconfigured. + pub provider: String, + /// The realtime model; for an AI Core provider the key of its `models` + /// map whose deployment serves voice mode. + pub model: String, + pub voice: String, + /// WebSocket URL overriding the one derived from the provider's + /// `base_url`, for compatible gateways. + #[serde(skip_serializing_if = "Option::is_none")] + pub url: Option, + /// Model transcribing the user's speech (for the transcript); empty + /// turns transcription off. + pub transcription_model: String, + /// Semantic turn detection eagerness: `low`, `medium`, `high`, `auto`. + pub vad_eagerness: String, + /// Silence after the model spoke before notifications may speak. + pub cooling_ms: u64, + pub notify: NotifyScope, +} + +impl Default for VoiceConfig { + fn default() -> Self { + Self { + provider: String::new(), + model: "gpt-realtime".into(), + voice: "marin".into(), + url: None, + transcription_model: "gpt-4o-mini-transcribe".into(), + vad_eagerness: "auto".into(), + cooling_ms: 1500, + notify: NotifyScope::All, + } + } +} + +impl VoiceConfig { + pub fn path() -> PathBuf { + crate::config_dir::config_dir().join("voice.json") + } + + /// Load the config; a missing or malformed file yields the defaults. + pub fn load() -> Self { + Self::load_from(&Self::path()) + } + + pub fn load_from(path: &Path) -> Self { + let Ok(content) = std::fs::read_to_string(path) else { + return Self::default(); + }; + serde_json::from_str(&content).unwrap_or_else(|e| { + warn!("Failed to parse {}: {e}; using defaults", path.display()); + Self::default() + }) + } + + pub fn save(&self) -> Result<()> { + atomic_write_json(&Self::path(), self) + } + + pub fn is_configured(&self) -> bool { + !self.provider.is_empty() && !self.model.is_empty() + } + + pub fn cooling(&self) -> Duration { + Duration::from_millis(self.cooling_ms) + } + + pub fn turn_detection(&self) -> Value { + let eagerness = if self.vad_eagerness.is_empty() { + "auto" + } else { + &self.vad_eagerness + }; + // The voice agent handles barge-in itself (cancel, truncate), so the + // server only detects turns and answers them. + json!({ + "type": "semantic_vad", + "eagerness": eagerness, + "create_response": true, + "interrupt_response": false, + }) + } + + pub fn transcription(&self) -> Option { + (!self.transcription_model.is_empty()).then(|| self.transcription_model.clone()) + } + + /// The connector for the configured provider in `providers.json`. + pub fn connector(&self) -> Result> { + if !self.is_configured() { + bail!("Voice mode is not configured: choose a provider and model in Settings → Voice"); + } + let providers = ConfigurationSystem::load_providers_config(None)?; + let provider = providers + .get(&self.provider) + .ok_or_else(|| anyhow!("Unknown provider '{}' in voice.json", self.provider))?; + llm::realtime::connector_for_provider(provider, &self.model, self.url.as_deref()) + } +} + +pub use llm::realtime::supports_realtime; + +#[cfg(test)] +mod tests { + use super::*; + use llm::provider_config::ProviderConfig; + + fn provider(kind: &str, config: Value) -> ProviderConfig { + ProviderConfig { + label: "p".into(), + provider: kind.into(), + config, + } + } + + #[test] + fn openai_and_ai_core_providers_serve_realtime() { + assert!(supports_realtime(&provider("openai-responses", json!({})))); + assert!(supports_realtime(&provider("ai-core", json!({})))); + assert!(!supports_realtime(&provider("anthropic", json!({})))); + } + + #[test] + fn ai_core_needs_a_deployment_for_the_voice_model() { + let p = provider( + "ai-core", + json!({ + "client_id": "id", "client_secret": "s", "token_url": "https://t", + "api_base_url": "https://api.example/v2/inference", + "models": {"gpt-realtime": {"deployment": "d1", "api_type": "openai"}} + }), + ); + let connector = llm::realtime::connector_for_provider(&p, "gpt-realtime", None).unwrap(); + assert!(!connector.declares_model()); + let missing = llm::realtime::connector_for_provider(&p, "other", None); + assert!(missing.err().unwrap().to_string().contains("no deployment")); + } + + #[test] + fn non_realtime_providers_are_rejected() { + let p = provider("anthropic", json!({"api_key": "k"})); + assert!(llm::realtime::connector_for_provider(&p, "m", None).is_err()); + } + + #[test] + fn missing_file_yields_defaults_and_partial_files_merge() { + let dir = tempfile::tempdir().unwrap(); + assert_eq!( + VoiceConfig::load_from(&dir.path().join("none.json")), + VoiceConfig::default() + ); + let path = dir.path().join("voice.json"); + std::fs::write(&path, r#"{"provider":"openai","notify":"touched"}"#).unwrap(); + let config = VoiceConfig::load_from(&path); + assert_eq!(config.provider, "openai"); + assert_eq!(config.notify, NotifyScope::Touched); + assert_eq!(config.model, "gpt-realtime"); + } +} diff --git a/crates/code_assistant_core/src/voice/floor.rs b/crates/code_assistant_core/src/voice/floor.rs new file mode 100644 index 00000000..018eaa8e --- /dev/null +++ b/crates/code_assistant_core/src/voice/floor.rs @@ -0,0 +1,719 @@ +//! Who has the floor in a voice conversation, and when background work may +//! speak up. +//! +//! [`Floor`] is a pure state machine: [`FloorInput`]s go in, [`FloorCommand`]s +//! come out. It does no I/O and has no clock — timers are commands the voice +//! agent runs and reports back as [`FloorInput::TimerFired`]. +//! +//! Two priority levels: +//! +//! 1. **Voice**: the user's speech, the model's answer and the tool calls of +//! that answer. +//! 2. **Background**: notifications about conversations. They are only +//! delivered on a free floor and never cancel or delay level 1. +//! +//! The model generates audio much faster than it plays, so `response.done` +//! does not mean the model stopped talking: the floor is free only once the +//! speakers drained ([`FloorInput::PlaybackDrained`], or the playback timer +//! as a fallback) and then stayed quiet for the cooling time. +//! +//! Invariants (each has a test below): +//! +//! 1. No `response.create` while a response is open. +//! 2. Neither a tool result nor a notification triggers a response while the +//! user has the floor. +//! 3. Notifications flush only from `Idle`: after playback drained *and* the +//! cooling time passed in silence. +//! 4. A notification never cancels a response or stops playback. +//! 5. Barge-in stops playback (the agent truncates the item to what was +//! heard) and cancels the open response. + +use super::notifications::{Notification, NotificationQueue}; +use std::time::Duration; + +#[derive(Debug, Clone)] +pub struct FloorConfig { + /// Silence after playback before background notifications may speak. + pub cooling: Duration, + /// Extra wait beyond the expected playback end before assuming the + /// speakers drained (in case the drain report is lost). + pub playback_grace: Duration, + /// How long after the user stopped speaking the floor stays theirs when + /// no response follows. + pub user_turn_grace: Duration, +} + +impl Default for FloorConfig { + fn default() -> Self { + Self { + cooling: Duration::from_millis(1500), + playback_grace: Duration::from_secs(5), + user_turn_grace: Duration::from_secs(8), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FloorState { + /// Silent, nothing pending. + Idle, + /// A response is being generated or its audio is still playing. + Speaking, + /// The user is speaking, or spoke and is owed an answer. + UserTurn, + /// Playback drained; waiting for the cooling time of silence. + Cooling, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Timer { + /// Fallback for a lost playback drain report. + Playback, + /// Silence after playback. + Cooling, + /// The user stopped speaking but no response followed. + UserTurn, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum FloorInput { + /// The server started a response (ours or one its turn detection made). + ResponseCreated, + /// The response finished generating; `pending_playback` is the audio of + /// it still queued for the speakers. + ResponseDone { + pending_playback: Duration, + }, + /// The user started speaking. + SpeechStarted, + /// The user stopped speaking. + SpeechStopped, + /// The speakers played everything queued. + PlaybackDrained, + /// The server reported an error. + ServerError, + /// A tool call started running. + ToolStarted, + /// A tool call finished. + ToolCompleted { + call_id: String, + output: String, + }, + Notification(Notification), + /// A conversation's wait for the user ended elsewhere. + WaitEnded { + conversation_id: String, + }, + /// The model read a conversation, so its notification is known. + ConversationRead { + conversation_id: String, + }, + TimerFired(Timer), +} + +#[derive(Debug, Clone, PartialEq)] +pub enum FloorCommand { + /// `response.cancel`. + CancelResponse, + /// Drop queued audio and truncate the current item to what was heard. + StopPlayback, + /// `conversation.item.create` with a function call output. + SendToolOutput { + call_id: String, + output: String, + }, + /// `conversation.item.create` with a background system message. + SendNotification(String), + /// `response.create`. + CreateResponse, + /// Arm the (single) timer, replacing an armed one. + StartTimer(Timer, Duration), + CancelTimer, +} + +#[derive(Debug)] +pub struct Floor { + config: FloorConfig, + state: FloorState, + /// A response is being generated (or we asked for one). + response_open: bool, + /// We sent `response.create` and wait for `response.created`. + create_pending: bool, + /// The user barged in while our `response.create` was in flight: cancel + /// that response as soon as it exists. + cancel_on_create: bool, + /// Tool outputs that completed while a response was open; attached at + /// `response.done`. + held_outputs: Vec<(String, String)>, + /// Outputs were attached after generation; respond once playback ends. + pending_trigger: bool, + tools_running: usize, + armed: Option, + queue: NotificationQueue, +} + +impl Floor { + pub fn new(config: FloorConfig) -> Self { + Self { + config, + state: FloorState::Idle, + response_open: false, + create_pending: false, + cancel_on_create: false, + held_outputs: Vec::new(), + pending_trigger: false, + tools_running: 0, + armed: None, + queue: NotificationQueue::default(), + } + } + + pub fn state(&self) -> FloorState { + self.state + } + + pub fn queued_notifications(&self) -> usize { + self.queue.len() + } + + /// The realtime session was replaced by a fresh one: everything tied to + /// the old one (open response, tool calls, held outputs) is gone. Queued + /// notifications stay and go out after a cooling period. + pub fn reconnected(&mut self) -> Vec { + let queue = std::mem::take(&mut self.queue); + *self = Self::new(self.config.clone()); + self.queue = queue; + let mut out = vec![FloorCommand::CancelTimer]; + if !self.queue.is_empty() { + self.state = FloorState::Cooling; + self.start_timer(Timer::Cooling, self.config.cooling, &mut out); + } + out + } + + pub fn handle(&mut self, input: FloorInput) -> Vec { + let mut out = Vec::new(); + match input { + FloorInput::ResponseCreated => { + self.create_pending = false; + self.response_open = true; + self.cancel_timer(&mut out); + if self.cancel_on_create { + self.cancel_on_create = false; + out.push(FloorCommand::CancelResponse); + } else { + self.state = FloorState::Speaking; + } + } + FloorInput::ResponseDone { pending_playback } => { + self.response_open = false; + self.create_pending = false; + self.cancel_on_create = false; + let attached = !self.held_outputs.is_empty(); + for (call_id, output) in self.held_outputs.drain(..) { + out.push(FloorCommand::SendToolOutput { call_id, output }); + } + // Over the user the model folds the outputs into its next + // answer instead (invariant 2). + if attached && self.state != FloorState::UserTurn { + self.pending_trigger = true; + } + if self.state == FloorState::Speaking { + if pending_playback.is_zero() { + self.playback_finished(&mut out); + } else { + self.start_timer( + Timer::Playback, + pending_playback + self.config.playback_grace, + &mut out, + ); + } + } + } + FloorInput::SpeechStarted => { + self.cancel_timer(&mut out); + if self.state == FloorState::Speaking { + if self.create_pending { + self.cancel_on_create = true; + } else if self.response_open { + out.push(FloorCommand::CancelResponse); + } + out.push(FloorCommand::StopPlayback); + } + self.pending_trigger = false; + self.state = FloorState::UserTurn; + } + FloorInput::SpeechStopped => { + if self.state == FloorState::UserTurn { + self.start_timer(Timer::UserTurn, self.config.user_turn_grace, &mut out); + } + } + FloorInput::PlaybackDrained => { + if self.waiting_for_playback() { + self.playback_finished(&mut out); + } + } + FloorInput::ServerError => { + if self.create_pending { + self.create_pending = false; + self.response_open = false; + self.cancel_on_create = false; + if self.state == FloorState::Speaking { + self.playback_finished(&mut out); + } + } + } + FloorInput::ToolStarted => self.tools_running += 1, + FloorInput::ToolCompleted { call_id, output } => { + self.tools_running = self.tools_running.saturating_sub(1); + if self.response_open { + self.held_outputs.push((call_id, output)); + } else { + out.push(FloorCommand::SendToolOutput { call_id, output }); + match self.state { + FloorState::Speaking => self.pending_trigger = true, + FloorState::UserTurn => {} + FloorState::Cooling | FloorState::Idle => { + self.cancel_timer(&mut out); + self.create_response(&mut out); + } + } + } + } + FloorInput::Notification(notification) => { + self.queue.push(notification); + match self.state { + // Notifications finishing close together go out as one. + FloorState::Cooling => { + self.start_timer(Timer::Cooling, self.config.cooling, &mut out) + } + FloorState::Idle => self.flush(&mut out), + FloorState::Speaking | FloorState::UserTurn => {} + } + } + FloorInput::WaitEnded { conversation_id } => { + self.queue.retract_waiting(&conversation_id) + } + FloorInput::ConversationRead { conversation_id } => { + self.queue.acknowledge(&conversation_id) + } + FloorInput::TimerFired(timer) => { + if self.armed != Some(timer) { + return out; + } + self.armed = None; + match timer { + Timer::Playback => { + if self.waiting_for_playback() { + self.playback_finished(&mut out); + } + } + Timer::Cooling if self.state == FloorState::Cooling => { + self.state = FloorState::Idle; + self.flush(&mut out); + } + Timer::UserTurn if self.state == FloorState::UserTurn => { + self.state = FloorState::Idle; + self.flush(&mut out); + } + Timer::Cooling | Timer::UserTurn => {} + } + } + } + out + } + + fn waiting_for_playback(&self) -> bool { + self.state == FloorState::Speaking && !self.response_open && !self.create_pending + } + + fn playback_finished(&mut self, out: &mut Vec) { + self.cancel_timer(out); + if self.pending_trigger { + self.pending_trigger = false; + self.create_response(out); + } else { + self.state = FloorState::Cooling; + self.start_timer(Timer::Cooling, self.config.cooling, out); + } + } + + /// Deliver queued notifications, only from a free floor and not while + /// a tool result is about to make the model speak anyway. + fn flush(&mut self, out: &mut Vec) { + if self.state != FloorState::Idle || self.tools_running > 0 { + return; + } + if let Some(message) = self.queue.take_message() { + out.push(FloorCommand::SendNotification(message)); + self.create_response(out); + } + } + + fn create_response(&mut self, out: &mut Vec) { + debug_assert!(!self.response_open, "invariant 1: one response at a time"); + self.create_pending = true; + self.response_open = true; + self.state = FloorState::Speaking; + out.push(FloorCommand::CreateResponse); + } + + fn start_timer(&mut self, timer: Timer, after: Duration, out: &mut Vec) { + self.armed = Some(timer); + out.push(FloorCommand::StartTimer(timer, after)); + } + + fn cancel_timer(&mut self, out: &mut Vec) { + if self.armed.take().is_some() { + out.push(FloorCommand::CancelTimer); + } + } +} + +#[cfg(test)] +mod tests { + use super::super::notifications::NotificationKind; + use super::*; + use FloorCommand as C; + use FloorInput as I; + + fn floor() -> Floor { + Floor::new(FloorConfig::default()) + } + + fn finished(id: &str) -> I { + I::Notification(Notification { + conversation_id: id.into(), + name: id.into(), + project: String::new(), + kind: NotificationKind::Finished, + }) + } + + fn done(ms: u64) -> I { + I::ResponseDone { + pending_playback: Duration::from_millis(ms), + } + } + + fn completed(call: &str) -> I { + I::ToolCompleted { + call_id: call.into(), + output: "out".into(), + } + } + + fn creates(commands: &[C]) -> bool { + commands.contains(&C::CreateResponse) + } + + fn notifies(commands: &[C]) -> bool { + commands.iter().any(|c| matches!(c, C::SendNotification(_))) + } + + /// Run a sequence and return the commands of the last input. + fn run(floor: &mut Floor, inputs: Vec) -> Vec { + let mut last = Vec::new(); + for input in inputs { + last = floor.handle(input); + } + last + } + + /// A model answer that played to the end and cooled down. + fn answered_and_cooled(floor: &mut Floor) { + run( + floor, + vec![ + I::ResponseCreated, + done(800), + I::PlaybackDrained, + I::TimerFired(Timer::Cooling), + ], + ); + assert_eq!(floor.state(), FloorState::Idle); + } + + #[test] + fn idle_floor_flushes_a_notification_at_once() { + let mut floor = floor(); + let out = floor.handle(finished("a")); + assert!(notifies(&out)); + assert!(creates(&out)); + assert_eq!(floor.state(), FloorState::Speaking); + } + + #[test] + fn notification_while_speaking_waits_for_drain_and_cooling() { + // Invariants 3 and 4. + let mut floor = floor(); + floor.handle(I::ResponseCreated); + let out = floor.handle(finished("a")); + assert!(out.is_empty(), "never interrupts the model: {out:?}"); + + let out = floor.handle(done(2000)); + assert_eq!( + out, + vec![C::StartTimer(Timer::Playback, Duration::from_millis(7000))] + ); + // Generation is done, the audio still plays. + assert_eq!(floor.state(), FloorState::Speaking); + + let out = floor.handle(I::PlaybackDrained); + assert!(!notifies(&out)); + assert_eq!(floor.state(), FloorState::Cooling); + assert!(out.contains(&C::StartTimer(Timer::Cooling, Duration::from_millis(1500)))); + + let out = floor.handle(I::TimerFired(Timer::Cooling)); + assert!(notifies(&out) && creates(&out)); + } + + #[test] + fn notification_during_user_turn_never_triggers() { + // Invariant 2. + let mut floor = floor(); + floor.handle(I::SpeechStarted); + assert!(floor.handle(finished("a")).is_empty()); + floor.handle(I::SpeechStopped); + // The server answers the user; the notification waits for that. + assert!(!creates(&floor.handle(I::ResponseCreated))); + assert!(!creates(&floor.handle(done(0)))); + let out = floor.handle(I::TimerFired(Timer::Cooling)); + assert!(notifies(&out)); + } + + #[test] + fn notifications_close_together_go_out_as_one() { + let mut floor = floor(); + run(&mut floor, vec![I::ResponseCreated, done(0)]); + assert_eq!(floor.state(), FloorState::Cooling); + let out = floor.handle(finished("a")); + assert_eq!( + out, + vec![C::StartTimer(Timer::Cooling, Duration::from_millis(1500))], + "restarts the silence timer" + ); + floor.handle(finished("b")); + let out = floor.handle(I::TimerFired(Timer::Cooling)); + let messages: Vec<_> = out + .iter() + .filter_map(|c| match c { + C::SendNotification(m) => Some(m), + _ => None, + }) + .collect(); + assert_eq!(messages.len(), 1); + assert!(messages[0].contains("id a") && messages[0].contains("id b")); + assert_eq!(out.iter().filter(|c| **c == C::CreateResponse).count(), 1); + } + + #[test] + fn barge_in_cancels_the_response_and_stops_playback() { + // Invariant 5. + let mut floor = floor(); + floor.handle(I::ResponseCreated); + let out = floor.handle(I::SpeechStarted); + assert!(out.contains(&C::CancelResponse)); + assert!(out.contains(&C::StopPlayback)); + assert_eq!(floor.state(), FloorState::UserTurn); + } + + #[test] + fn barge_in_after_generation_only_stops_playback() { + let mut floor = floor(); + run(&mut floor, vec![I::ResponseCreated, done(3000)]); + let out = floor.handle(I::SpeechStarted); + assert!(!out.contains(&C::CancelResponse), "nothing to cancel"); + assert!(out.contains(&C::StopPlayback)); + // The playback fallback timer must not fire into the user's turn. + assert!(out.contains(&C::CancelTimer)); + assert!(floor.handle(I::TimerFired(Timer::Playback)).is_empty()); + } + + #[test] + fn barge_in_before_our_response_exists_cancels_it_on_creation() { + let mut floor = floor(); + floor.handle(finished("a")); // flushes, response.create in flight + let out = floor.handle(I::SpeechStarted); + assert!(!out.contains(&C::CancelResponse)); + let out = floor.handle(I::ResponseCreated); + assert_eq!(out, vec![C::CancelResponse]); + assert_eq!(floor.state(), FloorState::UserTurn); + } + + #[test] + fn tool_call_round_trip_triggers_a_follow_up_response() { + let mut floor = floor(); + // The model's response is just a function call; the tool runs after + // it is done. + run( + &mut floor, + vec![I::ResponseCreated, I::ToolStarted, done(0)], + ); + assert_eq!(floor.state(), FloorState::Cooling); + let out = floor.handle(completed("c1")); + assert_eq!( + out, + vec![ + C::SendToolOutput { + call_id: "c1".into(), + output: "out".into() + }, + C::CancelTimer, + C::CreateResponse + ] + ); + } + + #[test] + fn tool_output_during_generation_is_held_until_done() { + // Invariant 1. + let mut floor = floor(); + run(&mut floor, vec![I::ResponseCreated, I::ToolStarted]); + assert!(floor.handle(completed("c1")).is_empty()); + let out = floor.handle(done(0)); + assert_eq!( + out, + vec![ + C::SendToolOutput { + call_id: "c1".into(), + output: "out".into() + }, + C::CreateResponse + ] + ); + } + + #[test] + fn tool_output_while_audio_plays_responds_after_drain() { + let mut floor = floor(); + run( + &mut floor, + vec![I::ResponseCreated, I::ToolStarted, done(1200)], + ); + let out = floor.handle(completed("c1")); + assert!(!creates(&out), "the speakers still play: {out:?}"); + let out = floor.handle(I::PlaybackDrained); + assert!(creates(&out)); + } + + #[test] + fn tool_output_over_the_user_attaches_without_trigger() { + // Invariant 2. + let mut floor = floor(); + run( + &mut floor, + vec![ + I::ResponseCreated, + I::ToolStarted, + done(0), + I::SpeechStarted, + ], + ); + let out = floor.handle(completed("c1")); + assert_eq!( + out, + vec![C::SendToolOutput { + call_id: "c1".into(), + output: "out".into() + }] + ); + } + + #[test] + fn barge_in_drops_a_pending_trigger() { + let mut floor = floor(); + run( + &mut floor, + vec![ + I::ResponseCreated, + I::ToolStarted, + done(1200), + completed("c1"), + ], + ); + floor.handle(I::SpeechStarted); + let out = run(&mut floor, vec![I::SpeechStopped, I::PlaybackDrained]); + assert!(!creates(&out)); + } + + #[test] + fn running_tools_hold_notifications_back() { + let mut floor = floor(); + run( + &mut floor, + vec![I::ResponseCreated, I::ToolStarted, done(0)], + ); + floor.handle(finished("a")); + let out = floor.handle(I::TimerFired(Timer::Cooling)); + assert!(!notifies(&out), "the tool result speaks first"); + let out = floor.handle(completed("c1")); + assert!(!notifies(&out) && creates(&out)); + let out = run( + &mut floor, + vec![I::ResponseCreated, done(0), I::TimerFired(Timer::Cooling)], + ); + assert!(notifies(&out)); + } + + #[test] + fn lost_drain_report_falls_back_to_the_playback_timer() { + let mut floor = floor(); + run(&mut floor, vec![I::ResponseCreated, done(500)]); + floor.handle(I::TimerFired(Timer::Playback)); + assert_eq!(floor.state(), FloorState::Cooling); + } + + #[test] + fn drain_during_generation_is_an_underrun_not_the_end() { + let mut floor = floor(); + floor.handle(I::ResponseCreated); + assert!(floor.handle(I::PlaybackDrained).is_empty()); + assert_eq!(floor.state(), FloorState::Speaking); + } + + #[test] + fn silent_user_turn_returns_the_floor() { + let mut floor = floor(); + run(&mut floor, vec![I::SpeechStarted, finished("a")]); + let out = floor.handle(I::SpeechStopped); + assert_eq!( + out, + vec![C::StartTimer(Timer::UserTurn, Duration::from_secs(8))] + ); + let out = floor.handle(I::TimerFired(Timer::UserTurn)); + assert!(notifies(&out)); + } + + #[test] + fn failed_create_does_not_wedge_the_floor() { + let mut floor = floor(); + floor.handle(finished("a")); + floor.handle(I::ServerError); + assert_eq!(floor.state(), FloorState::Cooling); + floor.handle(I::TimerFired(Timer::Cooling)); + assert_eq!(floor.state(), FloorState::Idle); + assert!(creates(&floor.handle(finished("b")))); + } + + #[test] + fn reading_a_conversation_drops_its_notification() { + let mut floor = floor(); + floor.handle(I::ResponseCreated); + floor.handle(finished("a")); + floor.handle(I::ConversationRead { + conversation_id: "a".into(), + }); + let out = run(&mut floor, vec![done(0), I::TimerFired(Timer::Cooling)]); + assert!(!notifies(&out)); + assert_eq!(floor.queued_notifications(), 0); + } + + #[test] + fn stale_timer_fires_are_ignored() { + let mut floor = floor(); + answered_and_cooled(&mut floor); + assert!(floor.handle(I::TimerFired(Timer::Cooling)).is_empty()); + assert!(floor.handle(I::TimerFired(Timer::Playback)).is_empty()); + } +} diff --git a/crates/code_assistant_core/src/voice/mod.rs b/crates/code_assistant_core/src/voice/mod.rs new file mode 100644 index 00000000..edd785fa --- /dev/null +++ b/crates/code_assistant_core/src/voice/mod.rs @@ -0,0 +1,106 @@ +//! Voice mode: a global, speech-to-speech assistant over all conversations. +//! +//! The voice agent is a small agent loop of its own next to the sessions. It +//! talks to a realtime model (`llm::realtime`), streams microphone and +//! speaker audio through a [`VoiceAudio`] implementation the application +//! injects, and only has tools that act on conversations ([`tools`]). When a +//! conversation's agent finishes, the voice agent is notified — but never +//! while the user or the model speaks: [`floor::Floor`] decides when the +//! floor is free. See `docs/voice-mode.md`. +//! +//! Frontends drive it through [`VoiceService`] and follow it through the +//! app-scoped [`crate::ui::UiEvent::VoiceStatusChanged`] and +//! [`crate::ui::UiEvent::VoiceTranscript`] events. + +mod agent; +pub mod config; +pub mod floor; +pub mod notifications; +mod service; +pub mod source; +pub mod tools; + +#[cfg(test)] +mod tests; + +pub use config::{NotifyScope, VoiceConfig}; +pub use service::{ConnectorFactory, VoiceService, default_connector_factory}; + +use anyhow::Result; +use std::sync::Arc; +use tokio::sync::mpsc; + +/// What the audio devices report. +#[derive(Debug, Clone, PartialEq)] +pub enum AudioEvent { + /// Microphone samples: PCM16, 24 kHz, mono. + Captured(Vec), + /// The speakers played everything queued. + Drained, + /// The devices stopped working. + Failed(String), +} + +/// Microphone and speakers of a voice session. Implementations run the +/// devices on threads of their own and report through the +/// [`AudioEvent`] sender they were opened with. +pub trait VoiceAudio: Send { + /// Queue samples (PCM16, 24 kHz, mono) for playback. + fn play(&mut self, samples: Vec); + /// Drop everything queued for playback. + fn clear_playback(&mut self); + /// Samples the speakers played since the devices opened. + fn played_samples(&self) -> u64; + /// Stop (or resume) reporting microphone samples. + fn set_capture_muted(&mut self, muted: bool); +} + +/// Opens the audio devices for a voice session. +pub type AudioFactory = + Arc) -> Result> + Send + Sync>; + +/// What the voice agent is doing, for the frontends. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum VoiceActivity { + #[default] + Off, + Connecting, + /// Connected; the floor is free. + Listening, + /// The user has the floor. + UserSpeaking, + /// The model speaks (or prepares its answer). + Speaking, + /// Voice mode stopped because of this error. + Failed(String), +} + +impl VoiceActivity { + pub fn is_active(&self) -> bool { + !matches!(self, Self::Off | Self::Failed(_)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct VoiceStatus { + pub activity: VoiceActivity, + pub muted: bool, + /// Notifications waiting for a free floor. + pub queued_notifications: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TranscriptRole { + User, + Assistant, + /// A tool call the model made. + Tool, + /// A background notification delivered to the model. + Notification, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TranscriptEntry { + pub role: TranscriptRole, + pub text: String, +} diff --git a/crates/code_assistant_core/src/voice/notifications.rs b/crates/code_assistant_core/src/voice/notifications.rs new file mode 100644 index 00000000..959a66ce --- /dev/null +++ b/crates/code_assistant_core/src/voice/notifications.rs @@ -0,0 +1,164 @@ +//! Background notifications about conversations, waiting for a free floor. +//! +//! The queue holds at most one entry per conversation: a newer event +//! replaces an older one, so a conversation that finishes twice before the +//! floor frees up is reported once. Reading a conversation +//! (`get_conversation`) acknowledges its entry — the model knows already. + +/// What happened in a conversation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NotificationKind { + /// The agent finished its turn. + Finished, + /// The agent stopped with an error. + Failed { message: String }, + /// The agent waits for a permission decision. + NeedsPermission { tool: String }, + /// The agent asked the user questions. + NeedsAnswer, +} + +impl NotificationKind { + /// Whether the conversation waits for the user (as opposed to having + /// ended its turn). + pub fn is_waiting(&self) -> bool { + matches!(self, Self::NeedsPermission { .. } | Self::NeedsAnswer) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Notification { + pub conversation_id: String, + pub name: String, + pub project: String, + pub kind: NotificationKind, +} + +#[derive(Debug, Default)] +pub struct NotificationQueue { + /// Arrival order, one entry per conversation. + entries: Vec, +} + +impl NotificationQueue { + pub fn push(&mut self, notification: Notification) { + self.entries + .retain(|entry| entry.conversation_id != notification.conversation_id); + self.entries.push(notification); + } + + /// Drop the entry of a conversation the model has just read. + pub fn acknowledge(&mut self, conversation_id: &str) { + self.entries + .retain(|entry| entry.conversation_id != conversation_id); + } + + /// Drop a conversation's "waiting for you" entry once the wait is over + /// (answered elsewhere). A finish or failure entry stays. + pub fn retract_waiting(&mut self, conversation_id: &str) { + self.entries + .retain(|entry| entry.conversation_id != conversation_id || !entry.kind.is_waiting()); + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Empty the queue into one message for the model, or `None` when + /// nothing is queued. + pub fn take_message(&mut self) -> Option { + if self.entries.is_empty() { + return None; + } + let entries = std::mem::take(&mut self.entries); + Some(render(&entries)) + } +} + +fn render(entries: &[Notification]) -> String { + let lines: Vec = entries + .iter() + .map(|entry| { + let what = match &entry.kind { + NotificationKind::Finished => "finished its turn".to_string(), + NotificationKind::Failed { message } => { + format!("stopped with an error ({message})") + } + NotificationKind::NeedsPermission { tool } => { + format!("is waiting for permission to run `{tool}`") + } + NotificationKind::NeedsAnswer => "is waiting for answers to its questions".into(), + }; + let project = if entry.project.is_empty() { + String::new() + } else { + format!(" in project {}", entry.project) + }; + format!( + "- Conversation \"{}\"{project} (id {}) {what}.", + entry.name, entry.conversation_id + ) + }) + .collect(); + format!( + "[background notification]\n{}\n\nThe user is not speaking right now. If it is useful \ + to them, mention this briefly. Call get_conversation before you summarise a result. \ + Otherwise say nothing.", + lines.join("\n") + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn note(id: &str, kind: NotificationKind) -> Notification { + Notification { + conversation_id: id.into(), + name: format!("name {id}"), + project: "proj".into(), + kind, + } + } + + #[test] + fn newer_event_of_a_conversation_replaces_the_older_one() { + let mut queue = NotificationQueue::default(); + queue.push(note("a", NotificationKind::NeedsAnswer)); + queue.push(note("b", NotificationKind::Finished)); + queue.push(note("a", NotificationKind::Finished)); + assert_eq!(queue.len(), 2); + let message = queue.take_message().unwrap(); + assert!(message.find("id b").unwrap() < message.find("id a").unwrap()); + assert!(!message.contains("questions")); + assert!(queue.is_empty()); + } + + #[test] + fn reading_a_conversation_acknowledges_it() { + let mut queue = NotificationQueue::default(); + queue.push(note("a", NotificationKind::Finished)); + queue.acknowledge("a"); + assert_eq!(queue.take_message(), None); + } + + #[test] + fn retracting_a_wait_keeps_a_finish() { + let mut queue = NotificationQueue::default(); + queue.push(note( + "a", + NotificationKind::NeedsPermission { + tool: "execute_command".into(), + }, + )); + queue.push(note("b", NotificationKind::Finished)); + queue.retract_waiting("a"); + queue.retract_waiting("b"); + assert_eq!(queue.len(), 1); + assert!(queue.take_message().unwrap().contains("id b")); + } +} diff --git a/crates/code_assistant_core/src/voice/service.rs b/crates/code_assistant_core/src/voice/service.rs new file mode 100644 index 00000000..a6797aa6 --- /dev/null +++ b/crates/code_assistant_core/src/voice/service.rs @@ -0,0 +1,174 @@ +//! [`VoiceService`]: the frontends' handle to start, stop and mute voice +//! mode. + +use super::agent::{Control, VoiceAgent}; +use super::{AudioFactory, VoiceActivity, VoiceConfig, VoiceStatus}; +use crate::session::SessionService; +use crate::session::event_stream::EventStream; +use crate::ui::UiEvent; +use anyhow::Result; +use llm::realtime::RealtimeConnector; +use std::future::Future; +use std::sync::Arc; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; +use tracing::{error, info}; + +/// Builds the connector for a voice session from its configuration. +pub type ConnectorFactory = + Arc Result> + Send + Sync>; + +/// The connector for the configured provider (OpenAI or AI Core). +pub fn default_connector_factory() -> ConnectorFactory { + Arc::new(|config: &VoiceConfig| config.connector()) +} + +#[derive(Debug)] +enum Command { + Start, + Stop, + SetMuted(bool), +} + +/// Cloneable handle; the voice agent runs in the worker future returned by +/// [`VoiceService::new`], which the application spawns on its backend +/// runtime. Status changes arrive as app-scoped +/// [`UiEvent::VoiceStatusChanged`] events. +#[derive(Clone)] +pub struct VoiceService { + tx: mpsc::UnboundedSender, +} + +impl VoiceService { + pub fn new( + sessions: SessionService, + events: EventStream, + audio: AudioFactory, + connectors: ConnectorFactory, + ) -> (Self, impl Future) { + Self::with_config_loader( + sessions, + events, + audio, + connectors, + Arc::new(VoiceConfig::load), + ) + } + + /// Like [`Self::new`], reading the configuration through `load_config` + /// on every start (tests pass a fixed one). + pub fn with_config_loader( + sessions: SessionService, + events: EventStream, + audio: AudioFactory, + connectors: ConnectorFactory, + load_config: Arc VoiceConfig + Send + Sync>, + ) -> (Self, impl Future) { + let (tx, mut rx) = mpsc::unbounded_channel::(); + let worker = async move { + let mut running: Option = None; + while let Some(command) = rx.recv().await { + if running.as_ref().is_some_and(|r| r.task.is_finished()) { + running = None; + } + match command { + Command::Start => { + if running.is_some() { + continue; + } + running = start(load_config(), &sessions, &events, &audio, &connectors); + } + Command::Stop => { + if let Some(r) = running.take() { + let _ = r.control.send(Control::Stop); + let _ = r.task.await; + } + publish(&events, VoiceStatus::default()); + } + Command::SetMuted(muted) => { + if let Some(r) = &running { + let _ = r.control.send(Control::SetMuted(muted)); + } + } + } + } + if let Some(r) = running { + let _ = r.control.send(Control::Stop); + let _ = r.task.await; + } + }; + (Self { tx }, worker) + } + + pub fn start(&self) { + let _ = self.tx.send(Command::Start); + } + + pub fn stop(&self) { + let _ = self.tx.send(Command::Stop); + } + + pub fn set_muted(&self, muted: bool) { + let _ = self.tx.send(Command::SetMuted(muted)); + } +} + +struct Running { + control: mpsc::UnboundedSender, + task: JoinHandle<()>, +} + +fn start( + config: VoiceConfig, + sessions: &SessionService, + events: &EventStream, + audio: &AudioFactory, + connectors: &ConnectorFactory, +) -> Option { + publish( + events, + VoiceStatus { + activity: VoiceActivity::Connecting, + ..VoiceStatus::default() + }, + ); + let connector = match connectors(&config) { + Ok(connector) => connector, + Err(e) => { + fail(events, e); + return None; + } + }; + let (control, control_rx) = mpsc::unbounded_channel(); + let sessions = sessions.clone(); + let events = events.clone(); + let audio = audio.clone(); + let task = tokio::spawn(async move { + let agent = + match VoiceAgent::start(config, connector, sessions, events.clone(), audio).await { + Ok(agent) => agent, + Err(e) => return fail(&events, e), + }; + info!("Voice mode started"); + match agent.run(control_rx).await { + Ok(()) => info!("Voice mode stopped"), + Err(e) => fail(&events, e), + } + }); + Some(Running { control, task }) +} + +fn fail(events: &EventStream, error: anyhow::Error) { + error!("Voice mode failed: {error:#}"); + publish( + events, + VoiceStatus { + activity: VoiceActivity::Failed(format!("{error:#}")), + ..VoiceStatus::default() + }, + ); +} + +fn publish(events: &EventStream, status: VoiceStatus) { + events.publish_app(UiEvent::VoiceStatusChanged { status }); +} diff --git a/crates/code_assistant_core/src/voice/source.rs b/crates/code_assistant_core/src/voice/source.rs new file mode 100644 index 00000000..7e45b2fb --- /dev/null +++ b/crates/code_assistant_core/src/voice/source.rs @@ -0,0 +1,282 @@ +//! Turns the core's event stream into notifications for the voice agent. + +use super::config::NotifyScope; +use super::floor::FloorInput; +use super::notifications::{Notification, NotificationKind}; +use crate::session::SessionService; +use crate::session::event_stream::{EventPayload, SessionEvent}; +use crate::session::instance::SessionActivityState; +use crate::ui::UiEvent; +use anyhow::Result; +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, Mutex}; + +/// Permission and question requests open right now, shared with the tools +/// so `list_conversations` can tell which conversation waits for the user. +#[derive(Clone, Default)] +pub struct OpenRequests { + /// request id → (conversation id, what it waits for) + inner: Arc>>, +} + +impl OpenRequests { + fn open(&self, request_id: String, conversation_id: String, kind: NotificationKind) { + self.inner + .lock() + .unwrap() + .insert(request_id, (conversation_id, kind)); + } + + /// Close a request; returns its conversation when that has no other + /// open request left. + fn close(&self, request_id: &str) -> Option { + let mut inner = self.inner.lock().unwrap(); + let (conversation_id, _) = inner.remove(request_id)?; + let still_waiting = inner.values().any(|(id, _)| *id == conversation_id); + (!still_waiting).then_some(conversation_id) + } + + fn close_conversation(&self, conversation_id: &str) { + self.inner + .lock() + .unwrap() + .retain(|_, (id, _)| id != conversation_id); + } + + /// A state label when the conversation waits for the user. + pub fn waiting_for(&self, conversation_id: &str) -> Option { + let inner = self.inner.lock().unwrap(); + inner + .values() + .find(|(id, _)| id == conversation_id) + .map(|(_, kind)| match kind { + NotificationKind::NeedsPermission { tool } => { + format!("waiting for permission to run {tool}") + } + _ => "waiting for answers to its questions".to_string(), + }) + } +} + +/// What happened, before names are looked up. +#[derive(Debug, Clone, PartialEq)] +enum Change { + Notify(String, NotificationKind), + WaitEnded(String), +} + +pub struct NotificationSource { + service: SessionService, + scope: NotifyScope, + touched: HashSet, + states: HashMap, + open_requests: OpenRequests, +} + +impl NotificationSource { + /// Start from the current activity states, so a run that was going on + /// before voice mode started is reported when it ends. + pub async fn new( + service: SessionService, + scope: NotifyScope, + open_requests: OpenRequests, + ) -> Result { + let states = service.session_activity_states().await?; + Ok(Self { + service, + scope, + touched: HashSet::new(), + states, + open_requests, + }) + } + + /// The voice agent created or messaged this conversation. + pub fn touch(&mut self, conversation_id: &str) { + self.touched.insert(conversation_id.to_string()); + } + + pub async fn on_event(&mut self, event: &SessionEvent) -> Vec { + let Some(change) = self.change_for(event) else { + return Vec::new(); + }; + self.resolve(vec![change]).await + } + + /// After the stream lagged: compare the activity states with the last + /// known ones. + pub async fn resync(&mut self) -> Vec { + let Ok(current) = self.service.session_activity_states().await else { + return Vec::new(); + }; + let mut changes = Vec::new(); + for (id, previous) in &self.states { + let now = current.get(id).cloned().unwrap_or_default(); + if let Some(kind) = ended(previous, &now) { + changes.push(Change::Notify(id.clone(), kind)); + } + } + self.states = current; + let changes = changes + .into_iter() + .filter(|change| self.in_scope(change)) + .collect(); + self.resolve(changes).await + } + + fn change_for(&mut self, event: &SessionEvent) -> Option { + let EventPayload::Ui(ui) = &event.payload else { + return None; + }; + let session_id = event.session_id.clone(); + let change = match ui { + UiEvent::UpdateSessionActivityState { + session_id, + activity_state, + } => { + let previous = self + .states + .insert(session_id.clone(), activity_state.clone()); + if activity_state.is_terminal() { + self.open_requests.close_conversation(session_id); + } + let kind = ended(&previous?, activity_state)?; + Change::Notify(session_id.clone(), kind) + } + UiEvent::RequestToolPermission { request } => { + let session_id = session_id?; + let kind = NotificationKind::NeedsPermission { + tool: request.tool_name.clone(), + }; + self.open_requests.open( + request.request_id.clone(), + session_id.clone(), + kind.clone(), + ); + Change::Notify(session_id, kind) + } + UiEvent::RequestUserQuestions { request } => { + let session_id = session_id?; + let kind = NotificationKind::NeedsAnswer; + self.open_requests.open( + request.request_id.clone(), + session_id.clone(), + kind.clone(), + ); + Change::Notify(session_id, kind) + } + UiEvent::ToolPermissionRequestResolved { request_id } + | UiEvent::UserQuestionsResolved { request_id } => { + Change::WaitEnded(self.open_requests.close(request_id)?) + } + _ => return None, + }; + self.in_scope(&change).then_some(change) + } + + fn in_scope(&self, change: &Change) -> bool { + match (self.scope, change) { + (NotifyScope::All, _) | (_, Change::WaitEnded(_)) => true, + (NotifyScope::Touched, Change::Notify(id, _)) => self.touched.contains(id), + } + } + + async fn resolve(&self, changes: Vec) -> Vec { + if changes.is_empty() { + return Vec::new(); + } + let sessions = self.service.list_sessions().await.unwrap_or_default(); + changes + .into_iter() + .map(|change| match change { + Change::WaitEnded(conversation_id) => FloorInput::WaitEnded { conversation_id }, + Change::Notify(conversation_id, kind) => { + let metadata = super::tools::find_metadata(&sessions, &conversation_id); + FloorInput::Notification(Notification { + name: metadata + .map(|m| m.name.clone()) + .unwrap_or_else(|| conversation_id.clone()), + project: metadata + .map(|m| m.initial_project.clone()) + .unwrap_or_default(), + conversation_id, + kind, + }) + } + }) + .collect() + } +} + +/// The notification for a run ending, when `previous → now` is one. +fn ended(previous: &SessionActivityState, now: &SessionActivityState) -> Option { + let was_running = matches!( + previous, + SessionActivityState::AgentRunning + | SessionActivityState::WaitingForResponse + | SessionActivityState::RateLimited { .. } + ); + if !was_running { + return None; + } + match now { + SessionActivityState::Idle => Some(NotificationKind::Finished), + SessionActivityState::Errored { message } => Some(NotificationKind::Failed { + message: message.clone(), + }), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_the_end_of_a_run_notifies() { + use SessionActivityState as S; + assert_eq!( + ended(&S::AgentRunning, &S::Idle), + Some(NotificationKind::Finished) + ); + assert_eq!( + ended( + &S::WaitingForResponse, + &S::Errored { + message: "x".into() + } + ), + Some(NotificationKind::Failed { + message: "x".into() + }) + ); + assert_eq!(ended(&S::AgentRunning, &S::WaitingForResponse), None); + // Clearing an error is not a run ending. + assert_eq!( + ended( + &S::Errored { + message: "x".into() + }, + &S::Idle + ), + None + ); + assert_eq!(ended(&S::Idle, &S::Idle), None); + } + + #[test] + fn a_conversation_waits_until_its_last_request_closes() { + let requests = OpenRequests::default(); + requests.open("r1".into(), "s".into(), NotificationKind::NeedsAnswer); + requests.open( + "r2".into(), + "s".into(), + NotificationKind::NeedsPermission { tool: "t".into() }, + ); + assert!(requests.waiting_for("s").is_some()); + assert_eq!(requests.close("r1"), None); + assert_eq!(requests.close("r2"), Some("s".into())); + assert_eq!(requests.waiting_for("s"), None); + assert_eq!(requests.close("r2"), None); + } +} diff --git a/crates/code_assistant_core/src/voice/tests.rs b/crates/code_assistant_core/src/voice/tests.rs new file mode 100644 index 00000000..8dc659b5 --- /dev/null +++ b/crates/code_assistant_core/src/voice/tests.rs @@ -0,0 +1,436 @@ +//! End-to-end tests of the voice agent: a scripted realtime server, fake +//! audio devices, and a real session service whose agents answer from a +//! mock LLM. + +use super::*; +use crate::mocks::{MockLLMProvider, create_test_response_text}; +use crate::persistence::FileSessionPersistence; +use crate::session::event_stream::EventStream; +use crate::session::service::{AgentRuntimeOptions, default_project_manager_factory}; +use crate::session::{SessionConfig, SessionManager, SessionService}; +use crate::ui::UiEvent; +use llm::realtime::{ + ClientEvent, ConversationItem, Incoming, RealtimeConnection, RealtimeConnector, ResponseInfo, + ServerEvent, encode_pcm16, +}; +use std::sync::Mutex; +use std::time::Duration; +use tokio::sync::mpsc; + +const WAIT: Duration = Duration::from_secs(5); + +/// The server side of one scripted realtime connection. +struct Server { + from_client: mpsc::UnboundedReceiver, + to_client: mpsc::UnboundedSender, +} + +impl Server { + fn send(&self, event: ServerEvent) { + self.to_client.send(Incoming::Event(event)).unwrap(); + } + + /// Read client events until one matches; audio appends are skipped. + async fn expect(&mut self, what: &str, pred: impl Fn(&ClientEvent) -> bool) -> ClientEvent { + tokio::time::timeout(WAIT, async { + loop { + let event = self.from_client.recv().await.expect("client hung up"); + if pred(&event) { + return event; + } + } + }) + .await + .unwrap_or_else(|_| panic!("timed out waiting for {what}")) + } + + /// Assert that nothing but audio arrives for `duration`. + async fn expect_quiet(&mut self, duration: Duration) { + let deadline = tokio::time::Instant::now() + duration; + while let Ok(Some(event)) = tokio::time::timeout_at(deadline, self.from_client.recv()).await + { + assert!( + matches!(event, ClientEvent::AppendAudio(_)), + "expected no client events, got {event:?}" + ); + } + } +} + +struct ScriptedConnector { + connections: Mutex>, +} + +#[async_trait::async_trait] +impl RealtimeConnector for ScriptedConnector { + async fn connect(&self) -> anyhow::Result { + self.connections + .lock() + .unwrap() + .pop() + .ok_or_else(|| anyhow::anyhow!("no more scripted connections")) + } +} + +fn scripted_connection() -> (RealtimeConnection, Server) { + let (outgoing, from_client) = mpsc::unbounded_channel(); + let (to_client, incoming) = mpsc::unbounded_channel(); + ( + RealtimeConnection { outgoing, incoming }, + Server { + from_client, + to_client, + }, + ) +} + +#[derive(Default)] +struct AudioState { + queued: u64, + played: u64, + clears: usize, + events: Option>, +} + +#[derive(Clone, Default)] +struct FakeAudio(Arc>); + +impl FakeAudio { + fn factory(&self) -> AudioFactory { + let audio = self.clone(); + Arc::new(move |events| { + audio.0.lock().unwrap().events = Some(events); + Ok(Box::new(audio.clone()) as Box) + }) + } + + /// The speakers played this many samples, and drained if that was all. + fn play(&self, samples: u64) { + let mut state = self.0.lock().unwrap(); + state.played = (state.played + samples).min(state.queued); + if state.played == state.queued { + let _ = state.events.as_ref().unwrap().send(AudioEvent::Drained); + } + } +} + +impl VoiceAudio for FakeAudio { + fn play(&mut self, samples: Vec) { + self.0.lock().unwrap().queued += samples.len() as u64; + } + fn clear_playback(&mut self) { + let mut state = self.0.lock().unwrap(); + state.clears += 1; + state.queued = state.played; + } + fn played_samples(&self) -> u64 { + self.0.lock().unwrap().played + } + fn set_capture_muted(&mut self, _muted: bool) {} +} + +struct Harness { + sessions: SessionService, + voice: VoiceService, + server: Server, + audio: FakeAudio, + _dir: tempfile::TempDir, +} + +/// A running voice session; every agent run in the session service answers +/// "done". +async fn harness() -> Harness { + let dir = tempfile::tempdir().unwrap(); + let events = EventStream::new(); + let manager = Arc::new(tokio::sync::Mutex::new(SessionManager::new( + FileSessionPersistence::new_with_root_dir(dir.path().to_path_buf()), + SessionConfig::default(), + "test-model".to_string(), + crate::tools::test_registry(), + events.clone(), + ))); + let runtime = Arc::new(AgentRuntimeOptions { + record_path: None, + playback_path: None, + fast_playback: false, + command_executor_factory: Arc::new(|_| { + Box::new(crate::mocks::create_command_executor_mock()) + }), + project_manager_factory: default_project_manager_factory(), + llm_client_factory: Some(Arc::new(|_| { + Ok(Box::new( + MockLLMProvider::new(vec![Ok(create_test_response_text("done"))]).streaming(), + )) + })), + }); + let (sessions, worker) = SessionService::new(manager, runtime, events.clone()); + tokio::spawn(worker); + + let (connection, mut server) = scripted_connection(); + let connector: Arc = Arc::new(ScriptedConnector { + connections: Mutex::new(vec![connection]), + }); + let audio = FakeAudio::default(); + let (voice, voice_worker) = VoiceService::with_config_loader( + sessions.clone(), + events, + audio.factory(), + Arc::new(move |_| Ok(connector.clone())), + Arc::new(|| VoiceConfig { + provider: "test".into(), + cooling_ms: 50, + ..VoiceConfig::default() + }), + ); + tokio::spawn(voice_worker); + voice.start(); + server + .expect("session.update", |e| { + matches!(e, ClientEvent::SessionUpdate(_)) + }) + .await; + server.send(ServerEvent::SessionCreated); + Harness { + sessions, + voice, + server, + audio, + _dir: dir, + } +} + +fn response_created() -> ServerEvent { + ServerEvent::ResponseCreated { + response: ResponseInfo { + id: "r".into(), + status: None, + }, + } +} + +fn response_done() -> ServerEvent { + ServerEvent::ResponseDone { + response: ResponseInfo { + id: "r".into(), + status: Some("completed".into()), + }, + } +} + +fn audio_delta(item: &str, samples: usize) -> ServerEvent { + ServerEvent::AudioDelta { + item_id: item.into(), + delta: encode_pcm16(&vec![0; samples]), + } +} + +fn function_call(name: &str, call_id: &str, arguments: serde_json::Value) -> ServerEvent { + ServerEvent::OutputItemDone { + item: ConversationItem { + id: format!("item-{call_id}"), + kind: "function_call".into(), + name: Some(name.into()), + call_id: Some(call_id.into()), + arguments: Some(arguments.to_string()), + }, + } +} + +async fn wait_idle(sessions: &SessionService, id: &str) { + tokio::time::timeout(WAIT, async { + while sessions.is_session_busy(id.to_string()).await.unwrap() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("session finished"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_finish_while_the_model_speaks_waits_for_playback_and_silence() { + let mut h = harness().await; + let id = h + .sessions + .create_session(Some("Fix flaky test".into()), None) + .await + .unwrap(); + + // The model is talking: one second of audio is queued. + h.server.send(response_created()); + h.server.send(audio_delta("speech", 24_000)); + h.sessions + .send_user_message(id.clone(), "go".into(), Vec::new(), None) + .await + .unwrap(); + wait_idle(&h.sessions, &id).await; + + // Generation ends, the speakers still play. + h.server.send(response_done()); + h.server.expect_quiet(Duration::from_millis(200)).await; + + h.audio.play(24_000); + let note = h + .server + .expect("notification", |e| { + matches!(e, ClientEvent::SystemMessage(_)) + }) + .await; + let ClientEvent::SystemMessage(text) = note else { + unreachable!() + }; + assert!(text.contains("Fix flaky test"), "{text}"); + assert!(text.contains("finished"), "{text}"); + h.server + .expect("response.create", |e| *e == ClientEvent::CreateResponse) + .await; + h.voice.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn message_conversation_starts_a_turn_and_its_end_is_announced() { + let mut h = harness().await; + // A session in "proj" makes the project known. + h.sessions + .create_session(None, Some("proj".into())) + .await + .unwrap(); + + h.server.send(response_created()); + h.server.send(function_call( + "message_conversation", + "c1", + serde_json::json!({"project": "proj", "title": "Docs", "message": "update the docs"}), + )); + h.server.send(response_done()); + + let output = h + .server + .expect("tool output", |e| { + matches!(e, ClientEvent::FunctionCallOutput { .. }) + }) + .await; + let ClientEvent::FunctionCallOutput { call_id, output } = output else { + unreachable!() + }; + assert_eq!(call_id, "c1"); + let output: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert_eq!(output["status"], "started"); + let new_id = output["conversation_id"].as_str().unwrap().to_string(); + // The output arrived on a free floor: the model answers at once. + h.server + .expect("response.create", |e| *e == ClientEvent::CreateResponse) + .await; + h.server.send(response_created()); + h.server.send(response_done()); + + wait_idle(&h.sessions, &new_id).await; + let note = h + .server + .expect("notification", |e| { + matches!(e, ClientEvent::SystemMessage(_)) + }) + .await; + let ClientEvent::SystemMessage(text) = note else { + unreachable!() + }; + assert!(text.contains(&new_id), "{text}"); + h.voice.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn reading_a_finished_conversation_returns_its_last_answer_and_drops_the_notification() { + let mut h = harness().await; + let id = h.sessions.create_session(None, None).await.unwrap(); + + // The user talks while the conversation finishes. + h.server.send(ServerEvent::SpeechStarted); + h.sessions + .send_user_message(id.clone(), "go".into(), Vec::new(), None) + .await + .unwrap(); + wait_idle(&h.sessions, &id).await; + h.server.send(ServerEvent::SpeechStopped); + h.server.send(response_created()); + h.server.send(function_call( + "get_conversation", + "c1", + serde_json::json!({"conversation_id": id, "full": true}), + )); + h.server.send(response_done()); + + let output = h + .server + .expect("tool output", |e| { + matches!(e, ClientEvent::FunctionCallOutput { .. }) + }) + .await; + let ClientEvent::FunctionCallOutput { output, .. } = output else { + unreachable!() + }; + let output: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert_eq!(output["state"], "idle"); + assert_eq!(output["turns"][0]["user"], "go"); + assert_eq!(output["turns"][0]["agent"], "done"); + + h.server + .expect("response.create", |e| *e == ClientEvent::CreateResponse) + .await; + h.server.send(response_created()); + h.server.send(response_done()); + // The model knows the result; no notification follows. + h.server.expect_quiet(Duration::from_millis(300)).await; + h.voice.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn barge_in_cancels_and_truncates_to_what_was_heard() { + let mut h = harness().await; + h.server.send(response_created()); + h.server.send(audio_delta("speech", 24_000)); + // Half a second played when the user interrupts. + tokio::time::sleep(Duration::from_millis(50)).await; + h.audio.play(12_000); + h.server.send(ServerEvent::SpeechStarted); + + h.server + .expect("response.cancel", |e| *e == ClientEvent::CancelResponse) + .await; + let truncate = h + .server + .expect("truncate", |e| { + matches!(e, ClientEvent::TruncateItem { .. }) + }) + .await; + assert_eq!( + truncate, + ClientEvent::TruncateItem { + item_id: "speech".into(), + audio_end_ms: 500 + } + ); + assert_eq!(h.audio.0.lock().unwrap().clears, 1); + h.voice.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn status_events_follow_the_floor() { + let h = harness().await; + let mut events = h.sessions.subscribe(); + h.server.send(response_created()); + let status = tokio::time::timeout(WAIT, async { + loop { + if let Ok(event) = events.recv().await + && let crate::session::event_stream::EventPayload::Ui(UiEvent::VoiceStatusChanged { + status, + }) = event.payload + && status.activity == VoiceActivity::Speaking + { + return status; + } + } + }) + .await + .expect("speaking status"); + assert!(!status.muted); + h.voice.stop(); +} diff --git a/crates/code_assistant_core/src/voice/tools.rs b/crates/code_assistant_core/src/voice/tools.rs new file mode 100644 index 00000000..1fd2490b --- /dev/null +++ b/crates/code_assistant_core/src/voice/tools.rs @@ -0,0 +1,496 @@ +//! The voice agent's tools: they act on conversations (sessions) and do +//! nothing else. +//! +//! Every tool returns at once. `message_conversation` starts or queues a +//! turn and leaves it running; the agent hears about its end through a +//! notification. + +use super::source::OpenRequests; +use crate::persistence::ChatMetadata; +use crate::session::SessionService; +use crate::session::instance::SessionActivityState; +use crate::session_query::{ContentItem, ContentKind, ContentPart, ContentProjection}; +use anyhow::{Context, Result, anyhow, bail}; +use llm::realtime::ToolDefinition; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::collections::BTreeSet; +use std::time::SystemTime; + +/// Turns `get_conversation` returns with `full`. +pub const FULL_TURNS: usize = 10; +/// Characters per message `get_conversation` returns. +pub const MAX_MESSAGE_CHARS: usize = 2_000; +/// Characters of all messages of one `get_conversation` call. +pub const MAX_TOTAL_CHARS: usize = 16_000; +const DEFAULT_LIST_LIMIT: usize = 20; +const MAX_LIST_LIMIT: usize = 50; + +/// What a tool call means for the voice agent beyond its output. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ToolEffect { + /// The model read the conversation: its notification is known. + Read(String), + /// The model created or messaged the conversation. + Touched(String), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ToolOutcome { + /// JSON text for the function call output. + pub output: String, + pub effects: Vec, +} + +#[derive(Clone)] +pub struct VoiceTools { + service: SessionService, + open_requests: OpenRequests, +} + +impl VoiceTools { + pub fn new(service: SessionService, open_requests: OpenRequests) -> Self { + Self { + service, + open_requests, + } + } + + pub fn definitions() -> Vec { + vec![ + ToolDefinition { + name: "list_conversations".into(), + description: "List the user's conversations with coding agents, most recently \ + active first, with their state (running, idle, errored, or waiting for the \ + user)." + .into(), + parameters: json!({ + "type": "object", + "properties": { + "project": { "type": "string", "description": "Only conversations of this project." }, + "include_settled": { "type": "boolean", "description": "Also list settled (archived) conversations. Default false." }, + "limit": { "type": "integer", "description": "Maximum number of conversations, default 20." } + }, + }), + }, + ToolDefinition { + name: "list_projects".into(), + description: "List the projects a new conversation can be started in.".into(), + parameters: json!({ "type": "object", "properties": {} }), + }, + ToolDefinition { + name: "message_conversation".into(), + description: "Send a message to a coding agent conversation. Without \ + conversation_id a new conversation is started in `project`. If the agent is \ + busy the message is queued for its next turn. Returns immediately; you get a \ + notification when the agent finishes." + .into(), + parameters: json!({ + "type": "object", + "properties": { + "conversation_id": { "type": "string", "description": "The conversation to message; empty to start a new one." }, + "project": { "type": "string", "description": "Project of a new conversation (see list_projects)." }, + "title": { "type": "string", "description": "Short title of a new conversation." }, + "message": { "type": "string", "description": "The message, written as the user's instruction to the agent." } + }, + "required": ["message"], + }), + }, + ToolDefinition { + name: "get_conversation".into(), + description: "Read a conversation. By default returns its state and the agent's \ + last message. With full=true returns the last 10 turns, each with the user's \ + message and the agent's final message of that turn." + .into(), + parameters: json!({ + "type": "object", + "properties": { + "conversation_id": { "type": "string" }, + "full": { "type": "boolean", "description": "Return the last 10 turns. Default false." } + }, + "required": ["conversation_id"], + }), + }, + ] + } + + /// Run a tool. Failures become an `{"error": …}` output the model can + /// read; they never end the voice session. + pub async fn call(&self, name: &str, arguments: &str) -> ToolOutcome { + let arguments: Value = if arguments.trim().is_empty() { + json!({}) + } else { + match serde_json::from_str(arguments) { + Ok(value) => value, + Err(e) => return error_outcome(anyhow!("invalid arguments: {e}")), + } + }; + let result = match name { + "list_conversations" => self.list_conversations(arguments).await, + "list_projects" => self.list_projects().await, + "message_conversation" => self.message_conversation(arguments).await, + "get_conversation" => self.get_conversation(arguments).await, + other => Err(anyhow!("unknown tool {other}")), + }; + result.unwrap_or_else(error_outcome) + } + + async fn list_conversations(&self, arguments: Value) -> Result { + #[derive(Deserialize)] + struct Args { + project: Option, + #[serde(default)] + include_settled: bool, + limit: Option, + } + let args: Args = serde_json::from_value(arguments)?; + let mut sessions = self.service.list_sessions().await?; + let lifecycles = self.service.list_session_lifecycles().await?; + let states = self.service.session_activity_states().await?; + let now = SystemTime::now(); + + sessions.sort_by_key(|s| std::cmp::Reverse(s.updated_at)); + let limit = args + .limit + .unwrap_or(DEFAULT_LIST_LIMIT) + .clamp(1, MAX_LIST_LIMIT); + let conversations: Vec = sessions + .iter() + .filter(|s| { + args.project + .as_deref() + .is_none_or(|p| s.initial_project.eq_ignore_ascii_case(p)) + }) + .filter(|s| { + args.include_settled || !lifecycles.get(&s.id).is_some_and(|l| l.is_settled()) + }) + .take(limit) + .map(|s| { + json!({ + "id": s.id, + "title": s.name, + "project": s.initial_project, + "branch": s.branch, + "state": self.state_label(&s.id, states.get(&s.id)), + "last_active": relative_time(now, s.updated_at), + }) + }) + .collect(); + Ok(ToolOutcome { + output: json!({ "conversations": conversations }).to_string(), + effects: Vec::new(), + }) + } + + async fn list_projects(&self) -> Result { + let projects = self.project_names().await?; + Ok(ToolOutcome { + output: json!({ "projects": projects }).to_string(), + effects: Vec::new(), + }) + } + + async fn project_names(&self) -> Result> { + let mut names: BTreeSet = crate::config::load_projects() + .unwrap_or_default() + .into_keys() + .collect(); + for session in self.service.list_sessions().await? { + if !session.initial_project.is_empty() { + names.insert(session.initial_project); + } + } + Ok(names.into_iter().collect()) + } + + async fn message_conversation(&self, arguments: Value) -> Result { + #[derive(Deserialize)] + struct Args { + conversation_id: Option, + project: Option, + title: Option, + message: String, + } + let args: Args = serde_json::from_value(arguments)?; + if args.message.trim().is_empty() { + bail!("message is empty"); + } + let existing = args.conversation_id.filter(|id| !id.trim().is_empty()); + + let (conversation_id, status) = match existing { + Some(id) => { + let busy = self.service.is_session_busy(id.clone()).await?; + self.service + .send_or_queue_user_message(id.clone(), args.message, Vec::new()) + .await?; + (id, if busy { "queued" } else { "started" }) + } + None => { + let project = args + .project + .filter(|p| !p.trim().is_empty()) + .context("project is required to start a new conversation")?; + let known = self.project_names().await?; + let project = known + .iter() + .find(|name| name.eq_ignore_ascii_case(&project)) + .cloned() + .ok_or_else(|| { + anyhow!( + "unknown project '{project}'; known projects: {}", + known.join(", ") + ) + })?; + let title = args.title.filter(|t| !t.trim().is_empty()); + let id = self.service.create_session(title, Some(project)).await?; + self.service + .send_user_message(id.clone(), args.message, Vec::new(), None) + .await?; + (id, "started") + } + }; + Ok(ToolOutcome { + output: json!({ "conversation_id": conversation_id, "status": status }).to_string(), + effects: vec![ToolEffect::Touched(conversation_id)], + }) + } + + async fn get_conversation(&self, arguments: Value) -> Result { + #[derive(Deserialize)] + struct Args { + conversation_id: String, + #[serde(default)] + full: bool, + } + let args: Args = serde_json::from_value(arguments)?; + let id = args.conversation_id; + let content = self + .service + .session_content( + id.clone(), + ContentProjection { + parts: vec![ContentPart::UserText, ContentPart::AssistantText], + ..Default::default() + }, + ) + .await?; + let states = self.service.session_activity_states().await?; + let turns = group_turns(&content.items); + + let mut output = json!({ + "id": id, + "title": content.name, + "project": content.project, + "state": self.state_label(&id, states.get(&id)), + }); + if args.full { + output["turns"] = json!(last_turns(&turns, FULL_TURNS)); + } else { + let last = turns + .iter() + .rev() + .find_map(|turn| turn.agent.as_deref()) + .map(|text| truncate(text, MAX_MESSAGE_CHARS)); + output["last_agent_message"] = json!(last); + } + Ok(ToolOutcome { + output: output.to_string(), + effects: vec![ToolEffect::Read(id)], + }) + } + + fn state_label(&self, id: &str, state: Option<&SessionActivityState>) -> String { + if let Some(waiting) = self.open_requests.waiting_for(id) { + return waiting; + } + match state { + None | Some(SessionActivityState::Idle) => "idle".into(), + Some(SessionActivityState::Errored { message }) => format!("errored: {message}"), + Some(SessionActivityState::RunningExternally) => "running in another window".into(), + Some( + SessionActivityState::AgentRunning + | SessionActivityState::WaitingForResponse + | SessionActivityState::RateLimited { .. }, + ) => "running".into(), + } + } +} + +fn error_outcome(error: anyhow::Error) -> ToolOutcome { + ToolOutcome { + output: json!({ "error": format!("{error:#}") }).to_string(), + effects: Vec::new(), + } +} + +/// One exchange: a user message and the agent's last text before the next +/// user message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Turn { + pub user: Option, + pub agent: Option, +} + +pub fn group_turns(items: &[ContentItem]) -> Vec { + let mut turns: Vec = Vec::new(); + for item in items { + match item.kind { + ContentKind::UserText => turns.push(Turn { + user: Some(item.text.clone()), + agent: None, + }), + ContentKind::AssistantText => { + if item.text.trim().is_empty() { + continue; + } + match turns.last_mut() { + Some(turn) => turn.agent = Some(item.text.clone()), + None => turns.push(Turn { + user: None, + agent: Some(item.text.clone()), + }), + } + } + _ => {} + } + } + turns +} + +/// The last `count` turns as JSON, each message truncated, oldest dropped +/// first when the total exceeds [`MAX_TOTAL_CHARS`]. +fn last_turns(turns: &[Turn], count: usize) -> Vec { + let start = turns.len().saturating_sub(count); + let mut selected: Vec<(Option, Option)> = turns[start..] + .iter() + .map(|turn| { + ( + turn.user.as_deref().map(|t| truncate(t, MAX_MESSAGE_CHARS)), + turn.agent + .as_deref() + .map(|t| truncate(t, MAX_MESSAGE_CHARS)), + ) + }) + .collect(); + let size = |turns: &[(Option, Option)]| -> usize { + turns + .iter() + .map(|(u, a)| u.as_ref().map_or(0, |t| t.len()) + a.as_ref().map_or(0, |t| t.len())) + .sum() + }; + while selected.len() > 1 && size(&selected) > MAX_TOTAL_CHARS { + selected.remove(0); + } + selected + .into_iter() + .map(|(user, agent)| json!({ "user": user, "agent": agent })) + .collect() +} + +fn truncate(text: &str, max_chars: usize) -> String { + if text.chars().count() <= max_chars { + return text.to_string(); + } + let cut: String = text.chars().take(max_chars).collect(); + format!("{cut}… [truncated]") +} + +fn relative_time(now: SystemTime, then: SystemTime) -> String { + let seconds = now.duration_since(then).map(|d| d.as_secs()).unwrap_or(0); + match seconds { + 0..60 => "just now".into(), + 60..3600 => format!("{} minutes ago", seconds / 60), + 3600..86_400 => format!("{} hours ago", seconds / 3600), + _ => format!("{} days ago", seconds / 86_400), + } +} + +/// The metadata of a conversation, for notifications. +pub(super) fn find_metadata<'a>( + sessions: &'a [ChatMetadata], + id: &str, +) -> Option<&'a ChatMetadata> { + sessions.iter().find(|s| s.id == id) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::persistence::NodeId; + use crate::session_query::Role; + + fn item(kind: ContentKind, text: &str) -> ContentItem { + ContentItem { + message_index: 0, + node_id: NodeId::default(), + role: if kind == ContentKind::UserText { + Role::User + } else { + Role::Assistant + }, + kind, + text: text.into(), + tool_name: None, + tool_input: None, + is_error: None, + truncated: false, + } + } + + #[test] + fn turns_keep_only_the_last_agent_text() { + let items = vec![ + item(ContentKind::UserText, "fix it"), + item(ContentKind::AssistantText, "looking"), + item(ContentKind::AssistantText, "fixed"), + item(ContentKind::UserText, "thanks"), + ]; + assert_eq!( + group_turns(&items), + vec![ + Turn { + user: Some("fix it".into()), + agent: Some("fixed".into()) + }, + Turn { + user: Some("thanks".into()), + agent: None + }, + ] + ); + } + + #[test] + fn last_turns_are_limited_in_count_and_size() { + let turns: Vec = (0..15) + .map(|i| Turn { + user: Some(format!("q{i}")), + agent: Some("a".repeat(5_000)), + }) + .collect(); + let selected = last_turns(&turns, FULL_TURNS); + // 2 000 chars per message (+ marker) caps at 7 turns of 16 000. + assert!(selected.len() <= FULL_TURNS); + assert_eq!(selected.last().unwrap()["user"], "q14"); + let total: usize = selected + .iter() + .map(|t| t["agent"].as_str().unwrap().len() + t["user"].as_str().unwrap().len()) + .sum(); + assert!(total <= MAX_TOTAL_CHARS); + assert!( + selected[0]["agent"] + .as_str() + .unwrap() + .ends_with("[truncated]") + ); + + let few = last_turns(&turns[..3], FULL_TURNS); + assert_eq!(few.len(), 3); + } + + #[test] + fn truncation_counts_characters() { + assert_eq!(truncate("äöü", 3), "äöü"); + assert_eq!(truncate("äöüß", 3), "äöü… [truncated]"); + } +} diff --git a/crates/llm/src/factory.rs b/crates/llm/src/factory.rs index 0794786c..29a7dadf 100644 --- a/crates/llm/src/factory.rs +++ b/crates/llm/src/factory.rs @@ -366,12 +366,12 @@ pub async fn create_llm_client_from_configs( /// } /// ``` #[derive(Debug)] -struct AiCoreDeployment { - deployment_uuid: String, - api_type: AiCoreApiType, +pub(crate) struct AiCoreDeployment { + pub(crate) deployment_uuid: String, + pub(crate) api_type: AiCoreApiType, } -fn parse_aicore_deployment(model_id: &str, value: &Value) -> Result { +pub(crate) fn parse_aicore_deployment(model_id: &str, value: &Value) -> Result { // Try simple string format first (backwards compatible) if let Some(uuid) = value.as_str() { return Ok(AiCoreDeployment { diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index 016de370..a7d4abbc 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -32,6 +32,7 @@ pub mod openai_responses_ws; pub mod openrouter; pub mod prompt_caching; pub mod provider_config; +pub mod realtime; pub mod recording; pub mod streaming; pub mod types; diff --git a/crates/llm/src/realtime/aicore.rs b/crates/llm/src/realtime/aicore.rs new file mode 100644 index 00000000..7d102ddd --- /dev/null +++ b/crates/llm/src/realtime/aicore.rs @@ -0,0 +1,175 @@ +//! Realtime sessions through AI Core: OAuth client credentials from the +//! provider config, the deployment from its `models` map. +//! +//! The WebSocket endpoint is the deployment's `deploymentUrl` (read from the +//! deployment resource on every connect, so a token refresh or redeployment +//! is picked up on reconnect) plus `/v1/realtime`. + +use super::transport::{RealtimeEndpoint, WsConnector}; +use super::{RealtimeConnection, RealtimeConnector}; +use crate::auth::TokenManager; +use anyhow::{Context, Result, anyhow}; +use std::sync::Arc; +use tokio::sync::OnceCell; +use tracing::info; + +const RESOURCE_GROUP: &str = "default"; + +pub struct AiCoreConnector { + client_id: String, + client_secret: String, + token_url: String, + /// `https://…/v2/inference`, as in the provider config. + api_base_url: String, + deployment_id: String, + /// Created on the first connect (it fetches a token right away). + token_manager: OnceCell>, +} + +impl AiCoreConnector { + pub fn new( + client_id: String, + client_secret: String, + token_url: String, + api_base_url: String, + deployment_id: String, + ) -> Self { + Self { + client_id, + client_secret, + token_url, + api_base_url, + deployment_id, + token_manager: OnceCell::new(), + } + } + + async fn token(&self) -> Result { + let manager = self + .token_manager + .get_or_try_init(|| { + TokenManager::new( + self.client_id.clone(), + self.client_secret.clone(), + self.token_url.clone(), + ) + }) + .await + .context("AI Core authentication failed")?; + manager.get_valid_token().await + } + + /// The deployment resource's `deploymentUrl`. + async fn deployment_url(&self, token: &str) -> Result { + let url = format!( + "{}/deployments/{}", + lm_base_url(&self.api_base_url), + self.deployment_id + ); + let response: serde_json::Value = reqwest::Client::new() + .get(&url) + .bearer_auth(token) + .header("AI-Resource-Group", RESOURCE_GROUP) + .send() + .await + .with_context(|| format!("Failed to read the AI Core deployment {url}"))? + .error_for_status() + .with_context(|| format!("AI Core deployment {} not readable", self.deployment_id))? + .json() + .await?; + response + .get("deploymentUrl") + .and_then(|v| v.as_str()) + .filter(|url| !url.is_empty()) + .map(str::to_string) + .ok_or_else(|| { + anyhow!( + "AI Core deployment {} has no URL (status: {})", + self.deployment_id, + response + .get("status") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + ) + }) + } +} + +#[async_trait::async_trait] +impl RealtimeConnector for AiCoreConnector { + async fn connect(&self) -> Result { + let token = self.token().await?; + let deployment_url = self.deployment_url(&token).await?; + let url = realtime_url(&deployment_url); + info!("AI Core realtime endpoint: {url}"); + WsConnector::new(RealtimeEndpoint { + url, + headers: vec![ + ("Authorization".to_string(), format!("Bearer {token}")), + ("AI-Resource-Group".to_string(), RESOURCE_GROUP.to_string()), + ], + }) + .connect() + .await + } + + fn declares_model(&self) -> bool { + false + } +} + +/// Where deployment resources live: `:///v2/lm`, whatever +/// path the configured inference URL has (`…/v2/inference`, `…/inference`). +fn lm_base_url(api_base_url: &str) -> String { + let base = api_base_url.trim_end_matches('/'); + let host_end = base + .find("://") + .map(|scheme| { + let rest = scheme + 3; + base[rest..] + .find('/') + .map_or(base.len(), |slash| rest + slash) + }) + .unwrap_or(base.len()); + format!("{}/v2/lm", &base[..host_end]) +} + +fn realtime_url(deployment_url: &str) -> String { + let base = deployment_url.trim_end_matches('/'); + let base = if let Some(rest) = base.strip_prefix("https://") { + format!("wss://{rest}") + } else if let Some(rest) = base.strip_prefix("http://") { + format!("ws://{rest}") + } else { + base.to_string() + }; + format!("{base}/v1/realtime") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deployment_resources_live_under_v2_lm_on_the_api_host() { + for base in [ + "https://api.ai.example.com/v2/inference/", + "https://api.ai.example.com/inference", + "https://api.ai.example.com", + ] { + assert_eq!( + lm_base_url(base), + "https://api.ai.example.com/v2/lm", + "{base}" + ); + } + } + + #[test] + fn realtime_url_appends_the_path_on_a_websocket_scheme() { + assert_eq!( + realtime_url("https://api.ai.example.com/v2/inference/deployments/d1"), + "wss://api.ai.example.com/v2/inference/deployments/d1/v1/realtime" + ); + } +} diff --git a/crates/llm/src/realtime/events.rs b/crates/llm/src/realtime/events.rs new file mode 100644 index 00000000..dd0b3eb2 --- /dev/null +++ b/crates/llm/src/realtime/events.rs @@ -0,0 +1,304 @@ +//! Wire types of the OpenAI Realtime protocol: the client events we send and +//! the subset of server events a voice agent reacts to. +//! +//! Server events not listed here parse as [`ServerEvent::Other`]; the +//! protocol sends many informational events (rate limits, content part +//! boundaries, buffer commits) a client can ignore. + +use base64::Engine; +use serde::Deserialize; +use serde_json::{Value, json}; + +/// Sample rate of the PCM16 mono audio both directions use. +pub const SAMPLE_RATE: u32 = 24_000; + +/// A function the model may call, as declared in the session. +#[derive(Debug, Clone, PartialEq)] +pub struct ToolDefinition { + pub name: String, + pub description: String, + /// JSON schema of the arguments object. + pub parameters: Value, +} + +/// The session settings sent with `session.update`. +#[derive(Debug, Clone, PartialEq)] +pub struct SessionSettings { + /// `None` when the endpoint routes to a fixed model. + pub model: Option, + pub instructions: String, + pub voice: String, + pub tools: Vec, + /// The `turn_detection` object, e.g. `{"type": "semantic_vad"}`. + pub turn_detection: Value, + /// Model transcribing the user's speech; `None` turns transcription off. + pub transcription_model: Option, +} + +/// An event the client sends. +#[derive(Debug, Clone, PartialEq)] +pub enum ClientEvent { + SessionUpdate(SessionSettings), + /// Microphone samples (PCM16, 24 kHz, mono). + AppendAudio(Vec), + /// Ask the model to respond to the conversation as it stands. + CreateResponse, + /// Stop the response being generated. + CancelResponse, + /// Cut an assistant audio item down to what the user actually heard. + TruncateItem { + item_id: String, + audio_end_ms: u64, + }, + /// The result of a function call. + FunctionCallOutput { + call_id: String, + output: String, + }, + /// A text item from the system: context the model did not hear. + SystemMessage(String), +} + +impl ClientEvent { + pub fn to_json(&self) -> Value { + match self { + ClientEvent::SessionUpdate(settings) => { + let tools: Vec = settings + .tools + .iter() + .map(|tool| { + json!({ + "type": "function", + "name": tool.name, + "description": tool.description, + "parameters": tool.parameters, + }) + }) + .collect(); + let mut input = json!({ + "format": { "type": "audio/pcm", "rate": SAMPLE_RATE }, + "turn_detection": settings.turn_detection, + }); + if let Some(model) = &settings.transcription_model { + input["transcription"] = json!({ "model": model }); + } + let mut event = json!({ + "type": "session.update", + "session": { + "type": "realtime", + "instructions": settings.instructions, + "output_modalities": ["audio"], + "audio": { + "input": input, + "output": { + "format": { "type": "audio/pcm", "rate": SAMPLE_RATE }, + "voice": settings.voice, + }, + }, + "tools": tools, + "tool_choice": "auto", + }, + }); + if let Some(model) = &settings.model { + event["session"]["model"] = json!(model); + } + event + } + ClientEvent::AppendAudio(samples) => json!({ + "type": "input_audio_buffer.append", + "audio": encode_pcm16(samples), + }), + ClientEvent::CreateResponse => json!({ "type": "response.create" }), + ClientEvent::CancelResponse => json!({ "type": "response.cancel" }), + ClientEvent::TruncateItem { + item_id, + audio_end_ms, + } => json!({ + "type": "conversation.item.truncate", + "item_id": item_id, + "content_index": 0, + "audio_end_ms": audio_end_ms, + }), + ClientEvent::FunctionCallOutput { call_id, output } => json!({ + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": call_id, + "output": output, + }, + }), + ClientEvent::SystemMessage(text) => json!({ + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "system", + "content": [{ "type": "input_text", "text": text }], + }, + }), + } + } +} + +/// An item of the conversation as the server reports it. +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct ConversationItem { + #[serde(default)] + pub id: String, + #[serde(rename = "type")] + pub kind: String, + #[serde(default)] + pub name: Option, + #[serde(default)] + pub call_id: Option, + #[serde(default)] + pub arguments: Option, +} + +/// The response a `response.created` / `response.done` event is about. +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct ResponseInfo { + #[serde(default)] + pub id: String, + /// `completed`, `cancelled`, `failed` or `incomplete` once done. + #[serde(default)] + pub status: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct ErrorInfo { + #[serde(default)] + pub message: String, + #[serde(default)] + pub code: Option, +} + +/// A server event. Event names follow the GA protocol; the beta names of +/// the audio events are accepted as aliases. +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(tag = "type")] +pub enum ServerEvent { + #[serde(rename = "session.created")] + SessionCreated, + #[serde(rename = "session.updated")] + SessionUpdated, + #[serde(rename = "response.created")] + ResponseCreated { response: ResponseInfo }, + #[serde(rename = "response.done")] + ResponseDone { response: ResponseInfo }, + #[serde(rename = "response.output_item.added")] + OutputItemAdded { item: ConversationItem }, + #[serde(rename = "response.output_item.done")] + OutputItemDone { item: ConversationItem }, + #[serde(rename = "response.output_audio.delta", alias = "response.audio.delta")] + AudioDelta { item_id: String, delta: String }, + #[serde( + rename = "response.output_audio_transcript.done", + alias = "response.audio_transcript.done" + )] + AudioTranscriptDone { item_id: String, transcript: String }, + #[serde(rename = "input_audio_buffer.speech_started")] + SpeechStarted, + #[serde(rename = "input_audio_buffer.speech_stopped")] + SpeechStopped, + #[serde(rename = "conversation.item.input_audio_transcription.completed")] + InputTranscription { item_id: String, transcript: String }, + #[serde(rename = "error")] + Error { error: ErrorInfo }, + #[serde(other)] + Other, +} + +impl ServerEvent { + /// Parse a text frame. A known event with an unexpected shape is + /// reported as an error rather than silently dropped. + pub fn parse(text: &str) -> anyhow::Result { + Ok(serde_json::from_str(text)?) + } +} + +pub fn encode_pcm16(samples: &[i16]) -> String { + let mut bytes = Vec::with_capacity(samples.len() * 2); + for sample in samples { + bytes.extend_from_slice(&sample.to_le_bytes()); + } + base64::engine::general_purpose::STANDARD.encode(bytes) +} + +pub fn decode_pcm16(data: &str) -> anyhow::Result> { + let bytes = base64::engine::general_purpose::STANDARD.decode(data)?; + Ok(bytes + .as_chunks::<2>() + .0 + .iter() + .map(|pair| i16::from_le_bytes(*pair)) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pcm16_round_trips() { + let samples = vec![0, 1, -1, i16::MAX, i16::MIN, 1234]; + assert_eq!(decode_pcm16(&encode_pcm16(&samples)).unwrap(), samples); + } + + #[test] + fn parses_audio_delta_under_both_names() { + for name in ["response.output_audio.delta", "response.audio.delta"] { + let text = + format!(r#"{{"type":"{name}","item_id":"i1","delta":"AAA=","response_id":"r"}}"#); + assert_eq!( + ServerEvent::parse(&text).unwrap(), + ServerEvent::AudioDelta { + item_id: "i1".into(), + delta: "AAA=".into() + } + ); + } + } + + #[test] + fn unknown_events_parse_as_other() { + let event = ServerEvent::parse(r#"{"type":"rate_limits.updated","rate_limits":[]}"#); + assert_eq!(event.unwrap(), ServerEvent::Other); + } + + #[test] + fn parses_function_call_items() { + let text = r#"{"type":"response.output_item.done","item":{"id":"it","type":"function_call", + "name":"list_conversations","call_id":"c1","arguments":"{}","status":"completed"}}"#; + let ServerEvent::OutputItemDone { item } = ServerEvent::parse(text).unwrap() else { + panic!("wrong event"); + }; + assert_eq!(item.kind, "function_call"); + assert_eq!(item.call_id.as_deref(), Some("c1")); + assert_eq!(item.name.as_deref(), Some("list_conversations")); + } + + #[test] + fn session_update_declares_tools_and_audio_formats() { + let event = ClientEvent::SessionUpdate(SessionSettings { + model: Some("gpt-realtime".into()), + instructions: "be brief".into(), + voice: "marin".into(), + tools: vec![ToolDefinition { + name: "t".into(), + description: "d".into(), + parameters: json!({"type": "object"}), + }], + turn_detection: json!({"type": "semantic_vad"}), + transcription_model: Some("gpt-4o-mini-transcribe".into()), + }) + .to_json(); + assert_eq!(event["session"]["tools"][0]["name"], "t"); + assert_eq!(event["session"]["model"], "gpt-realtime"); + assert_eq!(event["session"]["audio"]["input"]["format"]["rate"], 24000); + assert_eq!(event["session"]["audio"]["output"]["voice"], "marin"); + assert_eq!( + event["session"]["audio"]["input"]["transcription"]["model"], + "gpt-4o-mini-transcribe" + ); + } +} diff --git a/crates/llm/src/realtime/mod.rs b/crates/llm/src/realtime/mod.rs new file mode 100644 index 00000000..52148854 --- /dev/null +++ b/crates/llm/src/realtime/mod.rs @@ -0,0 +1,128 @@ +//! Realtime (speech-to-speech) model sessions over the OpenAI Realtime +//! protocol. +//! +//! A realtime session is one long-lived WebSocket: audio streams both ways, +//! the server detects turns and keeps the conversation state. This module +//! only moves typed events; deciding *when* to send what is the caller's job +//! (see the voice agent in `code_assistant_core::voice`). +//! +//! [`RealtimeConnector`] is the seam: the WebSocket implementation is +//! [`WsConnector`], tests hand out channel pairs of their own. + +mod aicore; +mod events; +mod transport; + +pub use aicore::AiCoreConnector; +pub use events::{ + ClientEvent, ConversationItem, ErrorInfo, ResponseInfo, SAMPLE_RATE, ServerEvent, + SessionSettings, ToolDefinition, decode_pcm16, encode_pcm16, +}; +pub use transport::{RealtimeEndpoint, WsConnector}; + +use crate::provider_config::ProviderConfig; +use anyhow::{Context, Result, anyhow, bail}; +use serde_json::Value; +use std::sync::Arc; +use tokio::sync::mpsc; + +/// What arrives from an open session. +#[derive(Debug, Clone, PartialEq)] +pub enum Incoming { + Event(ServerEvent), + /// The connection ended; carries the reason when one is known. Nothing + /// follows. + Closed(Option), +} + +/// An open realtime session as a pair of channels. Dropping `outgoing` +/// closes the connection. +pub struct RealtimeConnection { + pub outgoing: mpsc::UnboundedSender, + pub incoming: mpsc::UnboundedReceiver, +} + +/// Opens realtime sessions; called again to reconnect. +#[async_trait::async_trait] +pub trait RealtimeConnector: Send + Sync { + async fn connect(&self) -> Result; + + /// Whether `session.update` names the model. A gateway that routes to a + /// deployment (AI Core) knows the model already. + fn declares_model(&self) -> bool { + true + } +} + +/// Provider types that can serve realtime sessions. +pub const REALTIME_PROVIDER_TYPES: &[&str] = &[ + "openai", + "openai-responses", + "openai-responses-ws", + "ai-core", +]; + +const DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; + +/// Whether a provider can serve realtime sessions. +pub fn supports_realtime(provider: &ProviderConfig) -> bool { + REALTIME_PROVIDER_TYPES.contains(&provider.provider.as_str()) +} + +/// The connector for `model` on a provider from `providers.json`. +/// +/// - OpenAI types: `api_key` auth, URL derived from `base_url`. +/// - `ai-core`: client credentials; `model` must be a key of the provider's +/// `models` map, whose deployment serves the session. +/// +/// `url_override` replaces the derived URL (OpenAI types only). +pub fn connector_for_provider( + provider: &ProviderConfig, + model: &str, + url_override: Option<&str>, +) -> Result> { + let config = &provider.config; + let text = |key: &str| -> Result { + config + .get(key) + .and_then(Value::as_str) + .map(str::to_string) + .with_context(|| format!("The provider '{}' has no {key}", provider.label)) + }; + match provider.provider.as_str() { + "openai" | "openai-responses" | "openai-responses-ws" => { + let api_key = text("api_key")?; + let base_url = text("base_url").unwrap_or_else(|_| DEFAULT_OPENAI_BASE_URL.into()); + let mut endpoint = RealtimeEndpoint::openai(&base_url, &api_key, model); + if let Some(url) = url_override.filter(|url| !url.is_empty()) { + endpoint.url = url.to_string(); + } + Ok(Arc::new(WsConnector::new(endpoint))) + } + "ai-core" => { + let deployment = config + .get("models") + .and_then(|models| models.get(model)) + .ok_or_else(|| { + anyhow!( + "The AI Core provider '{}' has no deployment for model '{model}' \ + (add it to the provider's models)", + provider.label + ) + })?; + let deployment = crate::factory::parse_aicore_deployment(model, deployment)?; + Ok(Arc::new(AiCoreConnector::new( + text("client_id")?, + text("client_secret")?, + text("token_url")?, + text("api_base_url")?, + deployment.deployment_uuid, + ))) + } + other => bail!( + "Provider '{}' ({other}) has no realtime API; voice mode needs an OpenAI or AI Core \ + provider", + provider.label + ), + } +} diff --git a/crates/llm/src/realtime/transport.rs b/crates/llm/src/realtime/transport.rs new file mode 100644 index 00000000..e569b343 --- /dev/null +++ b/crates/llm/src/realtime/transport.rs @@ -0,0 +1,159 @@ +//! The WebSocket transport of a realtime session. + +use super::{ClientEvent, Incoming, RealtimeConnection, RealtimeConnector, ServerEvent}; +use anyhow::{Context, Result}; +use futures_util::{SinkExt, StreamExt}; +use tokio::sync::mpsc; +use tokio_tungstenite::{ + connect_async_with_config, + tungstenite::{ + client::IntoClientRequest, + http::{HeaderName, HeaderValue}, + protocol::Message as WsMessage, + }, +}; +use tracing::{debug, info, warn}; + +/// Where and how to open a realtime session. +#[derive(Clone)] +pub struct RealtimeEndpoint { + pub url: String, + pub headers: Vec<(String, String)>, +} + +impl std::fmt::Debug for RealtimeEndpoint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Header values carry credentials. + f.debug_struct("RealtimeEndpoint") + .field("url", &self.url) + .field( + "headers", + &self.headers.iter().map(|(k, _)| k).collect::>(), + ) + .finish() + } +} + +impl RealtimeEndpoint { + /// The endpoint of an OpenAI-compatible API: `base_url` is the HTTP base + /// (`https://api.openai.com/v1`), the session lives at + /// `wss://…/v1/realtime?model=`. + pub fn openai(base_url: &str, api_key: &str, model: &str) -> Self { + let base = base_url.trim_end_matches('/'); + let ws_base = if let Some(rest) = base.strip_prefix("https://") { + format!("wss://{rest}") + } else if let Some(rest) = base.strip_prefix("http://") { + format!("ws://{rest}") + } else { + base.to_string() + }; + Self { + url: format!("{ws_base}/realtime?model={}", urlencoding::encode(model)), + headers: vec![("Authorization".to_string(), format!("Bearer {api_key}"))], + } + } +} + +/// Opens realtime sessions over WebSocket. +pub struct WsConnector { + endpoint: RealtimeEndpoint, +} + +impl WsConnector { + pub fn new(endpoint: RealtimeEndpoint) -> Self { + Self { endpoint } + } +} + +#[async_trait::async_trait] +impl RealtimeConnector for WsConnector { + async fn connect(&self) -> Result { + let mut request = self + .endpoint + .url + .as_str() + .into_client_request() + .context("Failed to build the realtime WebSocket request")?; + for (key, value) in &self.endpoint.headers { + request.headers_mut().insert( + HeaderName::from_bytes(key.as_bytes()).context("Invalid header name")?, + HeaderValue::from_str(value).context("Invalid header value")?, + ); + } + info!("Connecting realtime session to {}", self.endpoint.url); + let (stream, _response) = connect_async_with_config(request, None, false) + .await + .with_context(|| format!("Realtime connection to {} failed", self.endpoint.url))?; + let (mut sink, mut source) = stream.split(); + + let (outgoing_tx, mut outgoing_rx) = mpsc::unbounded_channel::(); + let (incoming_tx, incoming_rx) = mpsc::unbounded_channel::(); + let (pong_tx, mut pong_rx) = mpsc::unbounded_channel::>(); + + // Writer: client events and pongs. Ends when the caller drops its + // sender, which closes the socket. + tokio::spawn(async move { + loop { + let message = tokio::select! { + event = outgoing_rx.recv() => match event { + Some(event) => WsMessage::Text(event.to_json().to_string().into()), + None => break, + }, + Some(payload) = pong_rx.recv() => WsMessage::Pong(payload.into()), + }; + if let Err(e) = sink.send(message).await { + warn!("Realtime send failed: {e}"); + return; + } + } + let _ = sink.send(WsMessage::Close(None)).await; + }); + + // Reader: server events until the socket closes. + tokio::spawn(async move { + let reason = loop { + match source.next().await { + Some(Ok(WsMessage::Text(text))) => match ServerEvent::parse(&text) { + Ok(event) => { + if incoming_tx.send(Incoming::Event(event)).is_err() { + return; + } + } + Err(e) => debug!("Unparsed realtime event ({e}): {text}"), + }, + Some(Ok(WsMessage::Ping(payload))) => { + let _ = pong_tx.send(payload.to_vec()); + } + Some(Ok(WsMessage::Close(frame))) => { + break frame.map(|f| format!("{} {}", f.code, f.reason)); + } + Some(Ok(_)) => {} + Some(Err(e)) => break Some(e.to_string()), + None => break None, + } + }; + let _ = incoming_tx.send(Incoming::Closed(reason)); + }); + + Ok(RealtimeConnection { + outgoing: outgoing_tx, + incoming: incoming_rx, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn openai_endpoint_derives_the_websocket_url() { + let endpoint = RealtimeEndpoint::openai("https://api.openai.com/v1/", "k", "gpt-realtime"); + assert_eq!( + endpoint.url, + "wss://api.openai.com/v1/realtime?model=gpt-realtime" + ); + assert_eq!(endpoint.headers[0].1, "Bearer k"); + assert!(!format!("{endpoint:?}").contains("Bearer")); + } +} diff --git a/crates/ui_acp/src/ui.rs b/crates/ui_acp/src/ui.rs index 9fcf2e43..a8107d70 100644 --- a/crates/ui_acp/src/ui.rs +++ b/crates/ui_acp/src/ui.rs @@ -994,6 +994,9 @@ impl UserInterface for ACPUserUI { UiEvent::ConfigChanged => { // Config file changes not relevant in ACP mode } + UiEvent::VoiceStatusChanged { .. } | UiEvent::VoiceTranscript { .. } => { + // Voice mode is GPUI-specific. + } } Ok(()) } diff --git a/crates/ui_gpui/assets/icons/audio_lines.svg b/crates/ui_gpui/assets/icons/audio_lines.svg new file mode 100644 index 00000000..da57dfcc --- /dev/null +++ b/crates/ui_gpui/assets/icons/audio_lines.svg @@ -0,0 +1 @@ + diff --git a/crates/ui_gpui/assets/icons/mic.svg b/crates/ui_gpui/assets/icons/mic.svg new file mode 100644 index 00000000..a23c62b2 --- /dev/null +++ b/crates/ui_gpui/assets/icons/mic.svg @@ -0,0 +1 @@ + diff --git a/crates/ui_gpui/assets/icons/mic_off.svg b/crates/ui_gpui/assets/icons/mic_off.svg new file mode 100644 index 00000000..27d951da --- /dev/null +++ b/crates/ui_gpui/assets/icons/mic_off.svg @@ -0,0 +1 @@ + diff --git a/crates/ui_gpui/src/app/event_loop.rs b/crates/ui_gpui/src/app/event_loop.rs index 2e975feb..c2666c37 100644 --- a/crates/ui_gpui/src/app/event_loop.rs +++ b/crates/ui_gpui/src/app/event_loop.rs @@ -1080,8 +1080,22 @@ impl Gpui { cx.refresh(); } + UiEvent::VoiceStatusChanged { status } => { + if let code_assistant_core::voice::VoiceActivity::Failed(message) = &status.activity + { + *self.current_error.lock().unwrap() = + Some(format!("Voice mode stopped: {message}")); + } + self.apply_voice_status(status); + cx.refresh(); + } + UiEvent::VoiceTranscript { entry } => { + self.append_voice_transcript(entry); + cx.refresh(); + } UiEvent::ConfigChanged => { debug!("UI: ConfigChanged event — config files modified on disk"); + self.refresh_voice_configured(); self.config_generation .fetch_add(1, std::sync::atomic::Ordering::Relaxed); diff --git a/crates/ui_gpui/src/app/mod.rs b/crates/ui_gpui/src/app/mod.rs index ff53f70a..d8d6fa11 100644 --- a/crates/ui_gpui/src/app/mod.rs +++ b/crates/ui_gpui/src/app/mod.rs @@ -9,3 +9,4 @@ pub(super) mod commands; pub(super) mod drafts; pub(super) mod event_bridge; pub(super) mod event_loop; +pub(super) mod voice; diff --git a/crates/ui_gpui/src/app/voice.rs b/crates/ui_gpui/src/app/voice.rs new file mode 100644 index 00000000..aaea2b59 --- /dev/null +++ b/crates/ui_gpui/src/app/voice.rs @@ -0,0 +1,81 @@ +//! Voice mode state mirrored from the core, and the commands to drive it. + +use crate::Gpui; +use code_assistant_core::voice::{TranscriptEntry, VoiceConfig, VoiceService, VoiceStatus}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +/// Transcript lines kept for the title-bar popover. +const TRANSCRIPT_LIMIT: usize = 200; + +#[derive(Clone, Default)] +pub struct VoiceState { + service: Arc>>, + status: Arc>, + transcript: Arc>>, + /// Whether `voice.json` names a provider and model. + configured: Arc, +} + +impl Gpui { + /// Install the voice service; without one the title bar shows no voice + /// control. + pub fn set_voice_service(&self, service: VoiceService) { + *self.voice.service.lock().unwrap() = Some(service); + self.refresh_voice_configured(); + } + + pub fn voice_available(&self) -> bool { + self.voice.service.lock().unwrap().is_some() + } + + /// Whether a voice provider and model are configured. + pub fn voice_configured(&self) -> bool { + self.voice.configured.load(Ordering::Relaxed) + } + + /// Re-read `voice.json` (after the settings or the file changed). + pub fn refresh_voice_configured(&self) { + self.voice + .configured + .store(VoiceConfig::load().is_configured(), Ordering::Relaxed); + } + + pub fn voice_status(&self) -> VoiceStatus { + self.voice.status.lock().unwrap().clone() + } + + pub fn voice_transcript(&self) -> Vec { + self.voice.transcript.lock().unwrap().clone() + } + + /// Start voice mode, or stop it when it runs. + pub fn toggle_voice(&self) { + let Some(service) = self.voice.service.lock().unwrap().clone() else { + return; + }; + if self.voice_status().activity.is_active() { + service.stop(); + } else { + self.voice.transcript.lock().unwrap().clear(); + service.start(); + } + } + + pub fn set_voice_muted(&self, muted: bool) { + if let Some(service) = self.voice.service.lock().unwrap().as_ref() { + service.set_muted(muted); + } + } + + pub(crate) fn apply_voice_status(&self, status: VoiceStatus) { + *self.voice.status.lock().unwrap() = status; + } + + pub(crate) fn append_voice_transcript(&self, entry: TranscriptEntry) { + let mut transcript = self.voice.transcript.lock().unwrap(); + transcript.push(entry); + let excess = transcript.len().saturating_sub(TRANSCRIPT_LIMIT); + transcript.drain(..excess); + } +} diff --git a/crates/ui_gpui/src/lib.rs b/crates/ui_gpui/src/lib.rs index 1ea9e0b0..7e307d3b 100644 --- a/crates/ui_gpui/src/lib.rs +++ b/crates/ui_gpui/src/lib.rs @@ -345,6 +345,10 @@ pub struct Gpui { /// input-area completion and submit-time invocation. Refreshed on /// session load via [`Gpui::refresh_skills`]. skills: Arc>>, + + /// Voice mode: the handle (absent when the build has no audio), its + /// status and the transcript of the current voice conversation. + voice: app::voice::VoiceState, } /// State for a pending message edit (for branching) @@ -655,6 +659,8 @@ impl Gpui { config_generation: Arc::new(std::sync::atomic::AtomicU64::new(0)), skills: Arc::new(Mutex::new(Vec::new())), + + voice: app::voice::VoiceState::default(), } } diff --git a/crates/ui_gpui/src/main_screen/mod.rs b/crates/ui_gpui/src/main_screen/mod.rs index 016506ce..77c8d81e 100644 --- a/crates/ui_gpui/src/main_screen/mod.rs +++ b/crates/ui_gpui/src/main_screen/mod.rs @@ -3,6 +3,7 @@ pub mod project_dialog; mod question_prompt; pub mod right_panel; mod status_popover; +mod voice_control; use crate::shared::{frame_profile, ui_state}; use crate::sidebar::{SessionSidebar, SessionSidebarEvent}; @@ -208,6 +209,8 @@ pub struct MainScreen { /// Form for the viewed session's oldest open `ask_question` request. question_prompt: Option>, + /// Whether the voice transcript popover below the title bar is open. + voice_transcript_open: bool, /// Whether the sidebar got the first session listing. Without it, an /// empty listing equals the initial `sessions` and never reaches the /// sidebar, which then shows no project folders. @@ -350,6 +353,7 @@ impl MainScreen { ui_scale: initial_scale, right_sidebar_width: px(initial_sidebar_width), right_sidebar_resizing: false, + voice_transcript_open: false, sidebar_sessions_synced: false, resize_start_x: 0.0, resize_start_width: 0.0, @@ -2102,6 +2106,8 @@ impl Render for MainScreen { ), ) }) + // Voice mode, next to the right panel views + .children(self.render_voice_controls(cx)) .child( div() .id("about-btn") @@ -2300,6 +2306,8 @@ impl Render for MainScreen { } }), ) + // Voice transcript popover below the title bar + .children(self.render_voice_transcript(cx)) // Modal dialog overlay for new project creation .when_some(new_project_dialog, |el, dialog| el.child(dialog)) // Modal "About" dialog overlay diff --git a/crates/ui_gpui/src/main_screen/voice_control.rs b/crates/ui_gpui/src/main_screen/voice_control.rs new file mode 100644 index 00000000..42e53f0a --- /dev/null +++ b/crates/ui_gpui/src/main_screen/voice_control.rs @@ -0,0 +1,224 @@ +//! Voice mode in the title bar: start/stop, mute, the voice agent's state, +//! and a popover with the voice conversation's transcript. + +use super::MainScreen; +use crate::Gpui; +use code_assistant_core::voice::{TranscriptRole, VoiceActivity}; +use gpui_kit::component::tooltip::Tooltip; +use gpui_kit::component::{ActiveTheme, Icon, Sizable, Size}; +use gpui_kit::{AnyElement, Context, SharedString, div, prelude::*, px}; + +impl MainScreen { + /// The title-bar controls, or `None` when the build has no voice mode. + pub(super) fn render_voice_controls(&self, cx: &mut Context) -> Option { + let gpui = cx.try_global::()?; + if !gpui.voice_available() { + return None; + } + let status = gpui.voice_status(); + let active = status.activity.is_active(); + let failed = matches!(status.activity, VoiceActivity::Failed(_)); + // Without a voice model there is nothing to start; a running voice + // session stays controllable either way. + if !gpui.voice_configured() && !active && !failed { + return None; + } + + let mic_color = if failed { + cx.theme().danger + } else if active { + cx.theme().primary + } else { + cx.theme().muted_foreground + }; + let mic_tooltip = if active { + "Stop voice mode" + } else { + "Start voice mode" + }; + let mic = div() + .id("voice-toggle-btn") + .size(px(28.)) + .rounded_sm() + .flex() + .items_center() + .justify_center() + .cursor_pointer() + .hover(|s| s.bg(cx.theme().muted)) + .when(active, |el| el.bg(cx.theme().muted)) + .child( + Icon::default() + .path(SharedString::from("icons/mic.svg")) + .with_size(Size::Small) + .text_color(mic_color), + ) + .tooltip(move |window, cx| Tooltip::new(mic_tooltip).build(window, cx)) + .on_click(cx.listener(|this, _, _, cx| { + if let Some(gpui) = cx.try_global::() { + gpui.toggle_voice(); + } + this.voice_transcript_open = false; + cx.notify(); + })); + + let mut row = div().flex().items_center().gap_1().mr_1(); + if active { + let muted = status.muted; + let label = match status.activity { + VoiceActivity::Connecting => "Connecting…", + VoiceActivity::UserSpeaking => "Hearing you", + VoiceActivity::Speaking => "Speaking", + _ if muted => "Muted", + _ => "Listening", + }; + let label = if status.queued_notifications > 0 { + format!("{label} · {} waiting", status.queued_notifications) + } else { + label.to_string() + }; + let speaking = status.activity == VoiceActivity::Speaking; + row = row + .child( + div() + .id("voice-status-chip") + .h(px(24.)) + .px_2() + .rounded_md() + .flex() + .items_center() + .gap_1() + .cursor_pointer() + .text_xs() + .text_color(cx.theme().muted_foreground) + .hover(|s| s.bg(cx.theme().muted)) + .when(self.voice_transcript_open, |el| el.bg(cx.theme().muted)) + .child( + Icon::default() + .path(SharedString::from("icons/audio_lines.svg")) + .with_size(Size::XSmall) + .text_color(if speaking { + cx.theme().primary + } else { + cx.theme().muted_foreground + }), + ) + .child(SharedString::from(label)) + .tooltip(|window, cx| Tooltip::new("Show the transcript").build(window, cx)) + .on_click(cx.listener(|this, _, _, cx| { + this.voice_transcript_open = !this.voice_transcript_open; + cx.notify(); + })), + ) + .child( + div() + .id("voice-mute-btn") + .size(px(28.)) + .rounded_sm() + .flex() + .items_center() + .justify_center() + .cursor_pointer() + .hover(|s| s.bg(cx.theme().muted)) + .child( + Icon::default() + .path(SharedString::from(if muted { + "icons/mic_off.svg" + } else { + "icons/mic.svg" + })) + .with_size(Size::Small) + .text_color(if muted { + cx.theme().danger + } else { + cx.theme().muted_foreground + }), + ) + .tooltip(move |window, cx| { + Tooltip::new(if muted { "Unmute" } else { "Mute" }).build(window, cx) + }) + .on_click(move |_, _, cx| { + if let Some(gpui) = cx.try_global::() { + gpui.set_voice_muted(!muted); + } + }), + ); + } + Some(row.child(mic).into_any_element()) + } + + /// The transcript popover below the title bar, while open. + pub(super) fn render_voice_transcript(&self, cx: &mut Context) -> Option { + if !self.voice_transcript_open { + return None; + } + let gpui = cx.try_global::()?; + if !gpui.voice_status().activity.is_active() { + return None; + } + let transcript = gpui.voice_transcript(); + + let entries = transcript.into_iter().map(|entry| { + let (label, color) = match entry.role { + TranscriptRole::User => ("You", cx.theme().foreground), + TranscriptRole::Assistant => ("Assistant", cx.theme().primary), + TranscriptRole::Tool => ("Tool", cx.theme().muted_foreground), + TranscriptRole::Notification => ("Notification", cx.theme().muted_foreground), + }; + // A notification's first lines name the conversations; the + // instructions to the model below them are noise here. + let text = match entry.role { + TranscriptRole::Notification => entry + .text + .lines() + .skip(1) + .take_while(|line| !line.trim().is_empty()) + .collect::>() + .join("\n"), + _ => entry.text, + }; + div() + .flex() + .flex_col() + .gap(px(2.)) + .child(div().text_xs().text_color(color).child(label)) + .child( + div() + .text_sm() + .text_color(cx.theme().foreground) + .child(SharedString::from(text)), + ) + }); + + let empty = div() + .text_sm() + .text_color(cx.theme().muted_foreground) + .child("Say something — the transcript appears here."); + + Some( + div() + .absolute() + .top(px(52.)) + .right(px(16.)) + .w(px(380.)) + .max_h(px(440.)) + .bg(cx.theme().popover) + .border_1() + .border_color(cx.theme().border) + .rounded_lg() + .shadow_lg() + .child( + div() + .id("voice-transcript") + .max_h(px(440.)) + .overflow_y_scroll() + .p_3() + .flex() + .flex_col() + .gap_3() + .children(entries) + .when(gpui.voice_transcript().is_empty(), |el| el.child(empty)), + ) + .into_any_element(), + ) + } +} diff --git a/crates/ui_gpui/src/settings_screen/mod.rs b/crates/ui_gpui/src/settings_screen/mod.rs index c06130e3..0488eb64 100644 --- a/crates/ui_gpui/src/settings_screen/mod.rs +++ b/crates/ui_gpui/src/settings_screen/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod provider_forms; pub(crate) mod provider_suggestions; mod providers_section; mod skills_section; +mod voice_section; use gpui_kit::component::scroll::ScrollableElement; use gpui_kit::component::{ActiveTheme, Icon, Sizable, Size}; @@ -23,6 +24,7 @@ pub enum SettingsSection { Models, Skills, McpServers, + Voice, } /// Events emitted by the settings screen. @@ -43,6 +45,7 @@ pub struct SettingsScreen { general_section: Entity, skills_section: Entity, mcp_section: Entity, + voice_section: Entity, } impl SettingsScreen { @@ -53,6 +56,7 @@ impl SettingsScreen { let general_section = cx.new(|cx| general_section::GeneralSection::new(window, cx)); let skills_section = cx.new(|cx| skills_section::SkillsSection::new(window, cx)); let mcp_section = cx.new(|cx| mcp_section::McpSection::new(window, cx)); + let voice_section = cx.new(|cx| voice_section::VoiceSection::new(window, cx)); Self { focus_handle: cx.focus_handle(), @@ -62,6 +66,7 @@ impl SettingsScreen { general_section, skills_section, mcp_section, + voice_section, } } @@ -104,6 +109,9 @@ impl SettingsScreen { SettingsSection::McpServers => { self.mcp_section.update(cx, |s, _cx| s.reload()); } + SettingsSection::Voice => { + self.voice_section.update(cx, |s, _cx| s.reload()); + } SettingsSection::General => {} } @@ -282,6 +290,12 @@ impl Render for SettingsScreen { "MCP Servers", "icons/braces.svg", cx, + )) + .child(self.render_nav_item( + SettingsSection::Voice, + "Voice", + "icons/mic.svg", + cx, )), ), ) @@ -304,6 +318,7 @@ impl Render for SettingsScreen { SettingsSection::McpServers => { self.mcp_section.clone().into_any_element() } + SettingsSection::Voice => self.voice_section.clone().into_any_element(), }, )), ) diff --git a/crates/ui_gpui/src/settings_screen/voice_section.rs b/crates/ui_gpui/src/settings_screen/voice_section.rs new file mode 100644 index 00000000..15249d78 --- /dev/null +++ b/crates/ui_gpui/src/settings_screen/voice_section.rs @@ -0,0 +1,366 @@ +//! Voice settings: the realtime model voice mode talks to, and when +//! conversations report back. Stored in `voice.json`. + +use code_assistant_core::voice::config::supports_realtime; +use code_assistant_core::voice::{NotifyScope, VoiceConfig}; +use gpui_kit::component::input::{Input, InputEvent, InputState}; +use gpui_kit::component::switch::Switch; +use gpui_kit::component::{ActiveTheme, Sizable, Size}; +use gpui_kit::{ + App, Context, Entity, FocusHandle, Focusable, SharedString, Subscription, div, prelude::*, px, +}; +use tracing::warn; + +const EAGERNESS: &[&str] = &["auto", "low", "medium", "high"]; + +/// Which text field an input edits. +#[derive(Clone, Copy)] +enum Field { + Model, + Voice, + Transcription, + Cooling, +} + +pub struct VoiceSection { + focus_handle: FocusHandle, + config: VoiceConfig, + /// (provider id, label) of the providers with a realtime API. + providers: Vec<(String, String)>, + model_input: Entity, + voice_input: Entity, + transcription_input: Entity, + cooling_input: Entity, + _subscriptions: Vec, +} + +impl VoiceSection { + pub fn new(window: &mut gpui_kit::Window, cx: &mut Context) -> Self { + let config = VoiceConfig::load(); + let mut subscriptions = Vec::new(); + let mut input = |value: String, placeholder: &str, field: Field, cx: &mut Context| { + let placeholder = placeholder.to_string(); + let state = cx.new(|cx| { + InputState::new(window, cx) + .placeholder(placeholder) + .default_value(value) + }); + subscriptions.push(cx.subscribe_in( + &state, + window, + move |this, input, event: &InputEvent, _window, cx| { + if matches!(event, InputEvent::Change) { + let value = input.read(cx).value().trim().to_string(); + this.on_field_changed(field, value, cx); + } + }, + )); + state + }; + let model_input = input(config.model.clone(), "gpt-realtime", Field::Model, cx); + let voice_input = input(config.voice.clone(), "marin", Field::Voice, cx); + let transcription_input = input( + config.transcription_model.clone(), + "gpt-4o-mini-transcribe", + Field::Transcription, + cx, + ); + let cooling_input = input(config.cooling_ms.to_string(), "1500", Field::Cooling, cx); + Self { + focus_handle: cx.focus_handle(), + config, + providers: realtime_providers(), + model_input, + voice_input, + transcription_input, + cooling_input, + _subscriptions: subscriptions, + } + } + + /// Re-read the providers (they may have changed in another section). + pub fn reload(&mut self) { + self.providers = realtime_providers(); + self.config = VoiceConfig::load(); + } + + fn on_field_changed(&mut self, field: Field, value: String, cx: &App) { + match field { + Field::Model => self.config.model = value, + Field::Voice => self.config.voice = value, + Field::Transcription => self.config.transcription_model = value, + Field::Cooling => { + let Ok(ms) = value.parse() else { return }; + self.config.cooling_ms = ms; + } + } + self.save(cx); + } + + fn update(&mut self, change: impl FnOnce(&mut VoiceConfig), cx: &mut Context) { + change(&mut self.config); + self.save(cx); + cx.notify(); + } + + fn save(&self, cx: &App) { + if let Err(e) = self.config.save() { + warn!("Failed to save the voice settings: {e:#}"); + } + // The title bar shows voice mode only once it is configured. + if let Some(gpui) = cx.try_global::() { + gpui.refresh_voice_configured(); + } + } + + fn render_choice( + &self, + id: String, + label: String, + selected: bool, + on_click: impl Fn(&mut Self, &mut Context) + 'static, + cx: &mut Context, + ) -> impl IntoElement { + div() + .id(SharedString::from(id)) + .px_3() + .py_1() + .rounded_md() + .border_1() + .cursor_pointer() + .text_sm() + .border_color(if selected { + cx.theme().primary + } else { + cx.theme().border + }) + .text_color(if selected { + cx.theme().foreground + } else { + cx.theme().muted_foreground + }) + .when(selected, |el| el.bg(cx.theme().muted)) + .hover(|s| s.bg(cx.theme().muted)) + .child(SharedString::from(label)) + .on_click(cx.listener(move |this, _, _, cx| on_click(this, cx))) + } + + fn render_field( + &self, + label: &str, + input: &Entity, + hint: &str, + cx: &mut Context, + ) -> impl IntoElement { + div() + .flex() + .flex_col() + .gap_1() + .child(caption(label, cx)) + .child(div().w(px(260.)).child(Input::new(input))) + .child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child(SharedString::from(hint.to_string())), + ) + } +} + +fn realtime_providers() -> Vec<(String, String)> { + let mut providers: Vec<(String, String)> = + llm::provider_config::ConfigurationSystem::load_providers_config(None) + .unwrap_or_default() + .into_iter() + .filter(|(_, provider)| supports_realtime(provider)) + .map(|(id, provider)| (id, provider.label)) + .collect(); + providers.sort_by(|a, b| a.1.cmp(&b.1)); + providers +} + +fn caption(text: &str, cx: &App) -> impl IntoElement { + div() + .text_xs() + .font_weight(gpui_kit::FontWeight::MEDIUM) + .text_color(cx.theme().muted_foreground) + .child(SharedString::from(text.to_string())) +} + +impl Focusable for VoiceSection { + fn focus_handle(&self, _: &App) -> FocusHandle { + self.focus_handle.clone() + } +} + +impl Render for VoiceSection { + fn render( + &mut self, + _window: &mut gpui_kit::Window, + cx: &mut Context, + ) -> impl IntoElement { + let provider_choices: Vec<_> = self + .providers + .iter() + .map(|(id, label)| { + let selected = self.config.provider == *id; + let provider = id.clone(); + self.render_choice( + format!("voice-provider-{id}"), + label.clone(), + selected, + move |this, cx| { + let provider = provider.clone(); + this.update(|config| config.provider = provider, cx) + }, + cx, + ) + .into_any_element() + }) + .collect(); + let eagerness_choices: Vec<_> = EAGERNESS + .iter() + .map(|value| { + let selected = self.config.vad_eagerness == *value; + self.render_choice( + format!("voice-eagerness-{value}"), + value.to_string(), + selected, + move |this, cx| { + this.update(|config| config.vad_eagerness = value.to_string(), cx) + }, + cx, + ) + .into_any_element() + }) + .collect(); + let touched_only = self.config.notify == NotifyScope::Touched; + + div() + .flex() + .flex_col() + .gap_5() + .w_full() + .max_w(px(700.)) + .child( + div() + .text_xs() + .font_weight(gpui_kit::FontWeight::SEMIBOLD) + .text_color(cx.theme().muted_foreground) + .child("VOICE"), + ) + .child( + div() + .p_4() + .rounded_lg() + .border_1() + .border_color(cx.theme().border) + .bg(cx.theme().secondary) + .text_sm() + .text_color(cx.theme().foreground) + .child( + "Voice mode is a hands-free assistant for all your conversations: ask \ + what is running, start a conversation, or send one a message. Start it \ + with the microphone in the title bar. It tells you when a conversation \ + finishes, but never talks over you.", + ), + ) + .child( + div() + .flex() + .flex_col() + .gap_2() + .child(caption("Realtime provider", cx)) + .child(if provider_choices.is_empty() { + div() + .text_sm() + .text_color(cx.theme().muted_foreground) + .child( + "No provider with a realtime API yet. Add an OpenAI or AI Core \ + provider under Providers.", + ) + .into_any_element() + } else { + div() + .flex() + .flex_wrap() + .gap_2() + .children(provider_choices) + .into_any_element() + }), + ) + .child(self.render_field( + "Model", + &self.model_input, + "A speech-to-speech model of the provider. For AI Core: the name in the \ + provider's models map whose deployment to use.", + cx, + )) + .child(self.render_field( + "Voice", + &self.voice_input, + "The model's voice, e.g. marin, cedar, alloy.", + cx, + )) + .child(self.render_field( + "Transcription model", + &self.transcription_input, + "Transcribes your speech for the transcript; empty turns it off.", + cx, + )) + .child( + div() + .flex() + .flex_col() + .gap_2() + .child(caption("Turn detection", cx)) + .child(div().flex().gap_2().children(eagerness_choices)) + .child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child( + "How soon the model answers after you pause. Low waits \ + longer, high answers sooner.", + ), + ), + ) + .child(self.render_field( + "Quiet time before notifications (ms)", + &self.cooling_input, + "How long it must be silent after the model spoke before it tells you about \ + finished conversations.", + cx, + )) + .child( + div() + .flex() + .items_center() + .gap_2() + .child( + Switch::new("voice-notify-touched") + .checked(touched_only) + .with_size(Size::Small) + .on_click({ + let view = cx.entity(); + move |enabled, _window, app| { + let scope = if *enabled { + NotifyScope::Touched + } else { + NotifyScope::All + }; + view.update(app, |this, cx| { + this.update(|config| config.notify = scope, cx) + }); + } + }), + ) + .child( + div() + .text_sm() + .text_color(cx.theme().foreground) + .child("Only report conversations started or messaged by voice"), + ), + ) + } +} diff --git a/docs/voice-mode.md b/docs/voice-mode.md new file mode 100644 index 00000000..df028db3 --- /dev/null +++ b/docs/voice-mode.md @@ -0,0 +1,323 @@ +# Voice mode + +Status: implemented (2026-10-08). Code: `code_assistant_core::voice`, +`llm::realtime`, `crates/audio`, GPUI title bar and *Settings → Voice*. + +## Goal + +A global voice assistant, separate from any single session. The user +switches it on once (title bar) and then talks to it about all of their +conversations: "what is running?", "start a conversation in project X that +fixes the flaky test", "tell the refactoring session to also update the +docs", "what did the build session end with?". + +- **Global, not per session.** One voice agent per app process. It never + lives inside a session's transcript and is not tied to the session being + viewed. +- **Realtime model, configured in settings.** A speech-to-speech model + (OpenAI Realtime protocol first) chosen in a new *Voice* settings section. +- **Its own small agent loop.** It has no file, shell or browser tools. + Its only tools act on conversations, through `SessionService`. +- **Voice first.** When a background conversation finishes, the voice agent + is told, but the notification never interrupts the user or the voice + agent while either is speaking. It waits for real silence. + +Non-goals for v1: voice input into a normal session's composer (dictation), +answering permission prompts or questions by voice (see Follow-ups), the +terminal and ACP frontends. + +## Why not `agent_core::runtime` + +The existing agent loop is request/response: build the history, call +`LLMProvider::send_message`, run the tools, repeat. A realtime model works +differently. One long-lived WebSocket session keeps the conversation state +on the server, audio streams both ways all the time, the server detects +turns (VAD), and a response can be cancelled halfway through playback. So +the voice agent gets its own event loop. It reuses: + +- `SessionService` and `EventStream`, as its only link to the rest of the + app, exactly like a frontend; +- `session_query` — for reading transcripts. + +Its four tools are a small dispatch of their own (`voice::tools`), not a +`tools_core` registry: they need none of the tool context (command executor, +permissions, rendering) and are declared to the realtime session as plain +function schemas. + +## Architecture + +``` + ┌──────────── ui_gpui ─────────────┐ + │ title-bar toggle, voice indicator │ + │ Voice settings section │ + └──────▲──────────────┬────────────┘ + UiEvent::Voice* VoiceService::start/stop + │ │ +┌──────────────── code_assistant_core::voice ─────────────────┐ +│ VoiceService (start/stop, owns one VoiceAgent task) │ +│ │ +│ VoiceAgent event loop ── select! over: │ +│ • realtime server events (RealtimeConnection) │ +│ • audio events (captured, drained) │ +│ • tool completions (spawned tool futures) │ +│ • EventStream subscription → NotificationSource │ +│ • timers (cooling, playback fallback)│ +│ every input → Floor::handle(event) → Vec │ +│ │ +│ Floor (pure state machine) NotificationQueue voice tools │ +└───────▲───────────────────────────────▲──────────────────────┘ + │ │ + llm::realtime (protocol, audio (voice-processing I/O on + WebSocket transport) macOS, cpal elsewhere) +``` + +### New pieces by layer + +| Layer | Piece | Responsibility | +|---|---|---| +| 0 | `llm::realtime` | Protocol types (`ClientEvent`, `ServerEvent`), `WsConnector` over `tokio-tungstenite`, `RealtimeEndpoint::openai` (API-key auth). A connection is a pair of channels; the `RealtimeConnector` trait lets tests script the server. | +| 0 | crate `audio` | `AudioIo`: default microphone and speakers, PCM16 mono 24 kHz, a playback queue that counts played samples and reports `Drained`, `clear_playback()` for barge-in. macOS: one `VoiceProcessingIO` unit (echo cancellation, format conversion). Elsewhere: `cpal` with linear resampling and half-duplex gating. | +| 3 | `code_assistant_core::voice` | `VoiceConfig`, `VoiceService`, `VoiceAgent` loop, `Floor` state machine, `NotificationQueue`, `NotificationSource`, voice tools, instructions. Audio comes in through the `VoiceAudio` trait / `AudioFactory`. | +| 4 | `ui_gpui` | Title-bar toggle, mute button, status chip, transcript popover, *Voice* settings section. | +| 5 | `code_assistant` | Feature `voice` (default). Adapts `audio::AudioIo` to `VoiceAudio` and runs the `VoiceService` worker next to the `SessionService` worker. | + +The `audio` crate sits behind the `voice` feature, so headless builds +(`--no-default-features`) do not link CoreAudio, ALSA or WASAPI. The core +does not depend on it. + +`cargo run -p audio --example devices [-- --silence | --tone]` opens the +devices for two seconds and reports captured chunks, playback and drains. + +## Configuration + +Stored in `/voice.json` and edited in a new *Voice* settings +section. It follows the pattern of `handoff.json` and `lifecycle.json`. + +```json +{ + "provider": "openai", // provider id from providers.json + "model": "gpt-realtime", + "voice": "marin", + "url": "wss://…", // optional, overrides the derived URL + "transcription_model": "gpt-4o-mini-transcribe", // "" = no transcript + "vad_eagerness": "auto", // semantic VAD: low | medium | high | auto + "cooling_ms": 1500, + "notify": "all" // "all" | "touched" (see Notifications) +} +``` + +Credentials come from the referenced `providers.json` entry, so nothing is +duplicated. Realtime models stay out of `models.json` on purpose: they cannot +run a normal session, and adding them there would put them into every model +picker. The settings section offers the providers that can serve realtime +sessions (`llm::realtime::connector_for_provider`): + +- `openai`, `openai-responses`, `openai-responses-ws` with an `api_key`: the + URL is `wss:///…/realtime?model=…` (or `url`). +- `ai-core`: the provider's client credentials and token manager; `model` + names an entry of the provider's `models` map whose deployment serves the + session. On every connect the connector reads the deployment resource + (`…/v2/lm/deployments/{id}`), takes its `deploymentUrl`, appends + `/v1/realtime` and connects with the bearer token and + `AI-Resource-Group: default`. `session.update` then omits `model`. + +The devices are the system defaults. + +## Voice tools + +All tools are thin adapters over `SessionService` / `session_query` +(`SessionService::session_content` reads a stored session without making +it the active one; `session_activity_states` gives the loaded sessions' +states). Every tool returns at once. None waits for a conversation's agent to finish. + +| Tool | Parameters | Behaviour | +|---|---|---| +| `list_conversations` | `project?`, `include_settled?` (default false), `limit?` (default 20) | `list_sessions` + lifecycle + activity state → `[{id, title, project, branch, state, last_active}]`, where `state` is `running`, `idle`, `errored: …` or `waiting for …` (permission or answers). Sorted by recent activity. | +| `list_projects` | — | Project names, so `message_conversation` can create sessions in the right place. Can be merged into `list_conversations` later if it turns out to be noise. | +| `message_conversation` | `conversation_id?`, `project?`, `title?`, `message` | With no id, `create_session(title, project)` (the project must be known) and then send. With an id, `send_or_queue_user_message`. Returns `{conversation_id, status}` with `status` `started` or `queued`. Marks the session as *touched*. | +| `get_conversation` | `conversation_id`, `full?` (default false) | Default: `{state, last_agent_message}`, the final assistant text of the latest turn. With `full: true`: the last 10 turns, each with the user message and the turn's **last** assistant text only (no thinking, no tool calls, no tool results). Every text is truncated (about 2 000 characters each, with a total cap). Built on `session_query::get_session_content` with `UserText` + `AssistantText` plus grouping by turn. Calling it acknowledges any pending notification for that conversation. | + +`message_conversation` replaces separate *create* and *comment* tools, as +suggested: an empty `conversation_id` means create. The voice agent never +reads the visible session or switches it. A later option is a +`show_conversation` tool that asks the GPUI frontend to open a session. + +## Notifications + +`NotificationSource` subscribes to the `EventStream` and watches +`UiEvent::UpdateSessionActivityState`: + +- running → `Idle`: *finished* +- running → `Errored`: *failed* +- `RequestToolPermission` / `RequestUserQuestions`: *waiting for you* + (v1 only reports it; it cannot answer) + +With `notify: "all"` (the default), every session in this process is +reported. `"touched"` limits it to sessions the voice agent created or +messaged during this voice session. Sessions running in another process +(`RunningExternally`) send no events and are not covered. + +On `StreamError::Lagged` the source gets back in sync by diffing +`list_sessions` activity against its last known states. + +Each notification enters the `NotificationQueue`, keyed by session id. A +newer event for the same session replaces the older one, so a session that +finishes twice before it is flushed is reported once. A `get_conversation` +call for that session drops its entry: the model already knows. + +When flushed, the queued entries are put into **one** text item. One +`response.create` follows: + +> [background notification] +> - Conversation "Fix flaky test" in project proj-x (id …) finished its turn. +> - Conversation "Refactor parser" (id …) is waiting for permission to run +> `execute_command`. +> +> The user is not speaking right now. If it is useful to them, mention this +> briefly. Call get_conversation before you summarise a result. Otherwise +> say nothing. + +A "waiting for you" entry is withdrawn when the request is answered in the +UI before the floor frees up. + +## Two-level priority: the `Floor` state machine + +This is the core of the feature and the part to get right first. It is a +**pure** state machine. Inputs are events, outputs are commands; it has no +I/O and no clock (timers are commands the loop runs and reports back as +events). That makes it fully unit-testable. + +Priority levels: + +1. **Voice**: user speech, the model's answer to the user, and tool calls + and results that belong to that answer. +2. **Background**: conversation notifications. They are only ever delivered + when the floor is free, and never cancel or delay level 1. + +The model generates audio much faster than it plays, so `response.done` +does **not** mean "stopped talking". Because we own the speakers natively, +the playback queue reports `Drained` exactly, with no browser round trip. +A fallback timer of `audio duration + 5 s` still guards against a stuck +sink. + +### States + +| State | Meaning | +|---|---| +| `Speaking` | a response is being generated **or** its audio is still playing | +| `UserTurn` | the user is speaking, or has spoken and is owed an answer | +| `Cooling` | playback drained; silence timer (`cooling_ms`) running | +| `Idle` | silent, nothing pending | + +### Transitions + +| Input | Effect | +|---|---| +| server `response.created` | cancel timers → `Speaking` | +| server `response.output_audio.delta` | (agent) queue for the speakers; remember the item and the sample offset its audio starts at | +| server `response.output_item.done` (function call) | (agent) spawn the tool → `ToolStarted` | +| server `response.done` / cancelled | attach held tool outputs; in `Speaking`, wait for the drain (playback timer: pending audio + 5 s), or act as drained if nothing is pending | +| server `input_audio_buffer.speech_started` (barge-in) | cancel timers; if `Speaking`: `response.cancel` (deferred until `response.created` when our create is still in flight) and `StopPlayback` (agent: clear the queue, `conversation.item.truncate` to the samples played, drop late deltas of that item) → `UserTurn` | +| server `input_audio_buffer.speech_stopped` | stays `UserTurn`; the server's VAD creates the response. If none follows within 8 s the floor is free again | +| server `error` | when our `response.create` was in flight: give the floor back (no wedge) | +| sink `Drained` (or fallback timer) | if a trigger is pending (tool outputs attached during `Speaking`) → `response.create`; otherwise → `Cooling` + start the cooling timer | +| cooling timer fires | `Cooling` → `Idle`; flush notifications if any are queued | +| tool completed | response still open → hold until `response.done`; `Speaking` after generation → attach and set the pending trigger (fired on `Drained`); `UserTurn` → attach **without** `response.create` (the model uses it in its next answer); `Cooling`/`Idle` → attach + `response.create` | +| notification arrives | always queue; `Speaking`/`UserTurn` → nothing more; `Cooling` → restart the cooling timer (groups finishes that land close together); `Idle` → flush now | +| `get_conversation` read | drop that conversation's queued notification | + +While a tool call runs, notifications are not flushed: the tool's result +makes the model speak anyway, and the notification follows after that +answer. + +Invariants, each with its own test: + +1. No `response.create` while a response is open. +2. No `response.create` in `UserTurn` caused by a tool result or a + notification. +3. A notification is only flushed after `Drained` **and** `cooling_ms` of + silence, or straight from `Idle`. +4. A notification never causes `response.cancel` or `flush()`. +5. Barge-in always truncates the server-side item to what was actually + heard, so the model does not believe it said things the user never heard. + +### Function calls inside a response + +Voice tools run concurrently as spawned futures. Their outputs go through +the floor as *tool completed* events, never straight to the socket. So the +rules above apply equally to slow calls (`get_conversation` on a large +session) and to fast ones. + +## Session lifetime and reconnect + +- Start: open the realtime session → `session.update` (instructions, tools, + voice, turn detection, input transcription on) → start capture. +- Realtime sessions have a maximum duration and can drop. The agent keeps a + local text transcript (user transcripts + assistant transcripts + tool + calls). On a closed connection it reconnects (three attempts, backing + off), re-sends `session.update`, seeds the last 30 transcript lines as one + system message, and keeps the notification queue (flushed after a cooling + period). Tool results of the old connection are dropped. +- Stop: the toggle, or an error that cannot be retried. Capture and + playback stop, the socket closes, and the queue is dropped. +- Mute: capture is paused, and notifications are still delivered (they only + speak from `Idle`, which a muted user is in). +- Voice runs keep no transcript on disk in v1. The live transcript is kept + in memory for the UI popover and for reconnect. + +## UI (GPUI) + +- Title bar: a microphone toggle (start/stop), and while active a status + chip (Connecting… / Listening / Hearing you / Speaking / Muted, plus + "· N waiting" for queued notifications) and a mute button. Events arrive + as app-scoped `UiEvent::VoiceStatusChanged` and `UiEvent::VoiceTranscript` + (`session_id: None` on the `EventStream`); `Gpui` mirrors them + (`app/voice.rs`). A failure shows in the error popover. +- The status chip opens the transcript popover. +- Settings → *Voice*: provider, model, voice, transcription model, VAD + eagerness, cooling time, notify scope. +- macOS bundle: `NSMicrophoneUsageDescription` in `Info.plist` and the + `com.apple.security.device.audio-input` entitlement for the hardened + runtime. +- The voice agent posts into sessions like a user would, so those sessions + show the message as a normal user message. Later, a small "via voice" + marker on the message could be added. + +## Decisions + +1. **Echo cancellation: native.** On macOS the `audio` crate drives a + `VoiceProcessingIO` audio unit (the system's echo canceller, the one + FaceTime uses) for capture and playback, so the user can interrupt the + model while it speaks. Other platforms fall back to `cpal` with + half-duplex gating: the microphone is dropped while audio plays, so there + is no barge-in there. +2. **Provider: OpenAI Realtime protocol**, directly (`openai*` provider + types) or through AI Core (`ai-core`, deployment from the provider's + `models` map). `voice.json` can override the OpenAI URL for compatible + gateways. +3. **Notify scope: all conversations** by default. + +## Tests + +- `voice::floor` — every transition and invariant, table-style. +- `voice::notifications`, `voice::source`, `voice::tools` (turn grouping + and limits), `voice::config` (endpoint derivation). +- `voice::tests` — end to end: scripted realtime server, fake audio devices + and a real `SessionService` with a mock LLM. A conversation finishing while + the model speaks is announced only after drain plus cooling; a + `message_conversation` call starts a session whose end is announced; + `get_conversation` returns the turns and suppresses the notification; + barge-in cancels and truncates to what was heard. +- `llm::realtime` — event parsing and serialization, PCM16 encoding. +- `audio` — playback queue accounting and drain reports, resampler. + +## Follow-ups + +- Answer permission prompts and questions by voice + (`respond_permission`, `answer_questions` as voice tools, gated by an + explicit confirmation step). +- `stop_conversation` (`request_stop`) and `show_conversation` (open it in + the UI). +- Dictation into the normal composer, reusing the `audio` crate. +- Gemini Live transport. +- Choosing input and output devices. From 45b003af30bf99297999840f134ec8d424c349e3 Mon Sep 17 00:00:00 2001 From: Daniel Kurzynski Date: Thu, 8 Oct 2026 19:00:53 +0200 Subject: [PATCH 04/14] fix(core): give sessions created in a project the default model SessionService::create_session created a session bound to a project without a model configuration, so the first agent run failed with "Session has no model configuration". Sessions without a project already got the default model; project sessions now do too. --- .../src/session/service.rs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/code_assistant_core/src/session/service.rs b/crates/code_assistant_core/src/session/service.rs index 36a714c7..f023035a 100644 --- a/crates/code_assistant_core/src/session/service.rs +++ b/crates/code_assistant_core/src/session/service.rs @@ -453,7 +453,11 @@ impl SessionService { if let Some(path) = manager.resolve_project_path(&project) { config.init_path = Some(path); } - manager.create_session_with_config(name, Some(config), None) + // Like a session without a project, it starts on the + // default model: without one, no agent can run in it. + let model_config = + SessionModelConfig::new(manager.default_model_name().to_string()); + manager.create_session_with_config(name, Some(config), Some(model_config)) } else { manager.create_session(name) } @@ -2587,6 +2591,20 @@ mod tests { assert!(service.list_sessions().await.unwrap().is_empty()); } + #[tokio::test] + async fn create_session_in_a_project_gets_the_default_model() { + let tmp = tempfile::tempdir().unwrap(); + let (service, manager) = test_service_with_manager(tmp.path()); + + let id = service + .create_session(None, Some("project".to_string())) + .await + .unwrap(); + + let model_config = manager.lock().await.get_session_model_config(&id).unwrap(); + assert_eq!(model_config.unwrap().model_name, "test-model"); + } + #[tokio::test(flavor = "multi_thread")] async fn start_fresh_session_keeps_the_history_and_copies_the_settings() { use crate::session::{TurnDispatch, TurnRequest}; From 129a62a2a408ec3fccdee818d8b8f223006b3697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Fri, 9 Oct 2026 06:22:00 +0200 Subject: [PATCH 05/14] fix(voice): stop and time out a connection attempt that hangs VoiceService waited for the agent task on Stop, but the task does not read its control channel while it connects or backs off between reconnects, and no connection attempt had a timeout. On a network that swallows the handshake, voice mode stayed on "Connecting" and the worker hung, with later Start, Stop and Mute commands queued behind it. Every connection attempt now gives up after 20 seconds, and Stop aborts an agent that does not end within a second. --- crates/code_assistant_core/src/voice/agent.rs | 18 +++- .../code_assistant_core/src/voice/service.rs | 25 ++++- crates/code_assistant_core/src/voice/tests.rs | 102 +++++++++++++++--- 3 files changed, 125 insertions(+), 20 deletions(-) diff --git a/crates/code_assistant_core/src/voice/agent.rs b/crates/code_assistant_core/src/voice/agent.rs index 625cf983..0b96749d 100644 --- a/crates/code_assistant_core/src/voice/agent.rs +++ b/crates/code_assistant_core/src/voice/agent.rs @@ -34,6 +34,9 @@ pub(super) enum Control { } const RECONNECT_ATTEMPTS: u32 = 3; +/// Bounds one connection attempt, including the credentials and endpoint +/// lookups a connector does first. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(20); /// Transcript entries replayed into a renewed realtime session. const RESEED_ENTRIES: usize = 30; @@ -101,7 +104,7 @@ impl VoiceAgent { NotificationSource::new(service.clone(), config.notify, open_requests.clone()).await?; let tools = VoiceTools::new(service, open_requests); - let connection = connector.connect().await?; + let connection = connect(connector.as_ref()).await?; let (audio_tx, audio_rx) = mpsc::unbounded_channel(); let audio = audio_factory(audio_tx).context("Failed to open the audio devices")?; let (tool_tx, tool_rx) = mpsc::unbounded_channel(); @@ -361,7 +364,7 @@ impl VoiceAgent { if attempt > 0 { tokio::time::sleep(Duration::from_secs(1 << attempt)).await; } - match self.connector.connect().await { + match connect(self.connector.as_ref()).await { Ok(connection) => { self.connection = connection; self.generation += 1; @@ -456,6 +459,17 @@ impl VoiceAgent { } } +async fn connect(connector: &dyn RealtimeConnector) -> Result { + tokio::time::timeout(CONNECT_TIMEOUT, connector.connect()) + .await + .map_err(|_| { + anyhow!( + "No realtime connection after {}s", + CONNECT_TIMEOUT.as_secs() + ) + })? +} + fn samples_to_ms(samples: u64) -> u64 { samples * 1000 / SAMPLE_RATE as u64 } diff --git a/crates/code_assistant_core/src/voice/service.rs b/crates/code_assistant_core/src/voice/service.rs index a6797aa6..8a3ea445 100644 --- a/crates/code_assistant_core/src/voice/service.rs +++ b/crates/code_assistant_core/src/voice/service.rs @@ -10,6 +10,7 @@ use anyhow::Result; use llm::realtime::RealtimeConnector; use std::future::Future; use std::sync::Arc; +use std::time::Duration; use tokio::sync::mpsc; use tokio::task::JoinHandle; use tracing::{error, info}; @@ -80,8 +81,7 @@ impl VoiceService { } Command::Stop => { if let Some(r) = running.take() { - let _ = r.control.send(Control::Stop); - let _ = r.task.await; + r.stop().await; } publish(&events, VoiceStatus::default()); } @@ -93,8 +93,7 @@ impl VoiceService { } } if let Some(r) = running { - let _ = r.control.send(Control::Stop); - let _ = r.task.await; + r.stop().await; } }; (Self { tx }, worker) @@ -118,6 +117,24 @@ struct Running { task: JoinHandle<()>, } +/// How long a running agent gets to wind down on its own. +const STOP_GRACE: Duration = Duration::from_secs(1); + +impl Running { + /// Ask the agent to stop. One that does not get to its control channel + /// in time (a connection attempt or a reconnect backoff) is aborted. + async fn stop(mut self) { + let _ = self.control.send(Control::Stop); + if tokio::time::timeout(STOP_GRACE, &mut self.task) + .await + .is_err() + { + self.task.abort(); + let _ = self.task.await; + } + } +} + fn start( config: VoiceConfig, sessions: &SessionService, diff --git a/crates/code_assistant_core/src/voice/tests.rs b/crates/code_assistant_core/src/voice/tests.rs index 8dc659b5..eaabc814 100644 --- a/crates/code_assistant_core/src/voice/tests.rs +++ b/crates/code_assistant_core/src/voice/tests.rs @@ -137,10 +137,44 @@ struct Harness { _dir: tempfile::TempDir, } -/// A running voice session; every agent run in the session service answers -/// "done". -async fn harness() -> Harness { - let dir = tempfile::tempdir().unwrap(); +/// Never answers, like a connection attempt into a blackholed network. +struct HangingConnector; + +#[async_trait::async_trait] +impl RealtimeConnector for HangingConnector { + async fn connect(&self) -> anyhow::Result { + std::future::pending().await + } +} + +fn test_config() -> VoiceConfig { + VoiceConfig { + provider: "test".into(), + cooling_ms: 50, + ..VoiceConfig::default() + } +} + +/// A voice service whose connection attempts never finish, with a +/// subscription to its status events. +fn hanging_voice( + dir: &tempfile::TempDir, +) -> (VoiceService, crate::session::event_stream::Subscription) { + let (sessions, events) = session_service(dir); + let statuses = sessions.subscribe(); + let (voice, worker) = VoiceService::with_config_loader( + sessions, + events, + FakeAudio::default().factory(), + Arc::new(|_| Ok(Arc::new(HangingConnector) as Arc)), + Arc::new(test_config), + ); + tokio::spawn(worker); + (voice, statuses) +} + +/// A session service whose agent runs all answer "done". +fn session_service(dir: &tempfile::TempDir) -> (SessionService, EventStream) { let events = EventStream::new(); let manager = Arc::new(tokio::sync::Mutex::new(SessionManager::new( FileSessionPersistence::new_with_root_dir(dir.path().to_path_buf()), @@ -165,7 +199,13 @@ async fn harness() -> Harness { }); let (sessions, worker) = SessionService::new(manager, runtime, events.clone()); tokio::spawn(worker); + (sessions, events) +} +/// A running voice session over [`session_service`]. +async fn harness() -> Harness { + let dir = tempfile::tempdir().unwrap(); + let (sessions, events) = session_service(&dir); let (connection, mut server) = scripted_connection(); let connector: Arc = Arc::new(ScriptedConnector { connections: Mutex::new(vec![connection]), @@ -176,11 +216,7 @@ async fn harness() -> Harness { events, audio.factory(), Arc::new(move |_| Ok(connector.clone())), - Arc::new(|| VoiceConfig { - provider: "test".into(), - cooling_ms: 50, - ..VoiceConfig::default() - }), + Arc::new(test_config), ); tokio::spawn(voice_worker); voice.start(); @@ -417,20 +453,58 @@ async fn status_events_follow_the_floor() { let h = harness().await; let mut events = h.sessions.subscribe(); h.server.send(response_created()); - let status = tokio::time::timeout(WAIT, async { + let status = + wait_for_status(&mut events, WAIT, |s| s.activity == VoiceActivity::Speaking).await; + assert!(!status.muted); + h.voice.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn stop_ends_a_connection_attempt_that_hangs() { + let dir = tempfile::tempdir().unwrap(); + let (voice, mut statuses) = hanging_voice(&dir); + voice.start(); + wait_for_status(&mut statuses, WAIT, |s| { + s.activity == VoiceActivity::Connecting + }) + .await; + voice.stop(); + wait_for_status(&mut statuses, WAIT, |s| s.activity == VoiceActivity::Off).await; +} + +async fn wait_for_status( + events: &mut crate::session::event_stream::Subscription, + within: Duration, + pred: impl Fn(&VoiceStatus) -> bool, +) -> VoiceStatus { + tokio::time::timeout(within, async { loop { if let Ok(event) = events.recv().await && let crate::session::event_stream::EventPayload::Ui(UiEvent::VoiceStatusChanged { status, }) = event.payload - && status.activity == VoiceActivity::Speaking + && pred(&status) { return status; } } }) .await - .expect("speaking status"); - assert!(!status.muted); - h.voice.stop(); + .expect("voice status") +} + +#[tokio::test(start_paused = true)] +async fn a_connection_attempt_that_hangs_times_out() { + let dir = tempfile::tempdir().unwrap(); + let (voice, mut statuses) = hanging_voice(&dir); + voice.start(); + // Paused time runs ahead to the connect timeout. + let status = wait_for_status(&mut statuses, Duration::from_secs(60), |s| { + matches!(s.activity, VoiceActivity::Failed(_)) + }) + .await; + let VoiceActivity::Failed(message) = status.activity else { + unreachable!() + }; + assert!(message.contains("No realtime connection"), "{message}"); } From 1f595ab99137130344ee8c1b87f032c0f577e9ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Fri, 9 Oct 2026 06:23:02 +0200 Subject: [PATCH 06/14] fix(voice): drop audio of a response the user interrupted Late audio deltas were dropped only for the item whose playback the barge-in stopped. A response the user interrupted before its first audio arrived, for example a notification whose response.create was in flight when the user started to speak, had no such item: its audio played over the user, and nothing truncated it. Model audio now plays only while the model has the floor. --- crates/code_assistant_core/src/voice/agent.rs | 10 +++---- crates/code_assistant_core/src/voice/floor.rs | 7 +++++ crates/code_assistant_core/src/voice/tests.rs | 26 +++++++++++++++++++ docs/voice-mode.md | 2 +- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/crates/code_assistant_core/src/voice/agent.rs b/crates/code_assistant_core/src/voice/agent.rs index 0b96749d..1d9370e9 100644 --- a/crates/code_assistant_core/src/voice/agent.rs +++ b/crates/code_assistant_core/src/voice/agent.rs @@ -82,8 +82,6 @@ pub(super) struct VoiceAgent { /// The assistant item whose audio plays, with the sample offset its /// audio starts at. audio_item: Option<(String, u64)>, - /// The item playback was stopped for; its late audio deltas are dropped. - discarded_item: Option, } impl VoiceAgent { @@ -136,7 +134,6 @@ impl VoiceAgent { transcript: Vec::new(), queued_samples: 0, audio_item: None, - discarded_item: None, }; agent.send(ClientEvent::SessionUpdate(agent.session_settings())); Ok(agent) @@ -247,7 +244,9 @@ impl VoiceAgent { } fn on_audio_delta(&mut self, item_id: String, delta: &str) { - if self.discarded_item.as_deref() == Some(item_id.as_str()) { + // Audio of a response the user interrupted, generated before the + // server got our cancel (invariant 5). + if !self.floor.plays_audio() { return; } let samples = match decode_pcm16(delta) { @@ -332,11 +331,10 @@ impl VoiceAgent { if played < self.queued_samples { let heard = played.saturating_sub(start); self.send(ClientEvent::TruncateItem { - item_id: item_id.clone(), + item_id, audio_end_ms: samples_to_ms(heard), }); } - self.discarded_item = Some(item_id); } self.audio.clear_playback(); self.queued_samples = played; diff --git a/crates/code_assistant_core/src/voice/floor.rs b/crates/code_assistant_core/src/voice/floor.rs index 018eaa8e..e942a3e8 100644 --- a/crates/code_assistant_core/src/voice/floor.rs +++ b/crates/code_assistant_core/src/voice/floor.rs @@ -172,6 +172,13 @@ impl Floor { self.state } + /// Whether model audio goes to the speakers. Only while the model has + /// the floor: after a barge-in, audio the server generated before it + /// got the cancel is dropped. + pub fn plays_audio(&self) -> bool { + self.state == FloorState::Speaking + } + pub fn queued_notifications(&self) -> usize { self.queue.len() } diff --git a/crates/code_assistant_core/src/voice/tests.rs b/crates/code_assistant_core/src/voice/tests.rs index eaabc814..b19d9a91 100644 --- a/crates/code_assistant_core/src/voice/tests.rs +++ b/crates/code_assistant_core/src/voice/tests.rs @@ -448,6 +448,32 @@ async fn barge_in_cancels_and_truncates_to_what_was_heard() { h.voice.stop(); } +#[tokio::test(flavor = "multi_thread")] +async fn a_response_cancelled_before_its_audio_stays_silent() { + let mut h = harness().await; + let id = h.sessions.create_session(None, None).await.unwrap(); + h.sessions + .send_user_message(id.clone(), "go".into(), Vec::new(), None) + .await + .unwrap(); + wait_idle(&h.sessions, &id).await; + // The finish is announced on the free floor... + h.server + .expect("response.create", |e| *e == ClientEvent::CreateResponse) + .await; + // ...and the user speaks up before that response exists. + h.server.send(ServerEvent::SpeechStarted); + h.server.send(response_created()); + h.server + .expect("response.cancel", |e| *e == ClientEvent::CancelResponse) + .await; + // Audio generated before the server handled the cancel. + h.server.send(audio_delta("notification", 2_400)); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(h.audio.0.lock().unwrap().queued, 0, "played over the user"); + h.voice.stop(); +} + #[tokio::test(flavor = "multi_thread")] async fn status_events_follow_the_floor() { let h = harness().await; diff --git a/docs/voice-mode.md b/docs/voice-mode.md index df028db3..1c5bd0a1 100644 --- a/docs/voice-mode.md +++ b/docs/voice-mode.md @@ -217,7 +217,7 @@ sink. | server `response.output_audio.delta` | (agent) queue for the speakers; remember the item and the sample offset its audio starts at | | server `response.output_item.done` (function call) | (agent) spawn the tool → `ToolStarted` | | server `response.done` / cancelled | attach held tool outputs; in `Speaking`, wait for the drain (playback timer: pending audio + 5 s), or act as drained if nothing is pending | -| server `input_audio_buffer.speech_started` (barge-in) | cancel timers; if `Speaking`: `response.cancel` (deferred until `response.created` when our create is still in flight) and `StopPlayback` (agent: clear the queue, `conversation.item.truncate` to the samples played, drop late deltas of that item) → `UserTurn` | +| server `input_audio_buffer.speech_started` (barge-in) | cancel timers; if `Speaking`: `response.cancel` (deferred until `response.created` when our create is still in flight) and `StopPlayback` (agent: clear the queue, `conversation.item.truncate` to the samples played). Audio deltas play only in `Speaking`, so late audio of the cancelled response is dropped → `UserTurn` | | server `input_audio_buffer.speech_stopped` | stays `UserTurn`; the server's VAD creates the response. If none follows within 8 s the floor is free again | | server `error` | when our `response.create` was in flight: give the floor back (no wedge) | | sink `Drained` (or fallback timer) | if a trigger is pending (tool outputs attached during `Speaking`) → `response.create`; otherwise → `Cooling` + start the cooling timer | From 752c8415ff4f2c59d423fe8ed9eaf776d38f3369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Fri, 9 Oct 2026 06:24:41 +0200 Subject: [PATCH 07/14] fix(voice): answer tool outputs once, and not lose them over the user Three gaps in the floor state machine: - Cancelling our own response on its creation (the user barged in while response.create was in flight) also cancelled the user-turn timer. If no answer to the user followed, the floor stayed with the user and no notification went out until the user spoke again. - A tool output that arrived during the user's turn waited for the model's answer to the user. When the turn ended without one, the output was never answered. - With several function calls in one response, the first output to complete triggered response.create. The model answered on partial results and the next output started another response. Tool outputs now count as unanswered until a response starts. The model answers them once no tool runs any more and the floor is free: after playback, from Cooling or Idle, or when the user's turn ends without a response. --- crates/code_assistant_core/src/voice/floor.rs | 140 ++++++++++++++---- docs/voice-mode.md | 6 +- 2 files changed, 118 insertions(+), 28 deletions(-) diff --git a/crates/code_assistant_core/src/voice/floor.rs b/crates/code_assistant_core/src/voice/floor.rs index e942a3e8..cbfac9ac 100644 --- a/crates/code_assistant_core/src/voice/floor.rs +++ b/crates/code_assistant_core/src/voice/floor.rs @@ -145,8 +145,10 @@ pub struct Floor { /// Tool outputs that completed while a response was open; attached at /// `response.done`. held_outputs: Vec<(String, String)>, - /// Outputs were attached after generation; respond once playback ends. - pending_trigger: bool, + /// Tool outputs were attached that no response has answered yet. The + /// model answers them once every tool is done and the floor is free: + /// after playback, or when the user's turn ends without a response. + unanswered_outputs: bool, tools_running: usize, armed: Option, queue: NotificationQueue, @@ -161,7 +163,7 @@ impl Floor { create_pending: false, cancel_on_create: false, held_outputs: Vec::new(), - pending_trigger: false, + unanswered_outputs: false, tools_running: 0, armed: None, queue: NotificationQueue::default(), @@ -204,11 +206,13 @@ impl Floor { FloorInput::ResponseCreated => { self.create_pending = false; self.response_open = true; - self.cancel_timer(&mut out); if self.cancel_on_create { + // The user's turn and its timer go on. self.cancel_on_create = false; out.push(FloorCommand::CancelResponse); } else { + self.cancel_timer(&mut out); + self.unanswered_outputs = false; self.state = FloorState::Speaking; } } @@ -216,15 +220,10 @@ impl Floor { self.response_open = false; self.create_pending = false; self.cancel_on_create = false; - let attached = !self.held_outputs.is_empty(); for (call_id, output) in self.held_outputs.drain(..) { + self.unanswered_outputs = true; out.push(FloorCommand::SendToolOutput { call_id, output }); } - // Over the user the model folds the outputs into its next - // answer instead (invariant 2). - if attached && self.state != FloorState::UserTurn { - self.pending_trigger = true; - } if self.state == FloorState::Speaking { if pending_playback.is_zero() { self.playback_finished(&mut out); @@ -247,7 +246,6 @@ impl Floor { } out.push(FloorCommand::StopPlayback); } - self.pending_trigger = false; self.state = FloorState::UserTurn; } FloorInput::SpeechStopped => { @@ -276,14 +274,16 @@ impl Floor { if self.response_open { self.held_outputs.push((call_id, output)); } else { + self.unanswered_outputs = true; out.push(FloorCommand::SendToolOutput { call_id, output }); - match self.state { - FloorState::Speaking => self.pending_trigger = true, - FloorState::UserTurn => {} - FloorState::Cooling | FloorState::Idle => { - self.cancel_timer(&mut out); - self.create_response(&mut out); - } + // While the model speaks, the answer waits for the + // drain; over the user, the model folds the outputs + // into its next answer (invariant 2). + if matches!(self.state, FloorState::Cooling | FloorState::Idle) + && self.tools_running == 0 + { + self.cancel_timer(&mut out); + self.answer_outputs(&mut out); } } } @@ -320,8 +320,10 @@ impl Floor { self.flush(&mut out); } Timer::UserTurn if self.state == FloorState::UserTurn => { - self.state = FloorState::Idle; - self.flush(&mut out); + if !self.answer_outputs(&mut out) { + self.state = FloorState::Idle; + self.flush(&mut out); + } } Timer::Cooling | Timer::UserTurn => {} } @@ -336,15 +338,22 @@ impl Floor { fn playback_finished(&mut self, out: &mut Vec) { self.cancel_timer(out); - if self.pending_trigger { - self.pending_trigger = false; - self.create_response(out); - } else { + if !self.answer_outputs(out) { self.state = FloorState::Cooling; self.start_timer(Timer::Cooling, self.config.cooling, out); } } + /// Have the model answer unanswered tool outputs, once all tools of + /// the response are done. Whether it did. + fn answer_outputs(&mut self, out: &mut Vec) -> bool { + if !self.unanswered_outputs || self.tools_running > 0 { + return false; + } + self.create_response(out); + true + } + /// Deliver queued notifications, only from a free floor and not while /// a tool result is about to make the model speak anyway. fn flush(&mut self, out: &mut Vec) { @@ -548,6 +557,20 @@ mod tests { assert_eq!(floor.state(), FloorState::UserTurn); } + #[test] + fn cancelling_our_response_keeps_the_user_turn_timer() { + let mut floor = floor(); + run( + &mut floor, + vec![finished("a"), I::SpeechStarted, I::SpeechStopped], + ); + assert_eq!(floor.handle(I::ResponseCreated), vec![C::CancelResponse]); + floor.handle(done(0)); + // No answer to the user follows: the floor frees up after all. + floor.handle(I::TimerFired(Timer::UserTurn)); + assert_eq!(floor.state(), FloorState::Idle); + } + #[test] fn tool_call_round_trip_triggers_a_follow_up_response() { let mut floor = floor(); @@ -628,7 +651,74 @@ mod tests { } #[test] - fn barge_in_drops_a_pending_trigger() { + fn a_tool_output_over_the_user_is_answered_when_the_user_says_nothing() { + let mut floor = floor(); + run( + &mut floor, + vec![ + I::ResponseCreated, + I::ToolStarted, + done(0), + I::SpeechStarted, + completed("c1"), + I::SpeechStopped, + ], + ); + let out = floor.handle(I::TimerFired(Timer::UserTurn)); + assert!(creates(&out), "the result is still unanswered: {out:?}"); + } + + #[test] + fn a_response_to_the_user_answers_a_tool_output_too() { + let mut floor = floor(); + run( + &mut floor, + vec![ + I::ResponseCreated, + I::ToolStarted, + done(0), + I::SpeechStarted, + completed("c1"), + I::SpeechStopped, + I::ResponseCreated, + ], + ); + let out = run(&mut floor, vec![done(0), I::TimerFired(Timer::Cooling)]); + assert!(!creates(&out)); + assert_eq!(floor.state(), FloorState::Idle); + } + + #[test] + fn several_tool_calls_get_one_answer_once_all_are_done() { + let mut floor = floor(); + run( + &mut floor, + vec![I::ResponseCreated, I::ToolStarted, I::ToolStarted, done(0)], + ); + let out = floor.handle(completed("c1")); + assert!(!creates(&out), "c2 still runs: {out:?}"); + assert!(creates(&floor.handle(completed("c2")))); + } + + #[test] + fn several_tool_outputs_while_audio_plays_wait_for_the_last_one() { + let mut floor = floor(); + run( + &mut floor, + vec![ + I::ResponseCreated, + I::ToolStarted, + I::ToolStarted, + done(1200), + completed("c1"), + ], + ); + assert!(!creates(&floor.handle(I::PlaybackDrained))); + assert!(creates(&floor.handle(completed("c2")))); + } + + #[test] + fn barge_in_leaves_a_pending_answer_to_the_user_turn() { let mut floor = floor(); run( &mut floor, diff --git a/docs/voice-mode.md b/docs/voice-mode.md index 1c5bd0a1..5b60e0b5 100644 --- a/docs/voice-mode.md +++ b/docs/voice-mode.md @@ -218,11 +218,11 @@ sink. | server `response.output_item.done` (function call) | (agent) spawn the tool → `ToolStarted` | | server `response.done` / cancelled | attach held tool outputs; in `Speaking`, wait for the drain (playback timer: pending audio + 5 s), or act as drained if nothing is pending | | server `input_audio_buffer.speech_started` (barge-in) | cancel timers; if `Speaking`: `response.cancel` (deferred until `response.created` when our create is still in flight) and `StopPlayback` (agent: clear the queue, `conversation.item.truncate` to the samples played). Audio deltas play only in `Speaking`, so late audio of the cancelled response is dropped → `UserTurn` | -| server `input_audio_buffer.speech_stopped` | stays `UserTurn`; the server's VAD creates the response. If none follows within 8 s the floor is free again | +| server `input_audio_buffer.speech_stopped` | stays `UserTurn`; the server's VAD creates the response. If none follows within 8 s, the model answers unanswered tool outputs, or else the floor is free again | | server `error` | when our `response.create` was in flight: give the floor back (no wedge) | -| sink `Drained` (or fallback timer) | if a trigger is pending (tool outputs attached during `Speaking`) → `response.create`; otherwise → `Cooling` + start the cooling timer | +| sink `Drained` (or fallback timer) | if tool outputs are unanswered and no tool runs any more → `response.create`; otherwise → `Cooling` + start the cooling timer | | cooling timer fires | `Cooling` → `Idle`; flush notifications if any are queued | -| tool completed | response still open → hold until `response.done`; `Speaking` after generation → attach and set the pending trigger (fired on `Drained`); `UserTurn` → attach **without** `response.create` (the model uses it in its next answer); `Cooling`/`Idle` → attach + `response.create` | +| tool completed | response still open → hold until `response.done`; otherwise attach, and the output counts as unanswered until the next response starts. `Speaking` → answered after `Drained`; `UserTurn` → **no** `response.create` (the model uses it in its next answer, or answers when the user's turn ends without one); `Cooling`/`Idle` → `response.create` once the last running tool is done, so several calls of one response get one answer | | notification arrives | always queue; `Speaking`/`UserTurn` → nothing more; `Cooling` → restart the cooling timer (groups finishes that land close together); `Idle` → flush now | | `get_conversation` read | drop that conversation's queued notification | From 94ade558b38dfc1b387f01f40f85f71c67e0aa13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Fri, 9 Oct 2026 06:26:25 +0200 Subject: [PATCH 08/14] fix(voice): free the floor when the user mutes or a tool never returns Muting dropped the microphone audio but told the floor nothing. Muted mid-sentence, the server never reported the end of speech, so the floor stayed with the user and no notification was delivered, although muted users are meant to get them. Muting now ends the user's turn like the end of speech does. A voice tool that panicked or hung never reported back. The floor counted it as running for the rest of the voice session and held every notification back. Each call now reports an error after a panic or after 30 seconds. --- crates/code_assistant_core/src/voice/agent.rs | 22 ++++++++++++++++++- crates/code_assistant_core/src/voice/floor.rs | 18 ++++++++++++++- crates/code_assistant_core/src/voice/tools.rs | 20 +++++++++-------- docs/voice-mode.md | 6 +++-- 4 files changed, 53 insertions(+), 13 deletions(-) diff --git a/crates/code_assistant_core/src/voice/agent.rs b/crates/code_assistant_core/src/voice/agent.rs index 1d9370e9..c93a5a3b 100644 --- a/crates/code_assistant_core/src/voice/agent.rs +++ b/crates/code_assistant_core/src/voice/agent.rs @@ -37,6 +37,9 @@ const RECONNECT_ATTEMPTS: u32 = 3; /// Bounds one connection attempt, including the credentials and endpoint /// lookups a connector does first. const CONNECT_TIMEOUT: Duration = Duration::from_secs(20); +/// Bounds one voice tool call; all of them return without waiting for a +/// conversation's agent. +const TOOL_TIMEOUT: Duration = Duration::from_secs(30); /// Transcript entries replayed into a renewed realtime session. const RESEED_ENTRIES: usize = 30; @@ -149,6 +152,9 @@ impl VoiceAgent { Some(Control::SetMuted(muted)) => { self.muted = muted; self.audio.set_capture_muted(muted); + if muted { + self.floor_input(FloorInput::CaptureMuted); + } self.publish_status(); } Some(Control::Stop) | None => return Ok(()), @@ -275,7 +281,21 @@ impl VoiceAgent { let tx = self.tool_tx.clone(); let generation = self.generation; tokio::spawn(async move { - let outcome = tools.call(&name, &arguments).await; + // Every call must report back: the floor holds notifications + // while a tool runs. + let call = tokio::spawn(async move { tools.call(&name, &arguments).await }); + let abort = call.abort_handle(); + let outcome = match tokio::time::timeout(TOOL_TIMEOUT, call).await { + Ok(Ok(outcome)) => outcome, + Ok(Err(e)) => ToolOutcome::error(anyhow!("The tool failed: {e}")), + Err(_) => { + abort.abort(); + ToolOutcome::error(anyhow!( + "The tool did not finish within {}s", + TOOL_TIMEOUT.as_secs() + )) + } + }; let _ = tx.send((generation, call_id, outcome)); }); } diff --git a/crates/code_assistant_core/src/voice/floor.rs b/crates/code_assistant_core/src/voice/floor.rs index cbfac9ac..b10fcd88 100644 --- a/crates/code_assistant_core/src/voice/floor.rs +++ b/crates/code_assistant_core/src/voice/floor.rs @@ -88,6 +88,9 @@ pub enum FloorInput { SpeechStarted, /// The user stopped speaking. SpeechStopped, + /// The microphone was muted: the user cannot be speaking any more, + /// though the server never reports the end of their speech. + CaptureMuted, /// The speakers played everything queued. PlaybackDrained, /// The server reported an error. @@ -248,7 +251,7 @@ impl Floor { } self.state = FloorState::UserTurn; } - FloorInput::SpeechStopped => { + FloorInput::SpeechStopped | FloorInput::CaptureMuted => { if self.state == FloorState::UserTurn { self.start_timer(Timer::UserTurn, self.config.user_turn_grace, &mut out); } @@ -782,6 +785,19 @@ mod tests { assert!(notifies(&out)); } + #[test] + fn muting_mid_sentence_hands_the_floor_back() { + // The server hears no end of speech from a muted microphone. + let mut floor = floor(); + run(&mut floor, vec![I::SpeechStarted, finished("a")]); + let out = floor.handle(I::CaptureMuted); + assert_eq!( + out, + vec![C::StartTimer(Timer::UserTurn, Duration::from_secs(8))] + ); + assert!(notifies(&floor.handle(I::TimerFired(Timer::UserTurn)))); + } + #[test] fn failed_create_does_not_wedge_the_floor() { let mut floor = floor(); diff --git a/crates/code_assistant_core/src/voice/tools.rs b/crates/code_assistant_core/src/voice/tools.rs index 1fd2490b..a04abe93 100644 --- a/crates/code_assistant_core/src/voice/tools.rs +++ b/crates/code_assistant_core/src/voice/tools.rs @@ -42,6 +42,15 @@ pub struct ToolOutcome { pub effects: Vec, } +impl ToolOutcome { + pub fn error(error: anyhow::Error) -> Self { + Self { + output: json!({ "error": format!("{error:#}") }).to_string(), + effects: Vec::new(), + } + } +} + #[derive(Clone)] pub struct VoiceTools { service: SessionService, @@ -122,7 +131,7 @@ impl VoiceTools { } else { match serde_json::from_str(arguments) { Ok(value) => value, - Err(e) => return error_outcome(anyhow!("invalid arguments: {e}")), + Err(e) => return ToolOutcome::error(anyhow!("invalid arguments: {e}")), } }; let result = match name { @@ -132,7 +141,7 @@ impl VoiceTools { "get_conversation" => self.get_conversation(arguments).await, other => Err(anyhow!("unknown tool {other}")), }; - result.unwrap_or_else(error_outcome) + result.unwrap_or_else(ToolOutcome::error) } async fn list_conversations(&self, arguments: Value) -> Result { @@ -316,13 +325,6 @@ impl VoiceTools { } } -fn error_outcome(error: anyhow::Error) -> ToolOutcome { - ToolOutcome { - output: json!({ "error": format!("{error:#}") }).to_string(), - effects: Vec::new(), - } -} - /// One exchange: a user message and the agent's last text before the next /// user message. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/docs/voice-mode.md b/docs/voice-mode.md index 5b60e0b5..6eeac911 100644 --- a/docs/voice-mode.md +++ b/docs/voice-mode.md @@ -260,8 +260,10 @@ session) and to fast ones. period). Tool results of the old connection are dropped. - Stop: the toggle, or an error that cannot be retried. Capture and playback stop, the socket closes, and the queue is dropped. -- Mute: capture is paused, and notifications are still delivered (they only - speak from `Idle`, which a muted user is in). +- Mute: captured audio is dropped (the device stays open). A user turn in + progress ends as if the user had stopped speaking, since the server never + hears the end of it. Notifications are still delivered: they speak from + `Idle`, which a muted user is in. - Voice runs keep no transcript on disk in v1. The live transcript is kept in memory for the UI popover and for reconnect. From fa835c248b6d8c8c8d1e46ae615fd07cd3ed1ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Fri, 9 Oct 2026 06:28:14 +0200 Subject: [PATCH 09/14] perf(core): read session content without blocking the service worker SessionService::session_content ran on the global lane while holding the manager lock, and the file-backed SessionSource loaded sessions with every stored tool result resolved from its blob. A voice get_conversation on a large session held up all frontend commands and the agents' checkpoint commits, for tool results it never reads. session_content now runs as an I/O query on the blocking pool, and the SessionSource loads sessions unresolved: session queries read the messages only, never the execution records, which also speeds up the search_sessions and get_session_content tools. --- .../code_assistant_core/src/session/manager.rs | 16 ++++++---------- .../code_assistant_core/src/session/service.rs | 10 +++++++--- .../code_assistant_core/src/session_query/mod.rs | 4 +++- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/crates/code_assistant_core/src/session/manager.rs b/crates/code_assistant_core/src/session/manager.rs index 3ed04a39..2fdbc4e4 100644 --- a/crates/code_assistant_core/src/session/manager.rs +++ b/crates/code_assistant_core/src/session/manager.rs @@ -1284,7 +1284,7 @@ impl SessionManager { permission_handler.clone(), permissions.clone(), self.tool_registry.clone(), - Some(Arc::new(self.persistence.clone())), + Some(self.session_source()), self.hooks_factory.clone(), ) .with_parent_cancellation(cancellation.clone()) @@ -1323,7 +1323,7 @@ impl SessionManager { .active_sessions .get(session_id) .map(|instance| instance.terminal_interrupts.clone()), - session_source: Some(Arc::new(self.persistence.clone())), + session_source: Some(self.session_source()), hooks_factory: self.hooks_factory.clone(), }; @@ -1526,14 +1526,10 @@ impl SessionManager { self.persistence.list_chat_sessions() } - /// A filtered projection of a stored session, read without loading it - /// into the manager (see [`crate::session_query`]). - pub fn session_content( - &self, - session_id: &str, - projection: &crate::session_query::ContentProjection, - ) -> Result { - crate::session_query::get_session_content(&self.persistence, session_id, projection) + /// The stored sessions, for [`crate::session_query`] reads that do not + /// load a session into the manager. + pub fn session_source(&self) -> Arc { + Arc::new(self.persistence.clone()) } /// The activity state of every loaded session. Sessions not loaded are diff --git a/crates/code_assistant_core/src/session/service.rs b/crates/code_assistant_core/src/session/service.rs index 0dfcc3cf..05b105e5 100644 --- a/crates/code_assistant_core/src/session/service.rs +++ b/crates/code_assistant_core/src/session/service.rs @@ -609,9 +609,13 @@ impl SessionService { session_id: String, projection: crate::session_query::ContentProjection, ) -> Result { - self.call(move |ctx| async move { - let manager = ctx.manager.lock().await; - manager.session_content(&session_id, &projection) + self.call_io(move |ctx| async move { + let source = ctx.manager.lock().await.session_source(); + tokio::task::spawn_blocking(move || { + crate::session_query::get_session_content(source.as_ref(), &session_id, &projection) + }) + .await + .context("Reading the session was aborted")? }) .await } diff --git a/crates/code_assistant_core/src/session_query/mod.rs b/crates/code_assistant_core/src/session_query/mod.rs index 9224497f..f4a5dbf6 100644 --- a/crates/code_assistant_core/src/session_query/mod.rs +++ b/crates/code_assistant_core/src/session_query/mod.rs @@ -70,7 +70,9 @@ impl SessionSource for FileSessionPersistence { } fn load(&self, session_id: &str) -> Result> { - self.load_chat_session(session_id) + // Queries read the messages only; the stored tool results of the + // execution records can stay on disk. + self.load_chat_session_unresolved(session_id) } } From 1cb13680466e127bfff1bfe0324faea1ba20830c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Fri, 9 Oct 2026 06:29:41 +0200 Subject: [PATCH 10/14] fix(gpui): show a voice failure next to the microphone A voice failure went into the error popover, which belongs to the viewed session: any activity change of that session cleared the message, and dismissing it sent ClearError, which reset the session. The title bar now shows a "Voice stopped" chip with the reason in its tooltip until voice mode starts again. --- crates/ui_gpui/src/app/event_loop.rs | 7 ++----- .../ui_gpui/src/main_screen/voice_control.rs | 21 ++++++++++++++++++- docs/voice-mode.md | 3 ++- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/crates/ui_gpui/src/app/event_loop.rs b/crates/ui_gpui/src/app/event_loop.rs index ada90b45..3b8bfa3b 100644 --- a/crates/ui_gpui/src/app/event_loop.rs +++ b/crates/ui_gpui/src/app/event_loop.rs @@ -1114,11 +1114,8 @@ impl Gpui { } UiEvent::VoiceStatusChanged { status } => { - if let code_assistant_core::voice::VoiceActivity::Failed(message) = &status.activity - { - *self.current_error.lock().unwrap() = - Some(format!("Voice mode stopped: {message}")); - } + // A failure shows in the title bar's voice controls, not in + // the error popover: that belongs to the viewed session. self.apply_voice_status(status); cx.refresh(); } diff --git a/crates/ui_gpui/src/main_screen/voice_control.rs b/crates/ui_gpui/src/main_screen/voice_control.rs index 42e53f0a..b6d828da 100644 --- a/crates/ui_gpui/src/main_screen/voice_control.rs +++ b/crates/ui_gpui/src/main_screen/voice_control.rs @@ -17,7 +17,11 @@ impl MainScreen { } let status = gpui.voice_status(); let active = status.activity.is_active(); - let failed = matches!(status.activity, VoiceActivity::Failed(_)); + let failure = match &status.activity { + VoiceActivity::Failed(message) => Some(SharedString::from(message.clone())), + _ => None, + }; + let failed = failure.is_some(); // Without a voice model there is nothing to start; a running voice // session stays controllable either way. if !gpui.voice_configured() && !active && !failed { @@ -62,6 +66,21 @@ impl MainScreen { })); let mut row = div().flex().items_center().gap_1().mr_1(); + if let Some(message) = failure { + row = row.child( + div() + .id("voice-failure-chip") + .h(px(24.)) + .px_2() + .rounded_md() + .flex() + .items_center() + .text_xs() + .text_color(cx.theme().danger) + .child("Voice stopped") + .tooltip(move |window, cx| Tooltip::new(message.clone()).build(window, cx)), + ); + } if active { let muted = status.muted; let label = match status.activity { diff --git a/docs/voice-mode.md b/docs/voice-mode.md index 6eeac911..c6d02241 100644 --- a/docs/voice-mode.md +++ b/docs/voice-mode.md @@ -274,7 +274,8 @@ session) and to fast ones. "· N waiting" for queued notifications) and a mute button. Events arrive as app-scoped `UiEvent::VoiceStatusChanged` and `UiEvent::VoiceTranscript` (`session_id: None` on the `EventStream`); `Gpui` mirrors them - (`app/voice.rs`). A failure shows in the error popover. + (`app/voice.rs`). A failure shows as a "Voice stopped" chip next to the + microphone, with the reason in its tooltip, until voice mode starts again. - The status chip opens the transcript popover. - Settings → *Voice*: provider, model, voice, transcription model, VAD eagerness, cooling time, notify scope. From 20ff86ce5cf99d3bd250e6cb1473f30d48df1219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Fri, 9 Oct 2026 06:30:34 +0200 Subject: [PATCH 11/14] fix(audio): never block the output callback on the playback lock The output callback runs on the audio device's real-time thread and took the same mutex that push() held while it extended one VecDeque of samples. The model sends audio faster than real time, so a long answer grew that queue to megabytes, and a reallocation (or the tokio thread being preempted) under the lock made the callback miss its deadline: clicks and dropouts mid-answer. The cpal input callback took the lock too, for is_playing(). push() now converts a chunk before it takes the lock and only moves it into the queue, render() plays one silent buffer instead of waiting when the lock is held, and the playing flag is an atomic. --- crates/audio/src/playback.rs | 98 ++++++++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 22 deletions(-) diff --git a/crates/audio/src/playback.rs b/crates/audio/src/playback.rs index ed867363..4293b37d 100644 --- a/crates/audio/src/playback.rs +++ b/crates/audio/src/playback.rs @@ -3,10 +3,16 @@ use crate::{AudioReport, ReportSink}; use std::collections::VecDeque; use std::sync::Mutex; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +/// The output callback runs on a real-time thread: it never waits for the +/// lock (it plays silence for one buffer instead), and the handle holds the +/// lock only to move a converted chunk in or out, never while it copies or +/// allocates sample data. pub struct Playback { state: Mutex, + /// Something was queued since the last drain report. + playing: AtomicBool, /// Samples handed to the device since the queue was created. played: AtomicU64, sink: ReportSink, @@ -14,15 +20,16 @@ pub struct Playback { #[derive(Default)] struct State { - queue: VecDeque, - /// Something was queued since the last drain report. - playing: bool, + chunks: VecDeque>, + /// Samples of the front chunk already played. + offset: usize, } impl Playback { pub fn new(sink: ReportSink) -> Self { Self { state: Mutex::new(State::default()), + playing: AtomicBool::new(false), played: AtomicU64::new(0), sink, } @@ -32,19 +39,25 @@ impl Playback { if samples.is_empty() { return; } + let chunk: Vec = samples + .iter() + .map(|s| *s as f32 / i16::MAX as f32) + .collect(); let mut state = self.state.lock().unwrap(); - state - .queue - .extend(samples.iter().map(|s| *s as f32 / i16::MAX as f32)); - state.playing = true; + state.chunks.push_back(chunk); + self.playing.store(true, Ordering::Relaxed); } /// Drop everything queued. No drain report follows: the caller stopped /// playback on purpose. pub fn clear(&self) { - let mut state = self.state.lock().unwrap(); - state.queue.clear(); - state.playing = false; + let chunks = { + let mut state = self.state.lock().unwrap(); + state.offset = 0; + self.playing.store(false, Ordering::Relaxed); + std::mem::take(&mut state.chunks) + }; + drop(chunks); } pub fn played(&self) -> u64 { @@ -53,23 +66,36 @@ impl Playback { /// Whether audio is queued or was queued since the last drain. pub fn is_playing(&self) -> bool { - self.state.lock().unwrap().playing + self.playing.load(Ordering::Relaxed) } /// Fill a device buffer (24 kHz mono); silence once the queue is empty. /// Reports the drain when the last queued sample went out. pub fn render(&self, out: &mut [f32]) { - let mut state = self.state.lock().unwrap(); - let available = state.queue.len().min(out.len()); - for (slot, sample) in out.iter_mut().zip(state.queue.drain(..available)) { - *slot = sample; - } - out[available..].fill(0.0); - self.played.fetch_add(available as u64, Ordering::Relaxed); - let drained = state.playing && state.queue.is_empty(); - if drained { - state.playing = false; + let Ok(mut state) = self.state.try_lock() else { + out.fill(0.0); + return; + }; + let mut written = 0; + while written < out.len() { + let offset = state.offset; + let Some(chunk) = state.chunks.front() else { + break; + }; + let n = (chunk.len() - offset).min(out.len() - written); + out[written..written + n].copy_from_slice(&chunk[offset..offset + n]); + written += n; + if offset + n == chunk.len() { + // Freeing a chunk is cheap next to a missed deadline. + state.chunks.pop_front(); + state.offset = 0; + } else { + state.offset = offset + n; + } } + out[written..].fill(0.0); + self.played.fetch_add(written as u64, Ordering::Relaxed); + let drained = state.chunks.is_empty() && self.playing.swap(false, Ordering::Relaxed); drop(state); if drained { (self.sink)(AudioReport::Drained); @@ -116,4 +142,32 @@ mod tests { assert!(reports.lock().unwrap().is_empty()); assert!(!playback.is_playing()); } + + #[test] + fn plays_across_chunks_in_order() { + let (playback, reports) = recording(); + playback.push(&[i16::MAX; 3]); + playback.push(&[0; 2]); + playback.push(&[i16::MAX; 3]); + let mut buffer = [9.0; 4]; + playback.render(&mut buffer); + assert_eq!(buffer, [1.0, 1.0, 1.0, 0.0]); + playback.render(&mut buffer); + assert_eq!(buffer, [0.0, 1.0, 1.0, 1.0]); + assert_eq!(playback.played(), 8); + assert_eq!(*reports.lock().unwrap(), vec![AudioReport::Drained]); + } + + #[test] + fn a_held_lock_costs_one_silent_buffer_not_a_wait() { + let (playback, _) = recording(); + playback.push(&[i16::MAX; 64]); + let held = playback.state.lock().unwrap(); + let mut buffer = [1.0; 64]; + playback.render(&mut buffer); + assert!(buffer.iter().all(|s| *s == 0.0)); + drop(held); + playback.render(&mut buffer); + assert_eq!(playback.played(), 64); + } } From 62e59825d68ef0645f0b80078a3588f8ba286e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Fri, 9 Oct 2026 06:31:25 +0200 Subject: [PATCH 12/14] fix(audio): fail with a clear reason when the microphone is denied On macOS a denied microphone permission does not make the voice processing unit fail: it records silence. Voice mode then connected, showed "Listening" and never heard anything, with nothing pointing to the cause. The macOS backend now checks the AVFoundation authorization status before it opens the devices and fails with a message that names the setting. An undecided permission still lets the system ask on the first capture. --- Cargo.lock | 1 + crates/audio/Cargo.toml | 2 ++ crates/audio/src/macos.rs | 31 +++++++++++++++++++++++++++++++ docs/voice-mode.md | 4 ++++ 4 files changed, 38 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 8a33d5f9..e567285f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -822,6 +822,7 @@ dependencies = [ "anyhow", "coreaudio-rs 0.14.2", "cpal", + "objc2 0.6.4", "tracing", ] diff --git a/crates/audio/Cargo.toml b/crates/audio/Cargo.toml index 627de3a9..0ea11064 100644 --- a/crates/audio/Cargo.toml +++ b/crates/audio/Cargo.toml @@ -10,6 +10,8 @@ tracing = "0.1" # macOS: the voice-processing I/O unit (echo cancellation, as in FaceTime). [target.'cfg(target_os = "macos")'.dependencies] coreaudio-rs = { version = "0.14", default-features = false, features = ["audio_unit", "core_audio"] } +# The microphone permission (AVFoundation). +objc2 = "0.6" # Elsewhere: plain device streams with half-duplex gating. [target.'cfg(not(target_os = "macos"))'.dependencies] diff --git a/crates/audio/src/macos.rs b/crates/audio/src/macos.rs index 8575a106..b61b4334 100644 --- a/crates/audio/src/macos.rs +++ b/crates/audio/src/macos.rs @@ -9,6 +9,8 @@ use anyhow::{Context, Result}; use coreaudio::audio_unit::audio_format::LinearPcmFlags; use coreaudio::audio_unit::render_callback::{self, data}; use coreaudio::audio_unit::{AudioUnit, Element, IOType, SampleFormat, Scope, StreamFormat}; +use objc2::msg_send; +use objc2::runtime::{AnyClass, AnyObject}; use std::sync::Arc; use tracing::debug; @@ -27,12 +29,41 @@ struct DuckingConfiguration { type Args = render_callback::Args>; +#[link(name = "AVFoundation", kind = "framework")] +unsafe extern "C" { + static AVMediaTypeAudio: *const AnyObject; +} + +// AVAuthorizationStatus +const AUTHORIZATION_RESTRICTED: isize = 1; +const AUTHORIZATION_DENIED: isize = 2; + +/// A microphone the user denied delivers silence, not an error. Check up +/// front; an undecided permission lets the system ask on first capture. +fn check_microphone_permission() -> Result<()> { + let Some(device) = AnyClass::get(c"AVCaptureDevice") else { + return Ok(()); + }; + // SAFETY: a class method taking an AVMediaType (an NSString constant + // of the linked framework) and returning an NSInteger. + let status: isize = + unsafe { msg_send![device, authorizationStatusForMediaType: AVMediaTypeAudio] }; + match status { + AUTHORIZATION_DENIED | AUTHORIZATION_RESTRICTED => anyhow::bail!( + "Microphone access is denied; allow it in System Settings → Privacy & Security → \ + Microphone" + ), + _ => Ok(()), + } +} + pub struct Devices { unit: AudioUnit, } impl Devices { pub fn start(shared: Arc) -> Result { + check_microphone_permission()?; let mut unit = AudioUnit::new_uninitialized(IOType::VoiceProcessingIO) .context("No voice-processing audio unit")?; let enable: u32 = 1; diff --git a/docs/voice-mode.md b/docs/voice-mode.md index c6d02241..15174a5f 100644 --- a/docs/voice-mode.md +++ b/docs/voice-mode.md @@ -282,6 +282,10 @@ session) and to fast ones. - macOS bundle: `NSMicrophoneUsageDescription` in `Info.plist` and the `com.apple.security.device.audio-input` entitlement for the hardened runtime. +- A microphone the user denied records silence instead of failing, so on + macOS the `audio` crate checks the permission first: voice mode does not + start and the failure names the setting. An undecided permission lets the + system ask on the first capture. - The voice agent posts into sessions like a user would, so those sessions show the message as a normal user message. Later, a small "via voice" marker on the message could be added. From 3cc437b532ccdcfc7a6927fd60e548694062f4fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Fri, 9 Oct 2026 06:32:32 +0200 Subject: [PATCH 13/14] style(voice): collapse the truncate condition --- crates/code_assistant_core/src/voice/agent.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/code_assistant_core/src/voice/agent.rs b/crates/code_assistant_core/src/voice/agent.rs index c93a5a3b..ec158088 100644 --- a/crates/code_assistant_core/src/voice/agent.rs +++ b/crates/code_assistant_core/src/voice/agent.rs @@ -347,14 +347,14 @@ impl VoiceAgent { /// the rest. fn stop_playback(&mut self) { let played = self.audio.played_samples(); - if let Some((item_id, start)) = self.audio_item.take() { - if played < self.queued_samples { - let heard = played.saturating_sub(start); - self.send(ClientEvent::TruncateItem { - item_id, - audio_end_ms: samples_to_ms(heard), - }); - } + if let Some((item_id, start)) = self.audio_item.take() + && played < self.queued_samples + { + let heard = played.saturating_sub(start); + self.send(ClientEvent::TruncateItem { + item_id, + audio_end_ms: samples_to_ms(heard), + }); } self.audio.clear_playback(); self.queued_samples = played; From 58af8b804fb3a72f4e4813127dfa7c2612b00eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Fri, 9 Oct 2026 07:27:29 +0200 Subject: [PATCH 14/14] test(voice): wait for the agent's device calls instead of racing them The barge-in test asserted that the speakers were cleared as soon as it received the truncate event, but the agent clears them only after it sent that event, on its own task. On a busy CI runner the assertion ran first. The test also slept 50 ms and assumed the audio delta was queued by then. Tests now wait for the fake devices to reach the expected state. A 20 ms delay injected before clear_playback reproduced the CI failure and passes now; 320 runs in eight parallel loops passed. --- crates/code_assistant_core/src/voice/tests.rs | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/crates/code_assistant_core/src/voice/tests.rs b/crates/code_assistant_core/src/voice/tests.rs index b19d9a91..15c72720 100644 --- a/crates/code_assistant_core/src/voice/tests.rs +++ b/crates/code_assistant_core/src/voice/tests.rs @@ -104,6 +104,18 @@ impl FakeAudio { }) } + /// Wait until the voice agent, on its own task, brought the devices + /// into the state `pred` expects. + async fn wait_for(&self, what: &str, pred: impl Fn(&AudioState) -> bool) { + tokio::time::timeout(WAIT, async { + while !pred(&self.0.lock().unwrap()) { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("timed out waiting for {what}")); + } + /// The speakers played this many samples, and drained if that was all. fn play(&self, samples: u64) { let mut state = self.0.lock().unwrap(); @@ -294,6 +306,9 @@ async fn a_finish_while_the_model_speaks_waits_for_playback_and_silence() { // The model is talking: one second of audio is queued. h.server.send(response_created()); h.server.send(audio_delta("speech", 24_000)); + h.audio + .wait_for("the queued audio", |a| a.queued == 24_000) + .await; h.sessions .send_user_message(id.clone(), "go".into(), Vec::new(), None) .await @@ -423,8 +438,10 @@ async fn barge_in_cancels_and_truncates_to_what_was_heard() { let mut h = harness().await; h.server.send(response_created()); h.server.send(audio_delta("speech", 24_000)); + h.audio + .wait_for("the queued audio", |a| a.queued == 24_000) + .await; // Half a second played when the user interrupts. - tokio::time::sleep(Duration::from_millis(50)).await; h.audio.play(12_000); h.server.send(ServerEvent::SpeechStarted); @@ -444,7 +461,10 @@ async fn barge_in_cancels_and_truncates_to_what_was_heard() { audio_end_ms: 500 } ); - assert_eq!(h.audio.0.lock().unwrap().clears, 1); + // The agent clears the speakers right after it sent the truncate. + h.audio + .wait_for("the cleared speakers", |a| a.clears == 1) + .await; h.voice.stop(); }