diff --git a/AGENTS.md b/AGENTS.md index e65e6c2d..470162c0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ This is a Rust-based AI coding agent harness with multiple operational modes. ### Crate Layers ``` -Layer 0 (generic): llm command_executor pty_session fs_explorer sandbox web git terminal terminal_output +Layer 0 (generic): llm command_executor pty_session fs_explorer sandbox web git terminal terminal_output audio Layer 1 (generic): tools_core — tool trait, registry, render, spec, permissions mcp_client — wraps MCP server tools as registry tools Layer 2 (generic): agent_core — agent loop, hook traits, dialect trait, AgentUi trait @@ -74,6 +74,20 @@ Layer 5 (binary): code_assistant — CLI, config, feature-gated frontend through `UserInterface::ask_questions` and `SessionService::answer_questions`; see `docs/ask-question-tool.md` +### Voice Mode +- A global voice agent next to the sessions (`code_assistant_core::voice`): + a realtime speech model (`llm::realtime`, OpenAI Realtime protocol) with + tools only for conversations (list, read, message/create) +- Configured in `/voice.json` (*Settings → Voice*), started from + the GPUI title bar; driven through `VoiceService`, observed via app-scoped + `UiEvent::VoiceStatusChanged` / `VoiceTranscript` +- `voice::floor::Floor` is a pure state machine deciding when background + notifications (a conversation finished / waits) may speak: never over the + user or the model, only after playback drained plus a quiet period +- Audio through the `VoiceAudio` seam; the binary's `voice` feature plugs in + the `audio` crate (macOS voice-processing I/O with echo cancellation, + `cpal` half-duplex elsewhere); see `docs/voice-mode.md` + ### Session Storage - One folder per session below `sessions/`, the ID is its path: `/-` (`persistence::layout`) diff --git a/Cargo.lock b/Cargo.lock index 2417e21e..e567285f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -249,7 +249,7 @@ dependencies = [ "home", "libc", "log", - "mach2", + "mach2 0.5.0", "miow", "parking_lot", "piper", @@ -288,6 +288,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.11.1", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -793,6 +815,17 @@ dependencies = [ "zbus 5.15.0", ] +[[package]] +name = "audio" +version = "0.1.0" +dependencies = [ + "anyhow", + "coreaudio-rs 0.14.2", + "cpal", + "objc2 0.6.4", + "tracing", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -1035,7 +1068,7 @@ dependencies = [ "bitflags 2.11.1", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.13.0", "log", "prettyplease", "proc-macro2", @@ -1370,6 +1403,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -1668,6 +1707,7 @@ version = "0.2.17" dependencies = [ "anyhow", "async-channel", + "audio", "clap", "code_assistant_core", "command_executor", @@ -2045,6 +2085,40 @@ dependencies = [ "libm", ] +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-rs" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58" +dependencies = [ + "bitflags 2.11.1", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4739a805a62757a83e5654fa3faabec0442666b263bb2287d5a8185bfd953" +dependencies = [ + "bindgen", +] + [[package]] name = "cosmic-text" version = "0.19.0" @@ -2069,6 +2143,29 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs 0.11.3", + "dasp_sample", + "jni 0.21.1", + "js-sys", + "libc", + "mach2 0.4.3", + "ndk", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -2213,7 +2310,7 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa", - "phf 0.10.1", + "phf 0.11.3", "smallvec", ] @@ -2339,6 +2436,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.11.0" @@ -4800,7 +4903,7 @@ dependencies = [ "itertools 0.14.0", "libc", "log", - "mach2", + "mach2 0.5.0", "objc", "objc2 0.6.4", "objc2-app-kit 0.3.2", @@ -6041,6 +6144,22 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -6584,6 +6703,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "mach2" version = "0.5.0" @@ -6951,6 +7079,35 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" +dependencies = [ + "bitflags 2.11.1", + "jni-sys 0.3.1", + "log", + "ndk-sys 0.5.0+25.2.9519653", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "ndk-sys" version = "0.6.0+11769913" @@ -7246,6 +7403,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate 3.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "oauth2" version = "4.4.2" @@ -7272,7 +7451,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "chrono", "getrandom 0.2.17", "http 1.4.0", @@ -7368,6 +7547,21 @@ dependencies = [ "objc2-quartz-core 0.3.2", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.11.1", + "libc", + "objc2 0.6.4", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.3.2" @@ -7682,6 +7876,29 @@ dependencies = [ "memchr", ] +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni 0.21.1", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -7998,9 +8215,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" dependencies = [ - "phf_macros 0.10.0", "phf_shared 0.10.0", - "proc-macro-hack", ] [[package]] @@ -8074,20 +8289,6 @@ dependencies = [ "phf_shared 0.13.1", ] -[[package]] -name = "phf_macros" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0" -dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", - "proc-macro-hack", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "phf_macros" version = "0.11.3" @@ -8426,12 +8627,6 @@ dependencies = [ "toml_edit 0.25.11+spec-1.1.0", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.106" @@ -9490,7 +9685,7 @@ checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ "core-foundation 0.10.0", "core-foundation-sys", - "jni", + "jni 0.22.4", "log", "once_cell", "rustls 0.23.40", @@ -9858,7 +10053,7 @@ checksum = "7bd22781911de0ca6debda95f073c8f18bec65d1a94f1fa9573f3102e514cea4" dependencies = [ "ahash", "annotate-snippets", - "base64 0.21.7", + "base64 0.22.1", "encoding_rs_io", "getrandom 0.3.4", "granit-parser", @@ -12885,7 +13080,7 @@ dependencies = [ "libloading", "log", "naga", - "ndk-sys", + "ndk-sys 0.6.0+11769913", "objc2 0.6.4", "objc2-core-foundation", "objc2-foundation 0.3.2", @@ -12999,6 +13194,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.57.0" @@ -13075,6 +13280,16 @@ dependencies = [ "windows-core 0.62.2", ] +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.57.0" @@ -13341,6 +13556,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -13377,6 +13601,21 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -13452,6 +13691,12 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -13470,6 +13715,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -13488,6 +13739,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -13518,6 +13775,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -13536,6 +13799,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -13554,6 +13823,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -13572,6 +13847,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/Cargo.toml b/Cargo.toml index 5877900e..888d9d02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["crates/agent_core", "crates/agent_orchestration", "crates/code_assistant", "crates/code_assistant_core", "crates/ui_acp", "crates/ui_gpui", "crates/ui_terminal", "crates/command_executor", "crates/fs_explorer", "crates/git", "crates/llm", "crates/mcp_client", "crates/mcp_server", "crates/pty_session", "crates/sandbox", "crates/terminal", "crates/terminal_output", "crates/terminal_test_app", "crates/terminal_view", "crates/tools_core", "crates/web"] +members = ["crates/agent_core", "crates/agent_orchestration", "crates/code_assistant", "crates/code_assistant_core", "crates/audio", "crates/ui_acp", "crates/ui_gpui", "crates/ui_terminal", "crates/command_executor", "crates/fs_explorer", "crates/git", "crates/llm", "crates/mcp_client", "crates/mcp_server", "crates/pty_session", "crates/sandbox", "crates/terminal", "crates/terminal_output", "crates/terminal_test_app", "crates/terminal_view", "crates/tools_core", "crates/web"] resolver = "2" diff --git a/crates/audio/Cargo.toml b/crates/audio/Cargo.toml new file mode 100644 index 00000000..0ea11064 --- /dev/null +++ b/crates/audio/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "audio" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1.0" +tracing = "0.1" + +# macOS: the voice-processing I/O unit (echo cancellation, as in FaceTime). +[target.'cfg(target_os = "macos")'.dependencies] +coreaudio-rs = { version = "0.14", default-features = false, features = ["audio_unit", "core_audio"] } +# The microphone permission (AVFoundation). +objc2 = "0.6" + +# Elsewhere: plain device streams with half-duplex gating. +[target.'cfg(not(target_os = "macos"))'.dependencies] +cpal = "0.15" diff --git a/crates/audio/examples/devices.rs b/crates/audio/examples/devices.rs new file mode 100644 index 00000000..0c879be5 --- /dev/null +++ b/crates/audio/examples/devices.rs @@ -0,0 +1,56 @@ +//! Open the voice devices and report what they deliver. +//! +//! `cargo run -p audio --example devices` listens for two seconds without +//! playing anything; `-- --tone` also plays a one-second tone and reports +//! when the speakers drained (`-- --silence` the same with silent samples). + +use audio::{AudioIo, AudioReport, SAMPLE_RATE}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +fn main() -> anyhow::Result<()> { + let tone = std::env::args().any(|a| a == "--tone"); + let silence = std::env::args().any(|a| a == "--silence"); + let chunks = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(Mutex::new(0i16)); + let drained = Arc::new(AtomicUsize::new(0)); + let (c, p, d) = (chunks.clone(), peak.clone(), drained.clone()); + let io = AudioIo::open(Arc::new(move |report| match report { + AudioReport::Captured(samples) => { + c.fetch_add(1, Ordering::Relaxed); + let max = samples + .iter() + .map(|s| s.saturating_abs()) + .max() + .unwrap_or(0); + let mut peak = p.lock().unwrap(); + *peak = (*peak).max(max); + } + AudioReport::Drained => { + d.fetch_add(1, Ordering::Relaxed); + } + AudioReport::Failed(e) => eprintln!("device failed: {e}"), + }))?; + if silence { + io.play(&vec![0; SAMPLE_RATE as usize]); + } + if tone { + let samples: Vec = (0..SAMPLE_RATE) + .map(|i| { + let t = i as f32 / SAMPLE_RATE as f32; + ((t * 440.0 * std::f32::consts::TAU).sin() * 3000.0) as i16 + }) + .collect(); + io.play(&samples); + } + std::thread::sleep(Duration::from_secs(2)); + println!( + "captured chunks: {} (expected ~40), peak level: {}, played samples: {}, drained: {}", + chunks.load(Ordering::Relaxed), + peak.lock().unwrap(), + io.played_samples(), + drained.load(Ordering::Relaxed) + ); + Ok(()) +} diff --git a/crates/audio/src/cpal_backend.rs b/crates/audio/src/cpal_backend.rs new file mode 100644 index 00000000..ec67d2dc --- /dev/null +++ b/crates/audio/src/cpal_backend.rs @@ -0,0 +1,146 @@ +//! Platforms without system echo cancellation: plain `cpal` streams. +//! +//! Without echo cancellation the microphone would hear the model, so it is +//! gated while audio plays and for a short tail after (half duplex). + +use crate::resample::Resampler; +use crate::{AudioReport, SAMPLE_RATE, Shared}; +use anyhow::{Context, Result, anyhow}; +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use cpal::{SampleFormat, Stream, StreamConfig}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +/// How long after playback the microphone stays gated (room echo). +const ECHO_TAIL: Duration = Duration::from_millis(300); + +pub struct Devices { + _input: Stream, + _output: Stream, +} + +impl Devices { + pub fn start(shared: Arc) -> Result { + let host = cpal::default_host(); + let input_device = host.default_input_device().context("No microphone found")?; + let output_device = host.default_output_device().context("No speakers found")?; + let input = build_input(&input_device, shared.clone())?; + let output = build_output(&output_device, shared)?; + input.play()?; + output.play()?; + Ok(Self { + _input: input, + _output: output, + }) + } +} + +fn report_error(shared: &Arc) -> impl FnMut(cpal::StreamError) + Send + 'static { + let shared = shared.clone(); + move |e| (shared.sink)(AudioReport::Failed(e.to_string())) +} + +fn build_input(device: &cpal::Device, shared: Arc) -> Result { + let supported = device.default_input_config()?; + let format = supported.sample_format(); + let config: StreamConfig = supported.into(); + let channels = config.channels as usize; + let mut resampler = Resampler::new(config.sample_rate.0, SAMPLE_RATE); + let mut mono = Vec::new(); + let mut resampled = Vec::new(); + let mut pending = Vec::new(); + let mut gated_until: Option = None; + let errors = report_error(&shared); + + let mut on_samples = move |frames: &mut dyn Iterator| { + mono.clear(); + let mut frame_sum = 0.0; + for (i, sample) in frames.enumerate() { + frame_sum += sample; + if (i + 1) % channels == 0 { + mono.push(frame_sum / channels as f32); + frame_sum = 0.0; + } + } + if shared.playback.is_playing() { + gated_until = Some(Instant::now() + ECHO_TAIL); + } + if gated_until.is_some_and(|until| Instant::now() < until) { + pending.clear(); + return; + } + resampled.clear(); + resampler.process(&mono, &mut resampled); + shared.capture(&mut pending, resampled.iter().copied()); + }; + + let stream = match format { + SampleFormat::F32 => device.build_input_stream( + &config, + move |data: &[f32], _| on_samples(&mut data.iter().copied()), + errors, + None, + )?, + SampleFormat::I16 => device.build_input_stream( + &config, + move |data: &[i16], _| { + on_samples(&mut data.iter().map(|s| *s as f32 / i16::MAX as f32)) + }, + errors, + None, + )?, + other => return Err(anyhow!("Unsupported microphone sample format {other:?}")), + }; + Ok(stream) +} + +fn build_output(device: &cpal::Device, shared: Arc) -> Result { + let supported = device.default_output_config()?; + let format = supported.sample_format(); + let config: StreamConfig = supported.into(); + let channels = config.channels as usize; + let device_rate = config.sample_rate.0; + let mut resampler = Resampler::new(SAMPLE_RATE, device_rate); + let mut source = Vec::new(); + let mut converted: Vec = Vec::new(); + let errors = report_error(&shared); + + // Fill `frames` device frames (at the device rate) from the 24 kHz queue. + let mut fill = move |frames: usize| -> Vec { + while converted.len() < frames { + let need = ((frames - converted.len()) as u64 * SAMPLE_RATE as u64 / device_rate as u64) + .max(1) as usize; + source.resize(need, 0.0); + shared.playback.render(&mut source); + resampler.process(&source, &mut converted); + } + converted.drain(..frames).collect() + }; + + let stream = match format { + SampleFormat::F32 => device.build_output_stream( + &config, + move |data: &mut [f32], _| { + let mono = fill(data.len() / channels); + for (frame, sample) in data.chunks_mut(channels).zip(mono) { + frame.fill(sample); + } + }, + errors, + None, + )?, + SampleFormat::I16 => device.build_output_stream( + &config, + move |data: &mut [i16], _| { + let mono = fill(data.len() / channels); + for (frame, sample) in data.chunks_mut(channels).zip(mono) { + frame.fill((sample * i16::MAX as f32) as i16); + } + }, + errors, + None, + )?, + other => return Err(anyhow!("Unsupported speaker sample format {other:?}")), + }; + Ok(stream) +} diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs new file mode 100644 index 00000000..d35988c6 --- /dev/null +++ b/crates/audio/src/lib.rs @@ -0,0 +1,144 @@ +//! Microphone capture and speaker playback for voice conversations. +//! +//! Audio is PCM16 mono at 24 kHz in both directions, the format of realtime +//! speech models. [`AudioIo::open`] starts the default devices on a thread of +//! their own; the handle queues playback and reports through a callback: +//! captured microphone chunks, the moment the speakers drained, and device +//! failures. +//! +//! On macOS the devices run through the voice-processing I/O unit, whose +//! echo cancellation keeps the speakers out of the microphone, so the user +//! can talk over the model. Elsewhere `cpal` streams are used and the +//! microphone is gated while audio plays (half duplex: no barge-in). + +#[cfg(not(target_os = "macos"))] +mod cpal_backend; +#[cfg(target_os = "macos")] +mod macos; +mod playback; +#[cfg(any(test, not(target_os = "macos")))] +mod resample; + +use anyhow::Result; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc; +use std::thread::JoinHandle; + +pub use playback::Playback; + +/// Sample rate of the audio this crate exchanges. +pub const SAMPLE_RATE: u32 = 24_000; +/// Samples per captured chunk (50 ms). +const CAPTURE_CHUNK: usize = 1_200; + +/// What the devices report. +#[derive(Debug, Clone, PartialEq)] +pub enum AudioReport { + /// Microphone samples. + Captured(Vec), + /// The speakers played everything queued. + Drained, + /// The devices stopped working. + Failed(String), +} + +pub type ReportSink = Arc; + +/// State shared between the handle and the device callbacks. +pub(crate) struct Shared { + pub playback: Playback, + pub muted: AtomicBool, + pub sink: ReportSink, +} + +impl Shared { + /// Collects microphone samples into chunks for the sink. + pub fn capture(&self, pending: &mut Vec, samples: impl Iterator) { + if self.muted.load(Ordering::Relaxed) { + pending.clear(); + return; + } + for sample in samples { + pending.push((sample.clamp(-1.0, 1.0) * i16::MAX as f32) as i16); + if pending.len() >= CAPTURE_CHUNK { + (self.sink)(AudioReport::Captured(std::mem::take(pending))); + } + } + } +} + +/// The open devices. Dropping the handle stops them. +pub struct AudioIo { + shared: Arc, + stop: Option>, + thread: Option>, +} + +impl AudioIo { + /// Open the default input and output devices. + pub fn open(sink: ReportSink) -> Result { + let shared = Arc::new(Shared { + playback: Playback::new(sink.clone()), + muted: AtomicBool::new(false), + sink, + }); + let (stop_tx, stop_rx) = mpsc::channel::<()>(); + let (ready_tx, ready_rx) = mpsc::channel::>(); + let thread_shared = shared.clone(); + // Device handles are not `Send` on every platform: they live and die + // on this thread. + let thread = std::thread::Builder::new() + .name("voice-audio".into()) + .spawn(move || { + #[cfg(target_os = "macos")] + let devices = macos::Devices::start(thread_shared); + #[cfg(not(target_os = "macos"))] + let devices = cpal_backend::Devices::start(thread_shared); + match devices { + Ok(devices) => { + let _ = ready_tx.send(Ok(())); + // Wait for the handle to drop (or the sender to go). + let _ = stop_rx.recv(); + drop(devices); + } + Err(e) => { + let _ = ready_tx.send(Err(e)); + } + } + })?; + ready_rx + .recv() + .map_err(|_| anyhow::anyhow!("The audio thread ended before starting"))??; + Ok(Self { + shared, + stop: Some(stop_tx), + thread: Some(thread), + }) + } + + pub fn play(&self, samples: &[i16]) { + self.shared.playback.push(samples); + } + + pub fn clear_playback(&self) { + self.shared.playback.clear(); + } + + pub fn played_samples(&self) -> u64 { + self.shared.playback.played() + } + + pub fn set_muted(&self, muted: bool) { + self.shared.muted.store(muted, Ordering::Relaxed); + } +} + +impl Drop for AudioIo { + fn drop(&mut self) { + drop(self.stop.take()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} diff --git a/crates/audio/src/macos.rs b/crates/audio/src/macos.rs new file mode 100644 index 00000000..b61b4334 --- /dev/null +++ b/crates/audio/src/macos.rs @@ -0,0 +1,156 @@ +//! macOS: one voice-processing I/O unit for microphone and speakers. +//! +//! The unit cancels the echo of what it plays from what it records, so the +//! model's voice does not reach the realtime server as user speech. It also +//! converts between the device formats and the 24 kHz mono stream we ask for. + +use crate::{SAMPLE_RATE, Shared}; +use anyhow::{Context, Result}; +use coreaudio::audio_unit::audio_format::LinearPcmFlags; +use coreaudio::audio_unit::render_callback::{self, data}; +use coreaudio::audio_unit::{AudioUnit, Element, IOType, SampleFormat, Scope, StreamFormat}; +use objc2::msg_send; +use objc2::runtime::{AnyClass, AnyObject}; +use std::sync::Arc; +use tracing::debug; + +// AudioUnitProperties.h +const PROPERTY_STREAM_FORMAT: u32 = 8; +const PROPERTY_ENABLE_IO: u32 = 2003; +const PROPERTY_OTHER_AUDIO_DUCKING: u32 = 2108; +const DUCKING_LEVEL_MIN: u32 = 10; + +/// `AUVoiceIOOtherAudioDuckingConfiguration` (macOS 14+). +#[repr(C)] +struct DuckingConfiguration { + enable_advanced_ducking: u8, + ducking_level: u32, +} + +type Args = render_callback::Args>; + +#[link(name = "AVFoundation", kind = "framework")] +unsafe extern "C" { + static AVMediaTypeAudio: *const AnyObject; +} + +// AVAuthorizationStatus +const AUTHORIZATION_RESTRICTED: isize = 1; +const AUTHORIZATION_DENIED: isize = 2; + +/// A microphone the user denied delivers silence, not an error. Check up +/// front; an undecided permission lets the system ask on first capture. +fn check_microphone_permission() -> Result<()> { + let Some(device) = AnyClass::get(c"AVCaptureDevice") else { + return Ok(()); + }; + // SAFETY: a class method taking an AVMediaType (an NSString constant + // of the linked framework) and returning an NSInteger. + let status: isize = + unsafe { msg_send![device, authorizationStatusForMediaType: AVMediaTypeAudio] }; + match status { + AUTHORIZATION_DENIED | AUTHORIZATION_RESTRICTED => anyhow::bail!( + "Microphone access is denied; allow it in System Settings → Privacy & Security → \ + Microphone" + ), + _ => Ok(()), + } +} + +pub struct Devices { + unit: AudioUnit, +} + +impl Devices { + pub fn start(shared: Arc) -> Result { + check_microphone_permission()?; + let mut unit = AudioUnit::new_uninitialized(IOType::VoiceProcessingIO) + .context("No voice-processing audio unit")?; + let enable: u32 = 1; + unit.set_property( + PROPERTY_ENABLE_IO, + Scope::Input, + Element::Input, + Some(&enable), + ) + .context("Failed to enable the microphone")?; + unit.set_property( + PROPERTY_ENABLE_IO, + Scope::Output, + Element::Output, + Some(&enable), + ) + .context("Failed to enable the speakers")?; + + let format = StreamFormat { + sample_rate: SAMPLE_RATE as f64, + sample_format: SampleFormat::F32, + flags: LinearPcmFlags::IS_FLOAT + | LinearPcmFlags::IS_PACKED + | LinearPcmFlags::IS_NON_INTERLEAVED, + channels: 1, + } + .to_asbd(); + // What the microphone side hands us, and what we hand the speakers. + unit.set_property( + PROPERTY_STREAM_FORMAT, + Scope::Output, + Element::Input, + Some(&format), + ) + .context("Failed to set the capture format")?; + unit.set_property( + PROPERTY_STREAM_FORMAT, + Scope::Input, + Element::Output, + Some(&format), + ) + .context("Failed to set the playback format")?; + + // Voice processing ducks other apps' audio hard by default; keep the + // user's music audible. Not available before macOS 14. + let ducking = DuckingConfiguration { + enable_advanced_ducking: 1, + ducking_level: DUCKING_LEVEL_MIN, + }; + if let Err(e) = unit.set_property( + PROPERTY_OTHER_AUDIO_DUCKING, + Scope::Global, + Element::Output, + Some(&ducking), + ) { + debug!("Ducking configuration not applied: {e}"); + } + + unit.initialize() + .context("Failed to initialize the voice-processing unit")?; + + let capture = shared.clone(); + let mut pending = Vec::with_capacity(crate::CAPTURE_CHUNK); + unit.set_input_callback(move |args: Args| { + if let Some(channel) = args.data.channels().next() { + capture.capture(&mut pending, channel.iter().copied()); + } + Ok(()) + }) + .context("Failed to install the capture callback")?; + + let render = shared; + unit.set_render_callback(move |mut args: Args| { + if let Some(channel) = args.data.channels_mut().next() { + render.playback.render(channel); + } + Ok(()) + }) + .context("Failed to install the playback callback")?; + + unit.start().context("Failed to start the audio devices")?; + Ok(Self { unit }) + } +} + +impl Drop for Devices { + fn drop(&mut self) { + let _ = self.unit.stop(); + } +} diff --git a/crates/audio/src/playback.rs b/crates/audio/src/playback.rs new file mode 100644 index 00000000..4293b37d --- /dev/null +++ b/crates/audio/src/playback.rs @@ -0,0 +1,173 @@ +//! The playback queue between the handle and the output callback. + +use crate::{AudioReport, ReportSink}; +use std::collections::VecDeque; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +/// The output callback runs on a real-time thread: it never waits for the +/// lock (it plays silence for one buffer instead), and the handle holds the +/// lock only to move a converted chunk in or out, never while it copies or +/// allocates sample data. +pub struct Playback { + state: Mutex, + /// Something was queued since the last drain report. + playing: AtomicBool, + /// Samples handed to the device since the queue was created. + played: AtomicU64, + sink: ReportSink, +} + +#[derive(Default)] +struct State { + chunks: VecDeque>, + /// Samples of the front chunk already played. + offset: usize, +} + +impl Playback { + pub fn new(sink: ReportSink) -> Self { + Self { + state: Mutex::new(State::default()), + playing: AtomicBool::new(false), + played: AtomicU64::new(0), + sink, + } + } + + pub fn push(&self, samples: &[i16]) { + if samples.is_empty() { + return; + } + let chunk: Vec = samples + .iter() + .map(|s| *s as f32 / i16::MAX as f32) + .collect(); + let mut state = self.state.lock().unwrap(); + state.chunks.push_back(chunk); + self.playing.store(true, Ordering::Relaxed); + } + + /// Drop everything queued. No drain report follows: the caller stopped + /// playback on purpose. + pub fn clear(&self) { + let chunks = { + let mut state = self.state.lock().unwrap(); + state.offset = 0; + self.playing.store(false, Ordering::Relaxed); + std::mem::take(&mut state.chunks) + }; + drop(chunks); + } + + pub fn played(&self) -> u64 { + self.played.load(Ordering::Relaxed) + } + + /// Whether audio is queued or was queued since the last drain. + pub fn is_playing(&self) -> bool { + self.playing.load(Ordering::Relaxed) + } + + /// Fill a device buffer (24 kHz mono); silence once the queue is empty. + /// Reports the drain when the last queued sample went out. + pub fn render(&self, out: &mut [f32]) { + let Ok(mut state) = self.state.try_lock() else { + out.fill(0.0); + return; + }; + let mut written = 0; + while written < out.len() { + let offset = state.offset; + let Some(chunk) = state.chunks.front() else { + break; + }; + let n = (chunk.len() - offset).min(out.len() - written); + out[written..written + n].copy_from_slice(&chunk[offset..offset + n]); + written += n; + if offset + n == chunk.len() { + // Freeing a chunk is cheap next to a missed deadline. + state.chunks.pop_front(); + state.offset = 0; + } else { + state.offset = offset + n; + } + } + out[written..].fill(0.0); + self.played.fetch_add(written as u64, Ordering::Relaxed); + let drained = state.chunks.is_empty() && self.playing.swap(false, Ordering::Relaxed); + drop(state); + if drained { + (self.sink)(AudioReport::Drained); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + fn recording() -> (Playback, Arc>>) { + let reports = Arc::new(Mutex::new(Vec::new())); + let sink_reports = reports.clone(); + let playback = Playback::new(Arc::new(move |r| sink_reports.lock().unwrap().push(r))); + (playback, reports) + } + + #[test] + fn counts_played_samples_and_reports_the_drain_once() { + let (playback, reports) = recording(); + playback.push(&[i16::MAX; 300]); + let mut buffer = [0.0; 256]; + playback.render(&mut buffer); + assert_eq!(playback.played(), 256); + assert!(reports.lock().unwrap().is_empty()); + playback.render(&mut buffer); + assert_eq!(playback.played(), 300); + assert!((buffer[43] - 1.0).abs() < 1e-6 && buffer[44] == 0.0); + playback.render(&mut buffer); + assert_eq!(*reports.lock().unwrap(), vec![AudioReport::Drained]); + } + + #[test] + fn clearing_drops_the_queue_without_a_drain_report() { + let (playback, reports) = recording(); + playback.push(&[1; 1000]); + playback.clear(); + let mut buffer = [1.0; 64]; + playback.render(&mut buffer); + assert!(buffer.iter().all(|s| *s == 0.0)); + assert_eq!(playback.played(), 0); + assert!(reports.lock().unwrap().is_empty()); + assert!(!playback.is_playing()); + } + + #[test] + fn plays_across_chunks_in_order() { + let (playback, reports) = recording(); + playback.push(&[i16::MAX; 3]); + playback.push(&[0; 2]); + playback.push(&[i16::MAX; 3]); + let mut buffer = [9.0; 4]; + playback.render(&mut buffer); + assert_eq!(buffer, [1.0, 1.0, 1.0, 0.0]); + playback.render(&mut buffer); + assert_eq!(buffer, [0.0, 1.0, 1.0, 1.0]); + assert_eq!(playback.played(), 8); + assert_eq!(*reports.lock().unwrap(), vec![AudioReport::Drained]); + } + + #[test] + fn a_held_lock_costs_one_silent_buffer_not_a_wait() { + let (playback, _) = recording(); + playback.push(&[i16::MAX; 64]); + let held = playback.state.lock().unwrap(); + let mut buffer = [1.0; 64]; + playback.render(&mut buffer); + assert!(buffer.iter().all(|s| *s == 0.0)); + drop(held); + playback.render(&mut buffer); + assert_eq!(playback.played(), 64); + } +} diff --git a/crates/audio/src/resample.rs b/crates/audio/src/resample.rs new file mode 100644 index 00000000..63372817 --- /dev/null +++ b/crates/audio/src/resample.rs @@ -0,0 +1,65 @@ +//! Linear resampling between device rates and 24 kHz. Plenty for speech. + +/// Streaming linear resampler; keeps its phase across calls. +pub struct Resampler { + /// Input samples per output sample. + step: f64, + /// Position of the next output sample, relative to `last`. + position: f64, + last: f32, +} + +impl Resampler { + pub fn new(from_rate: u32, to_rate: u32) -> Self { + Self { + step: from_rate as f64 / to_rate as f64, + position: 0.0, + last: 0.0, + } + } + + /// Resample a chunk of mono input, appending to `out`. + pub fn process(&mut self, input: &[f32], out: &mut Vec) { + for &sample in input { + // Emit every output position between `last` (0.0) and `sample` + // (1.0). + while self.position <= 1.0 { + let t = self.position as f32; + out.push(self.last + (sample - self.last) * t); + self.position += self.step; + } + self.position -= 1.0; + self.last = sample; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rate_ratio_sets_the_output_length() { + let input = vec![0.5; 4800]; + let mut out = Vec::new(); + Resampler::new(48_000, 24_000).process(&input, &mut out); + assert!((out.len() as i64 - 2400).abs() <= 2, "{}", out.len()); + + let mut up = Vec::new(); + Resampler::new(24_000, 48_000).process(&input, &mut up); + assert!((up.len() as i64 - 9600).abs() <= 2, "{}", up.len()); + } + + #[test] + fn phase_carries_across_chunks() { + let input: Vec = (0..441).map(|i| i as f32).collect(); + let mut whole = Vec::new(); + Resampler::new(44_100, 24_000).process(&input, &mut whole); + let mut chunked = Vec::new(); + let mut resampler = Resampler::new(44_100, 24_000); + for chunk in input.chunks(37) { + resampler.process(chunk, &mut chunked); + } + assert_eq!(whole, chunked); + } +} diff --git a/crates/code_assistant/Cargo.toml b/crates/code_assistant/Cargo.toml index c9e46579..9ea2ddd8 100644 --- a/crates/code_assistant/Cargo.toml +++ b/crates/code_assistant/Cargo.toml @@ -22,6 +22,7 @@ default = [ "mcp-server", "terminal-frontend", "document-conversion", + "voice", ] document-conversion = ["code_assistant_core/document-conversion"] # Each frontend and protocol mode is an optional compile unit; a headless @@ -32,12 +33,15 @@ gpui-frontend = ["dep:ui_gpui", "dep:async-channel"] browser-panel = ["gpui-frontend", "ui_gpui/browser-panel"] mcp-server = ["dep:mcp_server"] terminal-frontend = ["dep:ui_terminal"] +# Voice mode in the GPUI frontend: microphone and speakers via the audio crate. +voice = ["gpui-frontend", "dep:audio"] [dependencies] code_assistant_core = { path = "../code_assistant_core", default-features = false } mcp_server = { path = "../mcp_server", optional = true } ui_acp = { path = "../ui_acp", optional = true } ui_gpui = { path = "../ui_gpui", optional = true } +audio = { path = "../audio", optional = true } # Migration progress from the backend thread to the GPUI startup window async-channel = { version = "2.5.0", optional = true } ui_terminal = { path = "../ui_terminal", optional = true } diff --git a/crates/code_assistant/assets/Entitlements.plist b/crates/code_assistant/assets/Entitlements.plist index f81a8ce3..1d6391a5 100644 --- a/crates/code_assistant/assets/Entitlements.plist +++ b/crates/code_assistant/assets/Entitlements.plist @@ -21,5 +21,9 @@ com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.device.audio-input + diff --git a/crates/code_assistant/assets/Info.plist b/crates/code_assistant/assets/Info.plist index d3ee9fc7..54cce2eb 100644 --- a/crates/code_assistant/assets/Info.plist +++ b/crates/code_assistant/assets/Info.plist @@ -30,5 +30,7 @@ Code Assistant contributors LSApplicationCategoryType public.app-category.developer-tools + NSMicrophoneUsageDescription + Voice mode listens to you so you can talk to your coding conversations. diff --git a/crates/code_assistant/src/app/gpui.rs b/crates/code_assistant/src/app/gpui.rs index c278199d..70d12f8e 100644 --- a/crates/code_assistant/src/app/gpui.rs +++ b/crates/code_assistant/src/app/gpui.rs @@ -63,6 +63,8 @@ pub fn run(config: AgentRunConfig) -> Result<()> { // Create the session command service. The GUI gets the handle; the // worker runs on the backend tokio runtime below. The GUI consumes the // broadcast stream (see ui_gpui's event bridge). + #[cfg(feature = "voice")] + let voice_events = events.clone(); let (service, service_worker) = SessionService::new( multi_session_manager, Arc::new(AgentRuntimeOptions { @@ -78,6 +80,20 @@ pub fn run(config: AgentRunConfig) -> Result<()> { ); gui.set_session_service(service.clone()); + // Voice mode: a global voice agent over all sessions. Its worker runs + // on the backend runtime next to the session service's. + #[cfg(feature = "voice")] + let voice_worker = { + let (voice, worker) = code_assistant_core::voice::VoiceService::new( + service.clone(), + voice_events, + super::voice::audio_factory(), + code_assistant_core::voice::default_connector_factory(), + ); + gui.set_voice_service(voice); + worker + }; + let gui_for_thread = gui.clone(); let task = config.task.clone(); @@ -126,6 +142,8 @@ pub fn run(config: AgentRunConfig) -> Result<()> { tracing::warn!("Startup settlement failed: {e:#}"); } let worker = tokio::spawn(service_worker); + #[cfg(feature = "voice")] + tokio::spawn(voice_worker); // Pre-warm the registry (connects configured MCP servers) in the // background: remote servers take seconds, and the sidebar's diff --git a/crates/code_assistant/src/app/mod.rs b/crates/code_assistant/src/app/mod.rs index b5923ef5..cbd14919 100644 --- a/crates/code_assistant/src/app/mod.rs +++ b/crates/code_assistant/src/app/mod.rs @@ -6,6 +6,8 @@ pub mod gpui; pub mod server; #[cfg(feature = "terminal-frontend")] pub mod terminal; +#[cfg(feature = "voice")] +pub mod voice; pub use code_assistant_core::config::AgentRunConfig; use code_assistant_core::persistence::{FileSessionPersistence, MigrationProgress}; diff --git a/crates/code_assistant/src/app/voice.rs b/crates/code_assistant/src/app/voice.rs new file mode 100644 index 00000000..d87db855 --- /dev/null +++ b/crates/code_assistant/src/app/voice.rs @@ -0,0 +1,39 @@ +//! The audio devices of voice mode, from the `audio` crate. + +use code_assistant_core::voice::{AudioEvent, AudioFactory, VoiceAudio}; +use std::sync::Arc; + +struct Devices(audio::AudioIo); + +impl VoiceAudio for Devices { + fn play(&mut self, samples: Vec) { + self.0.play(&samples); + } + + fn clear_playback(&mut self) { + self.0.clear_playback(); + } + + fn played_samples(&self) -> u64 { + self.0.played_samples() + } + + fn set_capture_muted(&mut self, muted: bool) { + self.0.set_muted(muted); + } +} + +/// Opens the default microphone and speakers for each voice session. +pub fn audio_factory() -> AudioFactory { + Arc::new(|events| { + let sink: audio::ReportSink = Arc::new(move |report| { + let event = match report { + audio::AudioReport::Captured(samples) => AudioEvent::Captured(samples), + audio::AudioReport::Drained => AudioEvent::Drained, + audio::AudioReport::Failed(message) => AudioEvent::Failed(message), + }; + let _ = events.send(event); + }); + Ok(Box::new(Devices(audio::AudioIo::open(sink)?)) as Box) + }) +} diff --git a/crates/code_assistant_core/src/lib.rs b/crates/code_assistant_core/src/lib.rs index 2532f445..f06b66a5 100644 --- a/crates/code_assistant_core/src/lib.rs +++ b/crates/code_assistant_core/src/lib.rs @@ -29,6 +29,7 @@ pub mod types; pub mod ui; pub mod utils; pub mod version; +pub mod voice; // Mock building blocks (LLM provider, UI, project manager, tool fixtures). // Compiled for our own tests and, behind the `test-utils` feature, for diff --git a/crates/code_assistant_core/src/session/manager.rs b/crates/code_assistant_core/src/session/manager.rs index eef7d507..2fdbc4e4 100644 --- a/crates/code_assistant_core/src/session/manager.rs +++ b/crates/code_assistant_core/src/session/manager.rs @@ -1284,7 +1284,7 @@ impl SessionManager { permission_handler.clone(), permissions.clone(), self.tool_registry.clone(), - Some(Arc::new(self.persistence.clone())), + Some(self.session_source()), self.hooks_factory.clone(), ) .with_parent_cancellation(cancellation.clone()) @@ -1323,7 +1323,7 @@ impl SessionManager { .active_sessions .get(session_id) .map(|instance| instance.terminal_interrupts.clone()), - session_source: Some(Arc::new(self.persistence.clone())), + session_source: Some(self.session_source()), hooks_factory: self.hooks_factory.clone(), }; @@ -1526,6 +1526,23 @@ impl SessionManager { self.persistence.list_chat_sessions() } + /// The stored sessions, for [`crate::session_query`] reads that do not + /// load a session into the manager. + pub fn session_source(&self) -> Arc { + Arc::new(self.persistence.clone()) + } + + /// The activity state of every loaded session. Sessions not loaded are + /// idle. + pub fn activity_states( + &self, + ) -> HashMap { + self.active_sessions + .iter() + .map(|(id, instance)| (id.clone(), instance.get_activity_state())) + .collect() + } + /// Delete a session pub fn delete_session(&mut self, session_id: &str) -> Result<()> { // Remove from active sessions diff --git a/crates/code_assistant_core/src/session/service.rs b/crates/code_assistant_core/src/session/service.rs index 33c11217..05b105e5 100644 --- a/crates/code_assistant_core/src/session/service.rs +++ b/crates/code_assistant_core/src/session/service.rs @@ -453,7 +453,11 @@ impl SessionService { if let Some(path) = manager.resolve_project_path(&project) { config.init_path = Some(path); } - manager.create_session_with_config(name, Some(config), None) + // Like a session without a project, it starts on the + // default model: without one, no agent can run in it. + let model_config = + SessionModelConfig::new(manager.default_model_name().to_string()); + manager.create_session_with_config(name, Some(config), Some(model_config)) } else { manager.create_session(name) } @@ -598,6 +602,37 @@ impl SessionService { .await } + /// A filtered projection of a stored session's content, read without + /// making it the active session. + pub async fn session_content( + &self, + session_id: String, + projection: crate::session_query::ContentProjection, + ) -> Result { + self.call_io(move |ctx| async move { + let source = ctx.manager.lock().await.session_source(); + tokio::task::spawn_blocking(move || { + crate::session_query::get_session_content(source.as_ref(), &session_id, &projection) + }) + .await + .context("Reading the session was aborted")? + }) + .await + } + + /// The activity state of every loaded session; sessions absent from + /// the map are idle. + pub async fn session_activity_states( + &self, + ) -> Result> + { + self.call(move |ctx| async move { + let manager = ctx.manager.lock().await; + Ok(manager.activity_states()) + }) + .await + } + /// Incremental session refresh triggered by the file watcher. Compares /// the on-disk state with the in-memory state and emits only the delta /// as [`UiEvent`]s; falls back to a full reload if that fails. @@ -2576,6 +2611,20 @@ mod tests { assert!(service.list_sessions().await.unwrap().is_empty()); } + #[tokio::test] + async fn create_session_in_a_project_gets_the_default_model() { + let tmp = tempfile::tempdir().unwrap(); + let (service, manager) = test_service_with_manager(tmp.path()); + + let id = service + .create_session(None, Some("project".to_string())) + .await + .unwrap(); + + let model_config = manager.lock().await.get_session_model_config(&id).unwrap(); + assert_eq!(model_config.unwrap().model_name, "test-model"); + } + #[tokio::test(flavor = "multi_thread")] async fn start_fresh_session_keeps_the_history_and_copies_the_settings() { use crate::session::{TurnDispatch, TurnRequest}; diff --git a/crates/code_assistant_core/src/session_query/mod.rs b/crates/code_assistant_core/src/session_query/mod.rs index 9224497f..f4a5dbf6 100644 --- a/crates/code_assistant_core/src/session_query/mod.rs +++ b/crates/code_assistant_core/src/session_query/mod.rs @@ -70,7 +70,9 @@ impl SessionSource for FileSessionPersistence { } fn load(&self, session_id: &str) -> Result> { - self.load_chat_session(session_id) + // Queries read the messages only; the stored tool results of the + // execution records can stay on disk. + self.load_chat_session_unresolved(session_id) } } diff --git a/crates/code_assistant_core/src/ui/ui_events.rs b/crates/code_assistant_core/src/ui/ui_events.rs index 1e170b3e..7c87e86f 100644 --- a/crates/code_assistant_core/src/ui/ui_events.rs +++ b/crates/code_assistant_core/src/ui/ui_events.rs @@ -397,4 +397,13 @@ pub enum UiEvent { /// Configuration files (providers.json / models.json) were changed on disk. /// The UI should reload model lists, settings sections, etc. ConfigChanged, + + // === Voice mode (app-scoped) === + /// The voice agent's state changed. + VoiceStatusChanged { status: crate::voice::VoiceStatus }, + /// A line of the voice conversation: something said, a tool call, or a + /// notification delivered to the voice model. + VoiceTranscript { + entry: crate::voice::TranscriptEntry, + }, } diff --git a/crates/code_assistant_core/src/voice/agent.rs b/crates/code_assistant_core/src/voice/agent.rs new file mode 100644 index 00000000..ec158088 --- /dev/null +++ b/crates/code_assistant_core/src/voice/agent.rs @@ -0,0 +1,500 @@ +//! The voice agent's event loop. +//! +//! One task per voice session. It owns the realtime connection, the audio +//! devices and the [`Floor`], and turns everything that happens — server +//! events, audio reports, tool completions, conversation events, timers — +//! into floor inputs, then carries out the floor's commands. + +use super::floor::{Floor, FloorCommand, FloorConfig, FloorInput, FloorState, Timer}; +use super::source::{NotificationSource, OpenRequests}; +use super::tools::{ToolEffect, ToolOutcome, VoiceTools}; +use super::{ + AudioEvent, AudioFactory, TranscriptEntry, TranscriptRole, VoiceActivity, VoiceAudio, + VoiceConfig, VoiceStatus, +}; +use crate::session::SessionService; +use crate::session::event_stream::{EventStream, StreamError, Subscription}; +use crate::ui::UiEvent; +use anyhow::{Context, Result, anyhow}; +use llm::realtime::{ + ClientEvent, Incoming, RealtimeConnection, RealtimeConnector, SAMPLE_RATE, ServerEvent, + SessionSettings, decode_pcm16, +}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc; +use tokio::time::Instant; +use tracing::{debug, info, warn}; + +/// Commands from the [`super::VoiceService`] to a running agent. +#[derive(Debug)] +pub(super) enum Control { + Stop, + SetMuted(bool), +} + +const RECONNECT_ATTEMPTS: u32 = 3; +/// Bounds one connection attempt, including the credentials and endpoint +/// lookups a connector does first. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(20); +/// Bounds one voice tool call; all of them return without waiting for a +/// conversation's agent. +const TOOL_TIMEOUT: Duration = Duration::from_secs(30); +/// Transcript entries replayed into a renewed realtime session. +const RESEED_ENTRIES: usize = 30; + +const INSTRUCTIONS: &str = "\ +You are the voice assistant of a coding-agent app. The user talks to you hands-free about their \ +conversations with coding agents: what is running, what an agent found or changed, what an agent \ +should do next. You cannot read or change code yourself. You act only through your tools: list \ +conversations and projects, read a conversation, and send a message to a conversation (which starts \ +a new one when you give no conversation id). + +Speak briefly: one to three sentences unless the user asks for detail. Refer to conversations by \ +title, never read ids or paths aloud, and summarise agent answers instead of reading code or long \ +lists. When you start or message a conversation, confirm in a few words and move on; the agent \ +works in the background. When you write a message for an agent, phrase it as a clear instruction \ +with all the context the user gave you. If it is unclear which conversation the user means, ask. + +Messages marked [background notification] come from the app, not the user. They tell you that a \ +conversation finished or waits for the user. Mention them briefly when they are useful, after \ +calling get_conversation for results. Answer in the language the user speaks."; + +pub(super) struct VoiceAgent { + config: VoiceConfig, + connector: Arc, + events: EventStream, + tools: VoiceTools, + source: NotificationSource, + subscription: Subscription, + floor: Floor, + connection: RealtimeConnection, + audio: Box, + audio_rx: mpsc::UnboundedReceiver, + tool_tx: mpsc::UnboundedSender<(u64, String, ToolOutcome)>, + tool_rx: mpsc::UnboundedReceiver<(u64, String, ToolOutcome)>, + /// Bumped per realtime connection; tool results of an older one are + /// dropped (their call ids mean nothing to the new session). + generation: u64, + timer: Option<(Timer, Instant)>, + muted: bool, + status: VoiceStatus, + transcript: Vec, + /// Samples handed to the speakers since the devices opened. + queued_samples: u64, + /// The assistant item whose audio plays, with the sample offset its + /// audio starts at. + audio_item: Option<(String, u64)>, +} + +impl VoiceAgent { + /// Open audio and the realtime session. Fails without side effects + /// left running. + pub(super) async fn start( + config: VoiceConfig, + connector: Arc, + service: SessionService, + events: EventStream, + audio_factory: AudioFactory, + ) -> Result { + let open_requests = OpenRequests::default(); + // Subscribe before reading the initial states, so no transition + // falls in between. + let subscription = service.subscribe(); + let source = + NotificationSource::new(service.clone(), config.notify, open_requests.clone()).await?; + let tools = VoiceTools::new(service, open_requests); + + let connection = connect(connector.as_ref()).await?; + let (audio_tx, audio_rx) = mpsc::unbounded_channel(); + let audio = audio_factory(audio_tx).context("Failed to open the audio devices")?; + let (tool_tx, tool_rx) = mpsc::unbounded_channel(); + let floor = Floor::new(FloorConfig { + cooling: config.cooling(), + ..FloorConfig::default() + }); + + let agent = Self { + config, + connector, + events, + tools, + source, + subscription, + floor, + connection, + audio, + audio_rx, + tool_tx, + tool_rx, + generation: 0, + timer: None, + muted: false, + status: VoiceStatus { + activity: VoiceActivity::Connecting, + ..VoiceStatus::default() + }, + transcript: Vec::new(), + queued_samples: 0, + audio_item: None, + }; + agent.send(ClientEvent::SessionUpdate(agent.session_settings())); + Ok(agent) + } + + /// Run until stopped. Returns the error that ended the session, if any. + pub(super) async fn run(mut self, mut control: mpsc::UnboundedReceiver) -> Result<()> { + self.publish_status(); + loop { + let timer = self.timer; + tokio::select! { + command = control.recv() => match command { + Some(Control::SetMuted(muted)) => { + self.muted = muted; + self.audio.set_capture_muted(muted); + if muted { + self.floor_input(FloorInput::CaptureMuted); + } + self.publish_status(); + } + Some(Control::Stop) | None => return Ok(()), + }, + incoming = self.connection.incoming.recv() => match incoming { + Some(Incoming::Event(event)) => self.on_server_event(event), + Some(Incoming::Closed(reason)) => self.reconnect(reason).await?, + None => self.reconnect(None).await?, + }, + audio = self.audio_rx.recv() => match audio { + Some(AudioEvent::Captured(samples)) => { + if !self.muted { + self.send(ClientEvent::AppendAudio(samples)); + } + } + Some(AudioEvent::Drained) => self.floor_input(FloorInput::PlaybackDrained), + Some(AudioEvent::Failed(message)) => { + return Err(anyhow!("Audio device failed: {message}")); + } + None => return Err(anyhow!("The audio devices closed")), + }, + Some((generation, call_id, outcome)) = self.tool_rx.recv() => { + if generation == self.generation { + self.on_tool_completed(call_id, outcome); + } + } + event = self.subscription.recv() => { + let inputs = match event { + Ok(event) => self.source.on_event(&event).await, + Err(StreamError::Lagged { .. }) => self.source.resync().await, + Err(StreamError::Closed) => return Err(anyhow!("The app is shutting down")), + }; + for input in inputs { + self.floor_input(input); + } + } + _ = sleep_until(timer) => { + if let Some((fired, _)) = self.timer.take() { + self.floor_input(FloorInput::TimerFired(fired)); + } + } + } + } + } + + fn on_server_event(&mut self, event: ServerEvent) { + match event { + ServerEvent::SessionCreated | ServerEvent::SessionUpdated => { + if self.status.activity == VoiceActivity::Connecting { + self.status.activity = VoiceActivity::Listening; + self.publish_status(); + } + } + ServerEvent::ResponseCreated { .. } => self.floor_input(FloorInput::ResponseCreated), + ServerEvent::OutputItemAdded { .. } => {} + ServerEvent::AudioDelta { item_id, delta } => self.on_audio_delta(item_id, &delta), + ServerEvent::AudioTranscriptDone { transcript, .. } => { + self.record(TranscriptRole::Assistant, transcript) + } + ServerEvent::InputTranscription { transcript, .. } => { + self.record(TranscriptRole::User, transcript) + } + ServerEvent::SpeechStarted => self.floor_input(FloorInput::SpeechStarted), + ServerEvent::SpeechStopped => self.floor_input(FloorInput::SpeechStopped), + ServerEvent::OutputItemDone { item } => { + if item.kind == "function_call" { + let name = item.name.unwrap_or_default(); + let call_id = item.call_id.unwrap_or_default(); + let arguments = item.arguments.unwrap_or_default(); + self.start_tool(name, call_id, arguments); + } + } + ServerEvent::ResponseDone { response } => { + debug!( + "Realtime response {} done: {:?}", + response.id, response.status + ); + let pending = self.pending_playback(); + self.floor_input(FloorInput::ResponseDone { + pending_playback: pending, + }); + } + ServerEvent::Error { error } => { + warn!( + "Realtime error ({}): {}", + error.code.as_deref().unwrap_or("-"), + error.message + ); + self.floor_input(FloorInput::ServerError); + } + ServerEvent::Other => {} + } + } + + fn on_audio_delta(&mut self, item_id: String, delta: &str) { + // Audio of a response the user interrupted, generated before the + // server got our cancel (invariant 5). + if !self.floor.plays_audio() { + return; + } + let samples = match decode_pcm16(delta) { + Ok(samples) => samples, + Err(e) => { + warn!("Undecodable realtime audio: {e}"); + return; + } + }; + if self + .audio_item + .as_ref() + .is_none_or(|(id, _)| *id != item_id) + { + self.audio_item = Some((item_id, self.queued_samples)); + } + self.queued_samples += samples.len() as u64; + self.audio.play(samples); + } + + fn start_tool(&mut self, name: String, call_id: String, arguments: String) { + info!("Voice tool call {name} {arguments}"); + self.record(TranscriptRole::Tool, format!("{name} {arguments}")); + self.floor_input(FloorInput::ToolStarted); + let tools = self.tools.clone(); + let tx = self.tool_tx.clone(); + let generation = self.generation; + tokio::spawn(async move { + // Every call must report back: the floor holds notifications + // while a tool runs. + let call = tokio::spawn(async move { tools.call(&name, &arguments).await }); + let abort = call.abort_handle(); + let outcome = match tokio::time::timeout(TOOL_TIMEOUT, call).await { + Ok(Ok(outcome)) => outcome, + Ok(Err(e)) => ToolOutcome::error(anyhow!("The tool failed: {e}")), + Err(_) => { + abort.abort(); + ToolOutcome::error(anyhow!( + "The tool did not finish within {}s", + TOOL_TIMEOUT.as_secs() + )) + } + }; + let _ = tx.send((generation, call_id, outcome)); + }); + } + + fn on_tool_completed(&mut self, call_id: String, outcome: ToolOutcome) { + for effect in outcome.effects { + match effect { + ToolEffect::Read(conversation_id) => { + self.floor_input(FloorInput::ConversationRead { conversation_id }) + } + ToolEffect::Touched(conversation_id) => self.source.touch(&conversation_id), + } + } + self.floor_input(FloorInput::ToolCompleted { + call_id, + output: outcome.output, + }); + } + + fn floor_input(&mut self, input: FloorInput) { + let commands = self.floor.handle(input); + self.execute(commands); + self.sync_status(); + } + + fn execute(&mut self, commands: Vec) { + for command in commands { + match command { + FloorCommand::CancelResponse => self.send(ClientEvent::CancelResponse), + FloorCommand::StopPlayback => self.stop_playback(), + FloorCommand::SendToolOutput { call_id, output } => { + self.send(ClientEvent::FunctionCallOutput { call_id, output }) + } + FloorCommand::SendNotification(text) => { + self.record(TranscriptRole::Notification, text.clone()); + self.send(ClientEvent::SystemMessage(text)); + } + FloorCommand::CreateResponse => self.send(ClientEvent::CreateResponse), + FloorCommand::StartTimer(timer, after) => { + self.timer = Some((timer, Instant::now() + after)) + } + FloorCommand::CancelTimer => self.timer = None, + } + } + } + + /// Barge-in: silence the speakers and cut the server's copy of the + /// item to what the user heard, so the model does not believe it said + /// the rest. + fn stop_playback(&mut self) { + let played = self.audio.played_samples(); + if let Some((item_id, start)) = self.audio_item.take() + && played < self.queued_samples + { + let heard = played.saturating_sub(start); + self.send(ClientEvent::TruncateItem { + item_id, + audio_end_ms: samples_to_ms(heard), + }); + } + self.audio.clear_playback(); + self.queued_samples = played; + } + + fn pending_playback(&self) -> Duration { + let pending = self + .queued_samples + .saturating_sub(self.audio.played_samples()); + Duration::from_millis(samples_to_ms(pending)) + } + + /// Replace a closed realtime session with a fresh one that knows the + /// conversation so far. + async fn reconnect(&mut self, reason: Option) -> Result<()> { + info!("Realtime session closed ({reason:?}); reconnecting"); + self.status.activity = VoiceActivity::Connecting; + self.publish_status(); + self.audio.clear_playback(); + self.queued_samples = self.audio.played_samples(); + self.audio_item = None; + + let mut last_error = anyhow!("connection closed: {}", reason.unwrap_or_default()); + for attempt in 0..RECONNECT_ATTEMPTS { + if attempt > 0 { + tokio::time::sleep(Duration::from_secs(1 << attempt)).await; + } + match connect(self.connector.as_ref()).await { + Ok(connection) => { + self.connection = connection; + self.generation += 1; + self.send(ClientEvent::SessionUpdate(self.session_settings())); + if let Some(summary) = self.transcript_summary() { + self.send(ClientEvent::SystemMessage(summary)); + } + let commands = self.floor.reconnected(); + self.execute(commands); + return Ok(()); + } + Err(e) => { + warn!("Realtime reconnect attempt {} failed: {e:#}", attempt + 1); + last_error = e; + } + } + } + Err(last_error.context("The realtime session could not be renewed")) + } + + fn transcript_summary(&self) -> Option { + let start = self.transcript.len().saturating_sub(RESEED_ENTRIES); + let lines: Vec = self.transcript[start..] + .iter() + .filter_map(|entry| match entry.role { + TranscriptRole::User => Some(format!("User: {}", entry.text)), + TranscriptRole::Assistant => Some(format!("You: {}", entry.text)), + TranscriptRole::Tool => Some(format!("(tool call: {})", entry.text)), + TranscriptRole::Notification => None, + }) + .collect(); + (!lines.is_empty()).then(|| { + format!( + "The connection was renewed. The voice conversation so far:\n{}", + lines.join("\n") + ) + }) + } + + fn session_settings(&self) -> SessionSettings { + SessionSettings { + model: self + .connector + .declares_model() + .then(|| self.config.model.clone()), + instructions: INSTRUCTIONS.to_string(), + voice: self.config.voice.clone(), + tools: VoiceTools::definitions(), + turn_detection: self.config.turn_detection(), + transcription_model: self.config.transcription(), + } + } + + fn send(&self, event: ClientEvent) { + // A closed connection shows up on the incoming side. + let _ = self.connection.outgoing.send(event); + } + + fn record(&mut self, role: TranscriptRole, text: String) { + if text.trim().is_empty() { + return; + } + let entry = TranscriptEntry { role, text }; + self.events.publish_app(UiEvent::VoiceTranscript { + entry: entry.clone(), + }); + self.transcript.push(entry); + } + + fn sync_status(&mut self) { + if self.status.activity == VoiceActivity::Connecting { + return; + } + let activity = match self.floor.state() { + FloorState::Speaking => VoiceActivity::Speaking, + FloorState::UserTurn => VoiceActivity::UserSpeaking, + FloorState::Idle | FloorState::Cooling => VoiceActivity::Listening, + }; + let queued = self.floor.queued_notifications(); + if activity != self.status.activity || queued != self.status.queued_notifications { + self.status.activity = activity; + self.status.queued_notifications = queued; + self.publish_status(); + } + } + + fn publish_status(&mut self) { + self.status.muted = self.muted; + self.events.publish_app(UiEvent::VoiceStatusChanged { + status: self.status.clone(), + }); + } +} + +async fn connect(connector: &dyn RealtimeConnector) -> Result { + tokio::time::timeout(CONNECT_TIMEOUT, connector.connect()) + .await + .map_err(|_| { + anyhow!( + "No realtime connection after {}s", + CONNECT_TIMEOUT.as_secs() + ) + })? +} + +fn samples_to_ms(samples: u64) -> u64 { + samples * 1000 / SAMPLE_RATE as u64 +} + +async fn sleep_until(timer: Option<(Timer, Instant)>) { + match timer { + Some((_, deadline)) => tokio::time::sleep_until(deadline).await, + None => std::future::pending().await, + } +} diff --git a/crates/code_assistant_core/src/voice/config.rs b/crates/code_assistant_core/src/voice/config.rs new file mode 100644 index 00000000..25ec7b67 --- /dev/null +++ b/crates/code_assistant_core/src/voice/config.rs @@ -0,0 +1,186 @@ +//! Voice mode configuration, persisted at `/voice.json`. + +use crate::utils::file_utils::atomic_write_json; +use anyhow::{Result, anyhow, bail}; +use llm::provider_config::ConfigurationSystem; +use llm::realtime::RealtimeConnector; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; +use tracing::warn; + +/// Which conversations report to the voice agent. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub enum NotifyScope { + /// Every conversation of this process. + #[default] + All, + /// Only conversations the voice agent created or messaged. + Touched, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct VoiceConfig { + /// Provider id from `providers.json`; empty while unconfigured. + pub provider: String, + /// The realtime model; for an AI Core provider the key of its `models` + /// map whose deployment serves voice mode. + pub model: String, + pub voice: String, + /// WebSocket URL overriding the one derived from the provider's + /// `base_url`, for compatible gateways. + #[serde(skip_serializing_if = "Option::is_none")] + pub url: Option, + /// Model transcribing the user's speech (for the transcript); empty + /// turns transcription off. + pub transcription_model: String, + /// Semantic turn detection eagerness: `low`, `medium`, `high`, `auto`. + pub vad_eagerness: String, + /// Silence after the model spoke before notifications may speak. + pub cooling_ms: u64, + pub notify: NotifyScope, +} + +impl Default for VoiceConfig { + fn default() -> Self { + Self { + provider: String::new(), + model: "gpt-realtime".into(), + voice: "marin".into(), + url: None, + transcription_model: "gpt-4o-mini-transcribe".into(), + vad_eagerness: "auto".into(), + cooling_ms: 1500, + notify: NotifyScope::All, + } + } +} + +impl VoiceConfig { + pub fn path() -> PathBuf { + crate::config_dir::config_dir().join("voice.json") + } + + /// Load the config; a missing or malformed file yields the defaults. + pub fn load() -> Self { + Self::load_from(&Self::path()) + } + + pub fn load_from(path: &Path) -> Self { + let Ok(content) = std::fs::read_to_string(path) else { + return Self::default(); + }; + serde_json::from_str(&content).unwrap_or_else(|e| { + warn!("Failed to parse {}: {e}; using defaults", path.display()); + Self::default() + }) + } + + pub fn save(&self) -> Result<()> { + atomic_write_json(&Self::path(), self) + } + + pub fn is_configured(&self) -> bool { + !self.provider.is_empty() && !self.model.is_empty() + } + + pub fn cooling(&self) -> Duration { + Duration::from_millis(self.cooling_ms) + } + + pub fn turn_detection(&self) -> Value { + let eagerness = if self.vad_eagerness.is_empty() { + "auto" + } else { + &self.vad_eagerness + }; + // The voice agent handles barge-in itself (cancel, truncate), so the + // server only detects turns and answers them. + json!({ + "type": "semantic_vad", + "eagerness": eagerness, + "create_response": true, + "interrupt_response": false, + }) + } + + pub fn transcription(&self) -> Option { + (!self.transcription_model.is_empty()).then(|| self.transcription_model.clone()) + } + + /// The connector for the configured provider in `providers.json`. + pub fn connector(&self) -> Result> { + if !self.is_configured() { + bail!("Voice mode is not configured: choose a provider and model in Settings → Voice"); + } + let providers = ConfigurationSystem::load_providers_config(None)?; + let provider = providers + .get(&self.provider) + .ok_or_else(|| anyhow!("Unknown provider '{}' in voice.json", self.provider))?; + llm::realtime::connector_for_provider(provider, &self.model, self.url.as_deref()) + } +} + +pub use llm::realtime::supports_realtime; + +#[cfg(test)] +mod tests { + use super::*; + use llm::provider_config::ProviderConfig; + + fn provider(kind: &str, config: Value) -> ProviderConfig { + ProviderConfig { + label: "p".into(), + provider: kind.into(), + config, + } + } + + #[test] + fn openai_and_ai_core_providers_serve_realtime() { + assert!(supports_realtime(&provider("openai-responses", json!({})))); + assert!(supports_realtime(&provider("ai-core", json!({})))); + assert!(!supports_realtime(&provider("anthropic", json!({})))); + } + + #[test] + fn ai_core_needs_a_deployment_for_the_voice_model() { + let p = provider( + "ai-core", + json!({ + "client_id": "id", "client_secret": "s", "token_url": "https://t", + "api_base_url": "https://api.example/v2/inference", + "models": {"gpt-realtime": {"deployment": "d1", "api_type": "openai"}} + }), + ); + let connector = llm::realtime::connector_for_provider(&p, "gpt-realtime", None).unwrap(); + assert!(!connector.declares_model()); + let missing = llm::realtime::connector_for_provider(&p, "other", None); + assert!(missing.err().unwrap().to_string().contains("no deployment")); + } + + #[test] + fn non_realtime_providers_are_rejected() { + let p = provider("anthropic", json!({"api_key": "k"})); + assert!(llm::realtime::connector_for_provider(&p, "m", None).is_err()); + } + + #[test] + fn missing_file_yields_defaults_and_partial_files_merge() { + let dir = tempfile::tempdir().unwrap(); + assert_eq!( + VoiceConfig::load_from(&dir.path().join("none.json")), + VoiceConfig::default() + ); + let path = dir.path().join("voice.json"); + std::fs::write(&path, r#"{"provider":"openai","notify":"touched"}"#).unwrap(); + let config = VoiceConfig::load_from(&path); + assert_eq!(config.provider, "openai"); + assert_eq!(config.notify, NotifyScope::Touched); + assert_eq!(config.model, "gpt-realtime"); + } +} diff --git a/crates/code_assistant_core/src/voice/floor.rs b/crates/code_assistant_core/src/voice/floor.rs new file mode 100644 index 00000000..b10fcd88 --- /dev/null +++ b/crates/code_assistant_core/src/voice/floor.rs @@ -0,0 +1,832 @@ +//! Who has the floor in a voice conversation, and when background work may +//! speak up. +//! +//! [`Floor`] is a pure state machine: [`FloorInput`]s go in, [`FloorCommand`]s +//! come out. It does no I/O and has no clock — timers are commands the voice +//! agent runs and reports back as [`FloorInput::TimerFired`]. +//! +//! Two priority levels: +//! +//! 1. **Voice**: the user's speech, the model's answer and the tool calls of +//! that answer. +//! 2. **Background**: notifications about conversations. They are only +//! delivered on a free floor and never cancel or delay level 1. +//! +//! The model generates audio much faster than it plays, so `response.done` +//! does not mean the model stopped talking: the floor is free only once the +//! speakers drained ([`FloorInput::PlaybackDrained`], or the playback timer +//! as a fallback) and then stayed quiet for the cooling time. +//! +//! Invariants (each has a test below): +//! +//! 1. No `response.create` while a response is open. +//! 2. Neither a tool result nor a notification triggers a response while the +//! user has the floor. +//! 3. Notifications flush only from `Idle`: after playback drained *and* the +//! cooling time passed in silence. +//! 4. A notification never cancels a response or stops playback. +//! 5. Barge-in stops playback (the agent truncates the item to what was +//! heard) and cancels the open response. + +use super::notifications::{Notification, NotificationQueue}; +use std::time::Duration; + +#[derive(Debug, Clone)] +pub struct FloorConfig { + /// Silence after playback before background notifications may speak. + pub cooling: Duration, + /// Extra wait beyond the expected playback end before assuming the + /// speakers drained (in case the drain report is lost). + pub playback_grace: Duration, + /// How long after the user stopped speaking the floor stays theirs when + /// no response follows. + pub user_turn_grace: Duration, +} + +impl Default for FloorConfig { + fn default() -> Self { + Self { + cooling: Duration::from_millis(1500), + playback_grace: Duration::from_secs(5), + user_turn_grace: Duration::from_secs(8), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FloorState { + /// Silent, nothing pending. + Idle, + /// A response is being generated or its audio is still playing. + Speaking, + /// The user is speaking, or spoke and is owed an answer. + UserTurn, + /// Playback drained; waiting for the cooling time of silence. + Cooling, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Timer { + /// Fallback for a lost playback drain report. + Playback, + /// Silence after playback. + Cooling, + /// The user stopped speaking but no response followed. + UserTurn, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum FloorInput { + /// The server started a response (ours or one its turn detection made). + ResponseCreated, + /// The response finished generating; `pending_playback` is the audio of + /// it still queued for the speakers. + ResponseDone { + pending_playback: Duration, + }, + /// The user started speaking. + SpeechStarted, + /// The user stopped speaking. + SpeechStopped, + /// The microphone was muted: the user cannot be speaking any more, + /// though the server never reports the end of their speech. + CaptureMuted, + /// The speakers played everything queued. + PlaybackDrained, + /// The server reported an error. + ServerError, + /// A tool call started running. + ToolStarted, + /// A tool call finished. + ToolCompleted { + call_id: String, + output: String, + }, + Notification(Notification), + /// A conversation's wait for the user ended elsewhere. + WaitEnded { + conversation_id: String, + }, + /// The model read a conversation, so its notification is known. + ConversationRead { + conversation_id: String, + }, + TimerFired(Timer), +} + +#[derive(Debug, Clone, PartialEq)] +pub enum FloorCommand { + /// `response.cancel`. + CancelResponse, + /// Drop queued audio and truncate the current item to what was heard. + StopPlayback, + /// `conversation.item.create` with a function call output. + SendToolOutput { + call_id: String, + output: String, + }, + /// `conversation.item.create` with a background system message. + SendNotification(String), + /// `response.create`. + CreateResponse, + /// Arm the (single) timer, replacing an armed one. + StartTimer(Timer, Duration), + CancelTimer, +} + +#[derive(Debug)] +pub struct Floor { + config: FloorConfig, + state: FloorState, + /// A response is being generated (or we asked for one). + response_open: bool, + /// We sent `response.create` and wait for `response.created`. + create_pending: bool, + /// The user barged in while our `response.create` was in flight: cancel + /// that response as soon as it exists. + cancel_on_create: bool, + /// Tool outputs that completed while a response was open; attached at + /// `response.done`. + held_outputs: Vec<(String, String)>, + /// Tool outputs were attached that no response has answered yet. The + /// model answers them once every tool is done and the floor is free: + /// after playback, or when the user's turn ends without a response. + unanswered_outputs: bool, + tools_running: usize, + armed: Option, + queue: NotificationQueue, +} + +impl Floor { + pub fn new(config: FloorConfig) -> Self { + Self { + config, + state: FloorState::Idle, + response_open: false, + create_pending: false, + cancel_on_create: false, + held_outputs: Vec::new(), + unanswered_outputs: false, + tools_running: 0, + armed: None, + queue: NotificationQueue::default(), + } + } + + pub fn state(&self) -> FloorState { + self.state + } + + /// Whether model audio goes to the speakers. Only while the model has + /// the floor: after a barge-in, audio the server generated before it + /// got the cancel is dropped. + pub fn plays_audio(&self) -> bool { + self.state == FloorState::Speaking + } + + pub fn queued_notifications(&self) -> usize { + self.queue.len() + } + + /// The realtime session was replaced by a fresh one: everything tied to + /// the old one (open response, tool calls, held outputs) is gone. Queued + /// notifications stay and go out after a cooling period. + pub fn reconnected(&mut self) -> Vec { + let queue = std::mem::take(&mut self.queue); + *self = Self::new(self.config.clone()); + self.queue = queue; + let mut out = vec![FloorCommand::CancelTimer]; + if !self.queue.is_empty() { + self.state = FloorState::Cooling; + self.start_timer(Timer::Cooling, self.config.cooling, &mut out); + } + out + } + + pub fn handle(&mut self, input: FloorInput) -> Vec { + let mut out = Vec::new(); + match input { + FloorInput::ResponseCreated => { + self.create_pending = false; + self.response_open = true; + if self.cancel_on_create { + // The user's turn and its timer go on. + self.cancel_on_create = false; + out.push(FloorCommand::CancelResponse); + } else { + self.cancel_timer(&mut out); + self.unanswered_outputs = false; + self.state = FloorState::Speaking; + } + } + FloorInput::ResponseDone { pending_playback } => { + self.response_open = false; + self.create_pending = false; + self.cancel_on_create = false; + for (call_id, output) in self.held_outputs.drain(..) { + self.unanswered_outputs = true; + out.push(FloorCommand::SendToolOutput { call_id, output }); + } + if self.state == FloorState::Speaking { + if pending_playback.is_zero() { + self.playback_finished(&mut out); + } else { + self.start_timer( + Timer::Playback, + pending_playback + self.config.playback_grace, + &mut out, + ); + } + } + } + FloorInput::SpeechStarted => { + self.cancel_timer(&mut out); + if self.state == FloorState::Speaking { + if self.create_pending { + self.cancel_on_create = true; + } else if self.response_open { + out.push(FloorCommand::CancelResponse); + } + out.push(FloorCommand::StopPlayback); + } + self.state = FloorState::UserTurn; + } + FloorInput::SpeechStopped | FloorInput::CaptureMuted => { + if self.state == FloorState::UserTurn { + self.start_timer(Timer::UserTurn, self.config.user_turn_grace, &mut out); + } + } + FloorInput::PlaybackDrained => { + if self.waiting_for_playback() { + self.playback_finished(&mut out); + } + } + FloorInput::ServerError => { + if self.create_pending { + self.create_pending = false; + self.response_open = false; + self.cancel_on_create = false; + if self.state == FloorState::Speaking { + self.playback_finished(&mut out); + } + } + } + FloorInput::ToolStarted => self.tools_running += 1, + FloorInput::ToolCompleted { call_id, output } => { + self.tools_running = self.tools_running.saturating_sub(1); + if self.response_open { + self.held_outputs.push((call_id, output)); + } else { + self.unanswered_outputs = true; + out.push(FloorCommand::SendToolOutput { call_id, output }); + // While the model speaks, the answer waits for the + // drain; over the user, the model folds the outputs + // into its next answer (invariant 2). + if matches!(self.state, FloorState::Cooling | FloorState::Idle) + && self.tools_running == 0 + { + self.cancel_timer(&mut out); + self.answer_outputs(&mut out); + } + } + } + FloorInput::Notification(notification) => { + self.queue.push(notification); + match self.state { + // Notifications finishing close together go out as one. + FloorState::Cooling => { + self.start_timer(Timer::Cooling, self.config.cooling, &mut out) + } + FloorState::Idle => self.flush(&mut out), + FloorState::Speaking | FloorState::UserTurn => {} + } + } + FloorInput::WaitEnded { conversation_id } => { + self.queue.retract_waiting(&conversation_id) + } + FloorInput::ConversationRead { conversation_id } => { + self.queue.acknowledge(&conversation_id) + } + FloorInput::TimerFired(timer) => { + if self.armed != Some(timer) { + return out; + } + self.armed = None; + match timer { + Timer::Playback => { + if self.waiting_for_playback() { + self.playback_finished(&mut out); + } + } + Timer::Cooling if self.state == FloorState::Cooling => { + self.state = FloorState::Idle; + self.flush(&mut out); + } + Timer::UserTurn if self.state == FloorState::UserTurn => { + if !self.answer_outputs(&mut out) { + self.state = FloorState::Idle; + self.flush(&mut out); + } + } + Timer::Cooling | Timer::UserTurn => {} + } + } + } + out + } + + fn waiting_for_playback(&self) -> bool { + self.state == FloorState::Speaking && !self.response_open && !self.create_pending + } + + fn playback_finished(&mut self, out: &mut Vec) { + self.cancel_timer(out); + if !self.answer_outputs(out) { + self.state = FloorState::Cooling; + self.start_timer(Timer::Cooling, self.config.cooling, out); + } + } + + /// Have the model answer unanswered tool outputs, once all tools of + /// the response are done. Whether it did. + fn answer_outputs(&mut self, out: &mut Vec) -> bool { + if !self.unanswered_outputs || self.tools_running > 0 { + return false; + } + self.create_response(out); + true + } + + /// Deliver queued notifications, only from a free floor and not while + /// a tool result is about to make the model speak anyway. + fn flush(&mut self, out: &mut Vec) { + if self.state != FloorState::Idle || self.tools_running > 0 { + return; + } + if let Some(message) = self.queue.take_message() { + out.push(FloorCommand::SendNotification(message)); + self.create_response(out); + } + } + + fn create_response(&mut self, out: &mut Vec) { + debug_assert!(!self.response_open, "invariant 1: one response at a time"); + self.create_pending = true; + self.response_open = true; + self.state = FloorState::Speaking; + out.push(FloorCommand::CreateResponse); + } + + fn start_timer(&mut self, timer: Timer, after: Duration, out: &mut Vec) { + self.armed = Some(timer); + out.push(FloorCommand::StartTimer(timer, after)); + } + + fn cancel_timer(&mut self, out: &mut Vec) { + if self.armed.take().is_some() { + out.push(FloorCommand::CancelTimer); + } + } +} + +#[cfg(test)] +mod tests { + use super::super::notifications::NotificationKind; + use super::*; + use FloorCommand as C; + use FloorInput as I; + + fn floor() -> Floor { + Floor::new(FloorConfig::default()) + } + + fn finished(id: &str) -> I { + I::Notification(Notification { + conversation_id: id.into(), + name: id.into(), + project: String::new(), + kind: NotificationKind::Finished, + }) + } + + fn done(ms: u64) -> I { + I::ResponseDone { + pending_playback: Duration::from_millis(ms), + } + } + + fn completed(call: &str) -> I { + I::ToolCompleted { + call_id: call.into(), + output: "out".into(), + } + } + + fn creates(commands: &[C]) -> bool { + commands.contains(&C::CreateResponse) + } + + fn notifies(commands: &[C]) -> bool { + commands.iter().any(|c| matches!(c, C::SendNotification(_))) + } + + /// Run a sequence and return the commands of the last input. + fn run(floor: &mut Floor, inputs: Vec) -> Vec { + let mut last = Vec::new(); + for input in inputs { + last = floor.handle(input); + } + last + } + + /// A model answer that played to the end and cooled down. + fn answered_and_cooled(floor: &mut Floor) { + run( + floor, + vec![ + I::ResponseCreated, + done(800), + I::PlaybackDrained, + I::TimerFired(Timer::Cooling), + ], + ); + assert_eq!(floor.state(), FloorState::Idle); + } + + #[test] + fn idle_floor_flushes_a_notification_at_once() { + let mut floor = floor(); + let out = floor.handle(finished("a")); + assert!(notifies(&out)); + assert!(creates(&out)); + assert_eq!(floor.state(), FloorState::Speaking); + } + + #[test] + fn notification_while_speaking_waits_for_drain_and_cooling() { + // Invariants 3 and 4. + let mut floor = floor(); + floor.handle(I::ResponseCreated); + let out = floor.handle(finished("a")); + assert!(out.is_empty(), "never interrupts the model: {out:?}"); + + let out = floor.handle(done(2000)); + assert_eq!( + out, + vec![C::StartTimer(Timer::Playback, Duration::from_millis(7000))] + ); + // Generation is done, the audio still plays. + assert_eq!(floor.state(), FloorState::Speaking); + + let out = floor.handle(I::PlaybackDrained); + assert!(!notifies(&out)); + assert_eq!(floor.state(), FloorState::Cooling); + assert!(out.contains(&C::StartTimer(Timer::Cooling, Duration::from_millis(1500)))); + + let out = floor.handle(I::TimerFired(Timer::Cooling)); + assert!(notifies(&out) && creates(&out)); + } + + #[test] + fn notification_during_user_turn_never_triggers() { + // Invariant 2. + let mut floor = floor(); + floor.handle(I::SpeechStarted); + assert!(floor.handle(finished("a")).is_empty()); + floor.handle(I::SpeechStopped); + // The server answers the user; the notification waits for that. + assert!(!creates(&floor.handle(I::ResponseCreated))); + assert!(!creates(&floor.handle(done(0)))); + let out = floor.handle(I::TimerFired(Timer::Cooling)); + assert!(notifies(&out)); + } + + #[test] + fn notifications_close_together_go_out_as_one() { + let mut floor = floor(); + run(&mut floor, vec![I::ResponseCreated, done(0)]); + assert_eq!(floor.state(), FloorState::Cooling); + let out = floor.handle(finished("a")); + assert_eq!( + out, + vec![C::StartTimer(Timer::Cooling, Duration::from_millis(1500))], + "restarts the silence timer" + ); + floor.handle(finished("b")); + let out = floor.handle(I::TimerFired(Timer::Cooling)); + let messages: Vec<_> = out + .iter() + .filter_map(|c| match c { + C::SendNotification(m) => Some(m), + _ => None, + }) + .collect(); + assert_eq!(messages.len(), 1); + assert!(messages[0].contains("id a") && messages[0].contains("id b")); + assert_eq!(out.iter().filter(|c| **c == C::CreateResponse).count(), 1); + } + + #[test] + fn barge_in_cancels_the_response_and_stops_playback() { + // Invariant 5. + let mut floor = floor(); + floor.handle(I::ResponseCreated); + let out = floor.handle(I::SpeechStarted); + assert!(out.contains(&C::CancelResponse)); + assert!(out.contains(&C::StopPlayback)); + assert_eq!(floor.state(), FloorState::UserTurn); + } + + #[test] + fn barge_in_after_generation_only_stops_playback() { + let mut floor = floor(); + run(&mut floor, vec![I::ResponseCreated, done(3000)]); + let out = floor.handle(I::SpeechStarted); + assert!(!out.contains(&C::CancelResponse), "nothing to cancel"); + assert!(out.contains(&C::StopPlayback)); + // The playback fallback timer must not fire into the user's turn. + assert!(out.contains(&C::CancelTimer)); + assert!(floor.handle(I::TimerFired(Timer::Playback)).is_empty()); + } + + #[test] + fn barge_in_before_our_response_exists_cancels_it_on_creation() { + let mut floor = floor(); + floor.handle(finished("a")); // flushes, response.create in flight + let out = floor.handle(I::SpeechStarted); + assert!(!out.contains(&C::CancelResponse)); + let out = floor.handle(I::ResponseCreated); + assert_eq!(out, vec![C::CancelResponse]); + assert_eq!(floor.state(), FloorState::UserTurn); + } + + #[test] + fn cancelling_our_response_keeps_the_user_turn_timer() { + let mut floor = floor(); + run( + &mut floor, + vec![finished("a"), I::SpeechStarted, I::SpeechStopped], + ); + assert_eq!(floor.handle(I::ResponseCreated), vec![C::CancelResponse]); + floor.handle(done(0)); + // No answer to the user follows: the floor frees up after all. + floor.handle(I::TimerFired(Timer::UserTurn)); + assert_eq!(floor.state(), FloorState::Idle); + } + + #[test] + fn tool_call_round_trip_triggers_a_follow_up_response() { + let mut floor = floor(); + // The model's response is just a function call; the tool runs after + // it is done. + run( + &mut floor, + vec![I::ResponseCreated, I::ToolStarted, done(0)], + ); + assert_eq!(floor.state(), FloorState::Cooling); + let out = floor.handle(completed("c1")); + assert_eq!( + out, + vec![ + C::SendToolOutput { + call_id: "c1".into(), + output: "out".into() + }, + C::CancelTimer, + C::CreateResponse + ] + ); + } + + #[test] + fn tool_output_during_generation_is_held_until_done() { + // Invariant 1. + let mut floor = floor(); + run(&mut floor, vec![I::ResponseCreated, I::ToolStarted]); + assert!(floor.handle(completed("c1")).is_empty()); + let out = floor.handle(done(0)); + assert_eq!( + out, + vec![ + C::SendToolOutput { + call_id: "c1".into(), + output: "out".into() + }, + C::CreateResponse + ] + ); + } + + #[test] + fn tool_output_while_audio_plays_responds_after_drain() { + let mut floor = floor(); + run( + &mut floor, + vec![I::ResponseCreated, I::ToolStarted, done(1200)], + ); + let out = floor.handle(completed("c1")); + assert!(!creates(&out), "the speakers still play: {out:?}"); + let out = floor.handle(I::PlaybackDrained); + assert!(creates(&out)); + } + + #[test] + fn tool_output_over_the_user_attaches_without_trigger() { + // Invariant 2. + let mut floor = floor(); + run( + &mut floor, + vec![ + I::ResponseCreated, + I::ToolStarted, + done(0), + I::SpeechStarted, + ], + ); + let out = floor.handle(completed("c1")); + assert_eq!( + out, + vec![C::SendToolOutput { + call_id: "c1".into(), + output: "out".into() + }] + ); + } + + #[test] + fn a_tool_output_over_the_user_is_answered_when_the_user_says_nothing() { + let mut floor = floor(); + run( + &mut floor, + vec![ + I::ResponseCreated, + I::ToolStarted, + done(0), + I::SpeechStarted, + completed("c1"), + I::SpeechStopped, + ], + ); + let out = floor.handle(I::TimerFired(Timer::UserTurn)); + assert!(creates(&out), "the result is still unanswered: {out:?}"); + } + + #[test] + fn a_response_to_the_user_answers_a_tool_output_too() { + let mut floor = floor(); + run( + &mut floor, + vec![ + I::ResponseCreated, + I::ToolStarted, + done(0), + I::SpeechStarted, + completed("c1"), + I::SpeechStopped, + I::ResponseCreated, + ], + ); + let out = run(&mut floor, vec![done(0), I::TimerFired(Timer::Cooling)]); + assert!(!creates(&out)); + assert_eq!(floor.state(), FloorState::Idle); + } + + #[test] + fn several_tool_calls_get_one_answer_once_all_are_done() { + let mut floor = floor(); + run( + &mut floor, + vec![I::ResponseCreated, I::ToolStarted, I::ToolStarted, done(0)], + ); + let out = floor.handle(completed("c1")); + assert!(!creates(&out), "c2 still runs: {out:?}"); + assert!(creates(&floor.handle(completed("c2")))); + } + + #[test] + fn several_tool_outputs_while_audio_plays_wait_for_the_last_one() { + let mut floor = floor(); + run( + &mut floor, + vec![ + I::ResponseCreated, + I::ToolStarted, + I::ToolStarted, + done(1200), + completed("c1"), + ], + ); + assert!(!creates(&floor.handle(I::PlaybackDrained))); + assert!(creates(&floor.handle(completed("c2")))); + } + + #[test] + fn barge_in_leaves_a_pending_answer_to_the_user_turn() { + let mut floor = floor(); + run( + &mut floor, + vec![ + I::ResponseCreated, + I::ToolStarted, + done(1200), + completed("c1"), + ], + ); + floor.handle(I::SpeechStarted); + let out = run(&mut floor, vec![I::SpeechStopped, I::PlaybackDrained]); + assert!(!creates(&out)); + } + + #[test] + fn running_tools_hold_notifications_back() { + let mut floor = floor(); + run( + &mut floor, + vec![I::ResponseCreated, I::ToolStarted, done(0)], + ); + floor.handle(finished("a")); + let out = floor.handle(I::TimerFired(Timer::Cooling)); + assert!(!notifies(&out), "the tool result speaks first"); + let out = floor.handle(completed("c1")); + assert!(!notifies(&out) && creates(&out)); + let out = run( + &mut floor, + vec![I::ResponseCreated, done(0), I::TimerFired(Timer::Cooling)], + ); + assert!(notifies(&out)); + } + + #[test] + fn lost_drain_report_falls_back_to_the_playback_timer() { + let mut floor = floor(); + run(&mut floor, vec![I::ResponseCreated, done(500)]); + floor.handle(I::TimerFired(Timer::Playback)); + assert_eq!(floor.state(), FloorState::Cooling); + } + + #[test] + fn drain_during_generation_is_an_underrun_not_the_end() { + let mut floor = floor(); + floor.handle(I::ResponseCreated); + assert!(floor.handle(I::PlaybackDrained).is_empty()); + assert_eq!(floor.state(), FloorState::Speaking); + } + + #[test] + fn silent_user_turn_returns_the_floor() { + let mut floor = floor(); + run(&mut floor, vec![I::SpeechStarted, finished("a")]); + let out = floor.handle(I::SpeechStopped); + assert_eq!( + out, + vec![C::StartTimer(Timer::UserTurn, Duration::from_secs(8))] + ); + let out = floor.handle(I::TimerFired(Timer::UserTurn)); + assert!(notifies(&out)); + } + + #[test] + fn muting_mid_sentence_hands_the_floor_back() { + // The server hears no end of speech from a muted microphone. + let mut floor = floor(); + run(&mut floor, vec![I::SpeechStarted, finished("a")]); + let out = floor.handle(I::CaptureMuted); + assert_eq!( + out, + vec![C::StartTimer(Timer::UserTurn, Duration::from_secs(8))] + ); + assert!(notifies(&floor.handle(I::TimerFired(Timer::UserTurn)))); + } + + #[test] + fn failed_create_does_not_wedge_the_floor() { + let mut floor = floor(); + floor.handle(finished("a")); + floor.handle(I::ServerError); + assert_eq!(floor.state(), FloorState::Cooling); + floor.handle(I::TimerFired(Timer::Cooling)); + assert_eq!(floor.state(), FloorState::Idle); + assert!(creates(&floor.handle(finished("b")))); + } + + #[test] + fn reading_a_conversation_drops_its_notification() { + let mut floor = floor(); + floor.handle(I::ResponseCreated); + floor.handle(finished("a")); + floor.handle(I::ConversationRead { + conversation_id: "a".into(), + }); + let out = run(&mut floor, vec![done(0), I::TimerFired(Timer::Cooling)]); + assert!(!notifies(&out)); + assert_eq!(floor.queued_notifications(), 0); + } + + #[test] + fn stale_timer_fires_are_ignored() { + let mut floor = floor(); + answered_and_cooled(&mut floor); + assert!(floor.handle(I::TimerFired(Timer::Cooling)).is_empty()); + assert!(floor.handle(I::TimerFired(Timer::Playback)).is_empty()); + } +} diff --git a/crates/code_assistant_core/src/voice/mod.rs b/crates/code_assistant_core/src/voice/mod.rs new file mode 100644 index 00000000..edd785fa --- /dev/null +++ b/crates/code_assistant_core/src/voice/mod.rs @@ -0,0 +1,106 @@ +//! Voice mode: a global, speech-to-speech assistant over all conversations. +//! +//! The voice agent is a small agent loop of its own next to the sessions. It +//! talks to a realtime model (`llm::realtime`), streams microphone and +//! speaker audio through a [`VoiceAudio`] implementation the application +//! injects, and only has tools that act on conversations ([`tools`]). When a +//! conversation's agent finishes, the voice agent is notified — but never +//! while the user or the model speaks: [`floor::Floor`] decides when the +//! floor is free. See `docs/voice-mode.md`. +//! +//! Frontends drive it through [`VoiceService`] and follow it through the +//! app-scoped [`crate::ui::UiEvent::VoiceStatusChanged`] and +//! [`crate::ui::UiEvent::VoiceTranscript`] events. + +mod agent; +pub mod config; +pub mod floor; +pub mod notifications; +mod service; +pub mod source; +pub mod tools; + +#[cfg(test)] +mod tests; + +pub use config::{NotifyScope, VoiceConfig}; +pub use service::{ConnectorFactory, VoiceService, default_connector_factory}; + +use anyhow::Result; +use std::sync::Arc; +use tokio::sync::mpsc; + +/// What the audio devices report. +#[derive(Debug, Clone, PartialEq)] +pub enum AudioEvent { + /// Microphone samples: PCM16, 24 kHz, mono. + Captured(Vec), + /// The speakers played everything queued. + Drained, + /// The devices stopped working. + Failed(String), +} + +/// Microphone and speakers of a voice session. Implementations run the +/// devices on threads of their own and report through the +/// [`AudioEvent`] sender they were opened with. +pub trait VoiceAudio: Send { + /// Queue samples (PCM16, 24 kHz, mono) for playback. + fn play(&mut self, samples: Vec); + /// Drop everything queued for playback. + fn clear_playback(&mut self); + /// Samples the speakers played since the devices opened. + fn played_samples(&self) -> u64; + /// Stop (or resume) reporting microphone samples. + fn set_capture_muted(&mut self, muted: bool); +} + +/// Opens the audio devices for a voice session. +pub type AudioFactory = + Arc) -> Result> + Send + Sync>; + +/// What the voice agent is doing, for the frontends. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum VoiceActivity { + #[default] + Off, + Connecting, + /// Connected; the floor is free. + Listening, + /// The user has the floor. + UserSpeaking, + /// The model speaks (or prepares its answer). + Speaking, + /// Voice mode stopped because of this error. + Failed(String), +} + +impl VoiceActivity { + pub fn is_active(&self) -> bool { + !matches!(self, Self::Off | Self::Failed(_)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct VoiceStatus { + pub activity: VoiceActivity, + pub muted: bool, + /// Notifications waiting for a free floor. + pub queued_notifications: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TranscriptRole { + User, + Assistant, + /// A tool call the model made. + Tool, + /// A background notification delivered to the model. + Notification, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TranscriptEntry { + pub role: TranscriptRole, + pub text: String, +} diff --git a/crates/code_assistant_core/src/voice/notifications.rs b/crates/code_assistant_core/src/voice/notifications.rs new file mode 100644 index 00000000..959a66ce --- /dev/null +++ b/crates/code_assistant_core/src/voice/notifications.rs @@ -0,0 +1,164 @@ +//! Background notifications about conversations, waiting for a free floor. +//! +//! The queue holds at most one entry per conversation: a newer event +//! replaces an older one, so a conversation that finishes twice before the +//! floor frees up is reported once. Reading a conversation +//! (`get_conversation`) acknowledges its entry — the model knows already. + +/// What happened in a conversation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NotificationKind { + /// The agent finished its turn. + Finished, + /// The agent stopped with an error. + Failed { message: String }, + /// The agent waits for a permission decision. + NeedsPermission { tool: String }, + /// The agent asked the user questions. + NeedsAnswer, +} + +impl NotificationKind { + /// Whether the conversation waits for the user (as opposed to having + /// ended its turn). + pub fn is_waiting(&self) -> bool { + matches!(self, Self::NeedsPermission { .. } | Self::NeedsAnswer) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Notification { + pub conversation_id: String, + pub name: String, + pub project: String, + pub kind: NotificationKind, +} + +#[derive(Debug, Default)] +pub struct NotificationQueue { + /// Arrival order, one entry per conversation. + entries: Vec, +} + +impl NotificationQueue { + pub fn push(&mut self, notification: Notification) { + self.entries + .retain(|entry| entry.conversation_id != notification.conversation_id); + self.entries.push(notification); + } + + /// Drop the entry of a conversation the model has just read. + pub fn acknowledge(&mut self, conversation_id: &str) { + self.entries + .retain(|entry| entry.conversation_id != conversation_id); + } + + /// Drop a conversation's "waiting for you" entry once the wait is over + /// (answered elsewhere). A finish or failure entry stays. + pub fn retract_waiting(&mut self, conversation_id: &str) { + self.entries + .retain(|entry| entry.conversation_id != conversation_id || !entry.kind.is_waiting()); + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Empty the queue into one message for the model, or `None` when + /// nothing is queued. + pub fn take_message(&mut self) -> Option { + if self.entries.is_empty() { + return None; + } + let entries = std::mem::take(&mut self.entries); + Some(render(&entries)) + } +} + +fn render(entries: &[Notification]) -> String { + let lines: Vec = entries + .iter() + .map(|entry| { + let what = match &entry.kind { + NotificationKind::Finished => "finished its turn".to_string(), + NotificationKind::Failed { message } => { + format!("stopped with an error ({message})") + } + NotificationKind::NeedsPermission { tool } => { + format!("is waiting for permission to run `{tool}`") + } + NotificationKind::NeedsAnswer => "is waiting for answers to its questions".into(), + }; + let project = if entry.project.is_empty() { + String::new() + } else { + format!(" in project {}", entry.project) + }; + format!( + "- Conversation \"{}\"{project} (id {}) {what}.", + entry.name, entry.conversation_id + ) + }) + .collect(); + format!( + "[background notification]\n{}\n\nThe user is not speaking right now. If it is useful \ + to them, mention this briefly. Call get_conversation before you summarise a result. \ + Otherwise say nothing.", + lines.join("\n") + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn note(id: &str, kind: NotificationKind) -> Notification { + Notification { + conversation_id: id.into(), + name: format!("name {id}"), + project: "proj".into(), + kind, + } + } + + #[test] + fn newer_event_of_a_conversation_replaces_the_older_one() { + let mut queue = NotificationQueue::default(); + queue.push(note("a", NotificationKind::NeedsAnswer)); + queue.push(note("b", NotificationKind::Finished)); + queue.push(note("a", NotificationKind::Finished)); + assert_eq!(queue.len(), 2); + let message = queue.take_message().unwrap(); + assert!(message.find("id b").unwrap() < message.find("id a").unwrap()); + assert!(!message.contains("questions")); + assert!(queue.is_empty()); + } + + #[test] + fn reading_a_conversation_acknowledges_it() { + let mut queue = NotificationQueue::default(); + queue.push(note("a", NotificationKind::Finished)); + queue.acknowledge("a"); + assert_eq!(queue.take_message(), None); + } + + #[test] + fn retracting_a_wait_keeps_a_finish() { + let mut queue = NotificationQueue::default(); + queue.push(note( + "a", + NotificationKind::NeedsPermission { + tool: "execute_command".into(), + }, + )); + queue.push(note("b", NotificationKind::Finished)); + queue.retract_waiting("a"); + queue.retract_waiting("b"); + assert_eq!(queue.len(), 1); + assert!(queue.take_message().unwrap().contains("id b")); + } +} diff --git a/crates/code_assistant_core/src/voice/service.rs b/crates/code_assistant_core/src/voice/service.rs new file mode 100644 index 00000000..8a3ea445 --- /dev/null +++ b/crates/code_assistant_core/src/voice/service.rs @@ -0,0 +1,191 @@ +//! [`VoiceService`]: the frontends' handle to start, stop and mute voice +//! mode. + +use super::agent::{Control, VoiceAgent}; +use super::{AudioFactory, VoiceActivity, VoiceConfig, VoiceStatus}; +use crate::session::SessionService; +use crate::session::event_stream::EventStream; +use crate::ui::UiEvent; +use anyhow::Result; +use llm::realtime::RealtimeConnector; +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; +use tracing::{error, info}; + +/// Builds the connector for a voice session from its configuration. +pub type ConnectorFactory = + Arc Result> + Send + Sync>; + +/// The connector for the configured provider (OpenAI or AI Core). +pub fn default_connector_factory() -> ConnectorFactory { + Arc::new(|config: &VoiceConfig| config.connector()) +} + +#[derive(Debug)] +enum Command { + Start, + Stop, + SetMuted(bool), +} + +/// Cloneable handle; the voice agent runs in the worker future returned by +/// [`VoiceService::new`], which the application spawns on its backend +/// runtime. Status changes arrive as app-scoped +/// [`UiEvent::VoiceStatusChanged`] events. +#[derive(Clone)] +pub struct VoiceService { + tx: mpsc::UnboundedSender, +} + +impl VoiceService { + pub fn new( + sessions: SessionService, + events: EventStream, + audio: AudioFactory, + connectors: ConnectorFactory, + ) -> (Self, impl Future) { + Self::with_config_loader( + sessions, + events, + audio, + connectors, + Arc::new(VoiceConfig::load), + ) + } + + /// Like [`Self::new`], reading the configuration through `load_config` + /// on every start (tests pass a fixed one). + pub fn with_config_loader( + sessions: SessionService, + events: EventStream, + audio: AudioFactory, + connectors: ConnectorFactory, + load_config: Arc VoiceConfig + Send + Sync>, + ) -> (Self, impl Future) { + let (tx, mut rx) = mpsc::unbounded_channel::(); + let worker = async move { + let mut running: Option = None; + while let Some(command) = rx.recv().await { + if running.as_ref().is_some_and(|r| r.task.is_finished()) { + running = None; + } + match command { + Command::Start => { + if running.is_some() { + continue; + } + running = start(load_config(), &sessions, &events, &audio, &connectors); + } + Command::Stop => { + if let Some(r) = running.take() { + r.stop().await; + } + publish(&events, VoiceStatus::default()); + } + Command::SetMuted(muted) => { + if let Some(r) = &running { + let _ = r.control.send(Control::SetMuted(muted)); + } + } + } + } + if let Some(r) = running { + r.stop().await; + } + }; + (Self { tx }, worker) + } + + pub fn start(&self) { + let _ = self.tx.send(Command::Start); + } + + pub fn stop(&self) { + let _ = self.tx.send(Command::Stop); + } + + pub fn set_muted(&self, muted: bool) { + let _ = self.tx.send(Command::SetMuted(muted)); + } +} + +struct Running { + control: mpsc::UnboundedSender, + task: JoinHandle<()>, +} + +/// How long a running agent gets to wind down on its own. +const STOP_GRACE: Duration = Duration::from_secs(1); + +impl Running { + /// Ask the agent to stop. One that does not get to its control channel + /// in time (a connection attempt or a reconnect backoff) is aborted. + async fn stop(mut self) { + let _ = self.control.send(Control::Stop); + if tokio::time::timeout(STOP_GRACE, &mut self.task) + .await + .is_err() + { + self.task.abort(); + let _ = self.task.await; + } + } +} + +fn start( + config: VoiceConfig, + sessions: &SessionService, + events: &EventStream, + audio: &AudioFactory, + connectors: &ConnectorFactory, +) -> Option { + publish( + events, + VoiceStatus { + activity: VoiceActivity::Connecting, + ..VoiceStatus::default() + }, + ); + let connector = match connectors(&config) { + Ok(connector) => connector, + Err(e) => { + fail(events, e); + return None; + } + }; + let (control, control_rx) = mpsc::unbounded_channel(); + let sessions = sessions.clone(); + let events = events.clone(); + let audio = audio.clone(); + let task = tokio::spawn(async move { + let agent = + match VoiceAgent::start(config, connector, sessions, events.clone(), audio).await { + Ok(agent) => agent, + Err(e) => return fail(&events, e), + }; + info!("Voice mode started"); + match agent.run(control_rx).await { + Ok(()) => info!("Voice mode stopped"), + Err(e) => fail(&events, e), + } + }); + Some(Running { control, task }) +} + +fn fail(events: &EventStream, error: anyhow::Error) { + error!("Voice mode failed: {error:#}"); + publish( + events, + VoiceStatus { + activity: VoiceActivity::Failed(format!("{error:#}")), + ..VoiceStatus::default() + }, + ); +} + +fn publish(events: &EventStream, status: VoiceStatus) { + events.publish_app(UiEvent::VoiceStatusChanged { status }); +} diff --git a/crates/code_assistant_core/src/voice/source.rs b/crates/code_assistant_core/src/voice/source.rs new file mode 100644 index 00000000..7e45b2fb --- /dev/null +++ b/crates/code_assistant_core/src/voice/source.rs @@ -0,0 +1,282 @@ +//! Turns the core's event stream into notifications for the voice agent. + +use super::config::NotifyScope; +use super::floor::FloorInput; +use super::notifications::{Notification, NotificationKind}; +use crate::session::SessionService; +use crate::session::event_stream::{EventPayload, SessionEvent}; +use crate::session::instance::SessionActivityState; +use crate::ui::UiEvent; +use anyhow::Result; +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, Mutex}; + +/// Permission and question requests open right now, shared with the tools +/// so `list_conversations` can tell which conversation waits for the user. +#[derive(Clone, Default)] +pub struct OpenRequests { + /// request id → (conversation id, what it waits for) + inner: Arc>>, +} + +impl OpenRequests { + fn open(&self, request_id: String, conversation_id: String, kind: NotificationKind) { + self.inner + .lock() + .unwrap() + .insert(request_id, (conversation_id, kind)); + } + + /// Close a request; returns its conversation when that has no other + /// open request left. + fn close(&self, request_id: &str) -> Option { + let mut inner = self.inner.lock().unwrap(); + let (conversation_id, _) = inner.remove(request_id)?; + let still_waiting = inner.values().any(|(id, _)| *id == conversation_id); + (!still_waiting).then_some(conversation_id) + } + + fn close_conversation(&self, conversation_id: &str) { + self.inner + .lock() + .unwrap() + .retain(|_, (id, _)| id != conversation_id); + } + + /// A state label when the conversation waits for the user. + pub fn waiting_for(&self, conversation_id: &str) -> Option { + let inner = self.inner.lock().unwrap(); + inner + .values() + .find(|(id, _)| id == conversation_id) + .map(|(_, kind)| match kind { + NotificationKind::NeedsPermission { tool } => { + format!("waiting for permission to run {tool}") + } + _ => "waiting for answers to its questions".to_string(), + }) + } +} + +/// What happened, before names are looked up. +#[derive(Debug, Clone, PartialEq)] +enum Change { + Notify(String, NotificationKind), + WaitEnded(String), +} + +pub struct NotificationSource { + service: SessionService, + scope: NotifyScope, + touched: HashSet, + states: HashMap, + open_requests: OpenRequests, +} + +impl NotificationSource { + /// Start from the current activity states, so a run that was going on + /// before voice mode started is reported when it ends. + pub async fn new( + service: SessionService, + scope: NotifyScope, + open_requests: OpenRequests, + ) -> Result { + let states = service.session_activity_states().await?; + Ok(Self { + service, + scope, + touched: HashSet::new(), + states, + open_requests, + }) + } + + /// The voice agent created or messaged this conversation. + pub fn touch(&mut self, conversation_id: &str) { + self.touched.insert(conversation_id.to_string()); + } + + pub async fn on_event(&mut self, event: &SessionEvent) -> Vec { + let Some(change) = self.change_for(event) else { + return Vec::new(); + }; + self.resolve(vec![change]).await + } + + /// After the stream lagged: compare the activity states with the last + /// known ones. + pub async fn resync(&mut self) -> Vec { + let Ok(current) = self.service.session_activity_states().await else { + return Vec::new(); + }; + let mut changes = Vec::new(); + for (id, previous) in &self.states { + let now = current.get(id).cloned().unwrap_or_default(); + if let Some(kind) = ended(previous, &now) { + changes.push(Change::Notify(id.clone(), kind)); + } + } + self.states = current; + let changes = changes + .into_iter() + .filter(|change| self.in_scope(change)) + .collect(); + self.resolve(changes).await + } + + fn change_for(&mut self, event: &SessionEvent) -> Option { + let EventPayload::Ui(ui) = &event.payload else { + return None; + }; + let session_id = event.session_id.clone(); + let change = match ui { + UiEvent::UpdateSessionActivityState { + session_id, + activity_state, + } => { + let previous = self + .states + .insert(session_id.clone(), activity_state.clone()); + if activity_state.is_terminal() { + self.open_requests.close_conversation(session_id); + } + let kind = ended(&previous?, activity_state)?; + Change::Notify(session_id.clone(), kind) + } + UiEvent::RequestToolPermission { request } => { + let session_id = session_id?; + let kind = NotificationKind::NeedsPermission { + tool: request.tool_name.clone(), + }; + self.open_requests.open( + request.request_id.clone(), + session_id.clone(), + kind.clone(), + ); + Change::Notify(session_id, kind) + } + UiEvent::RequestUserQuestions { request } => { + let session_id = session_id?; + let kind = NotificationKind::NeedsAnswer; + self.open_requests.open( + request.request_id.clone(), + session_id.clone(), + kind.clone(), + ); + Change::Notify(session_id, kind) + } + UiEvent::ToolPermissionRequestResolved { request_id } + | UiEvent::UserQuestionsResolved { request_id } => { + Change::WaitEnded(self.open_requests.close(request_id)?) + } + _ => return None, + }; + self.in_scope(&change).then_some(change) + } + + fn in_scope(&self, change: &Change) -> bool { + match (self.scope, change) { + (NotifyScope::All, _) | (_, Change::WaitEnded(_)) => true, + (NotifyScope::Touched, Change::Notify(id, _)) => self.touched.contains(id), + } + } + + async fn resolve(&self, changes: Vec) -> Vec { + if changes.is_empty() { + return Vec::new(); + } + let sessions = self.service.list_sessions().await.unwrap_or_default(); + changes + .into_iter() + .map(|change| match change { + Change::WaitEnded(conversation_id) => FloorInput::WaitEnded { conversation_id }, + Change::Notify(conversation_id, kind) => { + let metadata = super::tools::find_metadata(&sessions, &conversation_id); + FloorInput::Notification(Notification { + name: metadata + .map(|m| m.name.clone()) + .unwrap_or_else(|| conversation_id.clone()), + project: metadata + .map(|m| m.initial_project.clone()) + .unwrap_or_default(), + conversation_id, + kind, + }) + } + }) + .collect() + } +} + +/// The notification for a run ending, when `previous → now` is one. +fn ended(previous: &SessionActivityState, now: &SessionActivityState) -> Option { + let was_running = matches!( + previous, + SessionActivityState::AgentRunning + | SessionActivityState::WaitingForResponse + | SessionActivityState::RateLimited { .. } + ); + if !was_running { + return None; + } + match now { + SessionActivityState::Idle => Some(NotificationKind::Finished), + SessionActivityState::Errored { message } => Some(NotificationKind::Failed { + message: message.clone(), + }), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_the_end_of_a_run_notifies() { + use SessionActivityState as S; + assert_eq!( + ended(&S::AgentRunning, &S::Idle), + Some(NotificationKind::Finished) + ); + assert_eq!( + ended( + &S::WaitingForResponse, + &S::Errored { + message: "x".into() + } + ), + Some(NotificationKind::Failed { + message: "x".into() + }) + ); + assert_eq!(ended(&S::AgentRunning, &S::WaitingForResponse), None); + // Clearing an error is not a run ending. + assert_eq!( + ended( + &S::Errored { + message: "x".into() + }, + &S::Idle + ), + None + ); + assert_eq!(ended(&S::Idle, &S::Idle), None); + } + + #[test] + fn a_conversation_waits_until_its_last_request_closes() { + let requests = OpenRequests::default(); + requests.open("r1".into(), "s".into(), NotificationKind::NeedsAnswer); + requests.open( + "r2".into(), + "s".into(), + NotificationKind::NeedsPermission { tool: "t".into() }, + ); + assert!(requests.waiting_for("s").is_some()); + assert_eq!(requests.close("r1"), None); + assert_eq!(requests.close("r2"), Some("s".into())); + assert_eq!(requests.waiting_for("s"), None); + assert_eq!(requests.close("r2"), None); + } +} diff --git a/crates/code_assistant_core/src/voice/tests.rs b/crates/code_assistant_core/src/voice/tests.rs new file mode 100644 index 00000000..15c72720 --- /dev/null +++ b/crates/code_assistant_core/src/voice/tests.rs @@ -0,0 +1,556 @@ +//! End-to-end tests of the voice agent: a scripted realtime server, fake +//! audio devices, and a real session service whose agents answer from a +//! mock LLM. + +use super::*; +use crate::mocks::{MockLLMProvider, create_test_response_text}; +use crate::persistence::FileSessionPersistence; +use crate::session::event_stream::EventStream; +use crate::session::service::{AgentRuntimeOptions, default_project_manager_factory}; +use crate::session::{SessionConfig, SessionManager, SessionService}; +use crate::ui::UiEvent; +use llm::realtime::{ + ClientEvent, ConversationItem, Incoming, RealtimeConnection, RealtimeConnector, ResponseInfo, + ServerEvent, encode_pcm16, +}; +use std::sync::Mutex; +use std::time::Duration; +use tokio::sync::mpsc; + +const WAIT: Duration = Duration::from_secs(5); + +/// The server side of one scripted realtime connection. +struct Server { + from_client: mpsc::UnboundedReceiver, + to_client: mpsc::UnboundedSender, +} + +impl Server { + fn send(&self, event: ServerEvent) { + self.to_client.send(Incoming::Event(event)).unwrap(); + } + + /// Read client events until one matches; audio appends are skipped. + async fn expect(&mut self, what: &str, pred: impl Fn(&ClientEvent) -> bool) -> ClientEvent { + tokio::time::timeout(WAIT, async { + loop { + let event = self.from_client.recv().await.expect("client hung up"); + if pred(&event) { + return event; + } + } + }) + .await + .unwrap_or_else(|_| panic!("timed out waiting for {what}")) + } + + /// Assert that nothing but audio arrives for `duration`. + async fn expect_quiet(&mut self, duration: Duration) { + let deadline = tokio::time::Instant::now() + duration; + while let Ok(Some(event)) = tokio::time::timeout_at(deadline, self.from_client.recv()).await + { + assert!( + matches!(event, ClientEvent::AppendAudio(_)), + "expected no client events, got {event:?}" + ); + } + } +} + +struct ScriptedConnector { + connections: Mutex>, +} + +#[async_trait::async_trait] +impl RealtimeConnector for ScriptedConnector { + async fn connect(&self) -> anyhow::Result { + self.connections + .lock() + .unwrap() + .pop() + .ok_or_else(|| anyhow::anyhow!("no more scripted connections")) + } +} + +fn scripted_connection() -> (RealtimeConnection, Server) { + let (outgoing, from_client) = mpsc::unbounded_channel(); + let (to_client, incoming) = mpsc::unbounded_channel(); + ( + RealtimeConnection { outgoing, incoming }, + Server { + from_client, + to_client, + }, + ) +} + +#[derive(Default)] +struct AudioState { + queued: u64, + played: u64, + clears: usize, + events: Option>, +} + +#[derive(Clone, Default)] +struct FakeAudio(Arc>); + +impl FakeAudio { + fn factory(&self) -> AudioFactory { + let audio = self.clone(); + Arc::new(move |events| { + audio.0.lock().unwrap().events = Some(events); + Ok(Box::new(audio.clone()) as Box) + }) + } + + /// Wait until the voice agent, on its own task, brought the devices + /// into the state `pred` expects. + async fn wait_for(&self, what: &str, pred: impl Fn(&AudioState) -> bool) { + tokio::time::timeout(WAIT, async { + while !pred(&self.0.lock().unwrap()) { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("timed out waiting for {what}")); + } + + /// The speakers played this many samples, and drained if that was all. + fn play(&self, samples: u64) { + let mut state = self.0.lock().unwrap(); + state.played = (state.played + samples).min(state.queued); + if state.played == state.queued { + let _ = state.events.as_ref().unwrap().send(AudioEvent::Drained); + } + } +} + +impl VoiceAudio for FakeAudio { + fn play(&mut self, samples: Vec) { + self.0.lock().unwrap().queued += samples.len() as u64; + } + fn clear_playback(&mut self) { + let mut state = self.0.lock().unwrap(); + state.clears += 1; + state.queued = state.played; + } + fn played_samples(&self) -> u64 { + self.0.lock().unwrap().played + } + fn set_capture_muted(&mut self, _muted: bool) {} +} + +struct Harness { + sessions: SessionService, + voice: VoiceService, + server: Server, + audio: FakeAudio, + _dir: tempfile::TempDir, +} + +/// Never answers, like a connection attempt into a blackholed network. +struct HangingConnector; + +#[async_trait::async_trait] +impl RealtimeConnector for HangingConnector { + async fn connect(&self) -> anyhow::Result { + std::future::pending().await + } +} + +fn test_config() -> VoiceConfig { + VoiceConfig { + provider: "test".into(), + cooling_ms: 50, + ..VoiceConfig::default() + } +} + +/// A voice service whose connection attempts never finish, with a +/// subscription to its status events. +fn hanging_voice( + dir: &tempfile::TempDir, +) -> (VoiceService, crate::session::event_stream::Subscription) { + let (sessions, events) = session_service(dir); + let statuses = sessions.subscribe(); + let (voice, worker) = VoiceService::with_config_loader( + sessions, + events, + FakeAudio::default().factory(), + Arc::new(|_| Ok(Arc::new(HangingConnector) as Arc)), + Arc::new(test_config), + ); + tokio::spawn(worker); + (voice, statuses) +} + +/// A session service whose agent runs all answer "done". +fn session_service(dir: &tempfile::TempDir) -> (SessionService, EventStream) { + let events = EventStream::new(); + let manager = Arc::new(tokio::sync::Mutex::new(SessionManager::new( + FileSessionPersistence::new_with_root_dir(dir.path().to_path_buf()), + SessionConfig::default(), + "test-model".to_string(), + crate::tools::test_registry(), + events.clone(), + ))); + let runtime = Arc::new(AgentRuntimeOptions { + record_path: None, + playback_path: None, + fast_playback: false, + command_executor_factory: Arc::new(|_| { + Box::new(crate::mocks::create_command_executor_mock()) + }), + project_manager_factory: default_project_manager_factory(), + llm_client_factory: Some(Arc::new(|_| { + Ok(Box::new( + MockLLMProvider::new(vec![Ok(create_test_response_text("done"))]).streaming(), + )) + })), + }); + let (sessions, worker) = SessionService::new(manager, runtime, events.clone()); + tokio::spawn(worker); + (sessions, events) +} + +/// A running voice session over [`session_service`]. +async fn harness() -> Harness { + let dir = tempfile::tempdir().unwrap(); + let (sessions, events) = session_service(&dir); + let (connection, mut server) = scripted_connection(); + let connector: Arc = Arc::new(ScriptedConnector { + connections: Mutex::new(vec![connection]), + }); + let audio = FakeAudio::default(); + let (voice, voice_worker) = VoiceService::with_config_loader( + sessions.clone(), + events, + audio.factory(), + Arc::new(move |_| Ok(connector.clone())), + Arc::new(test_config), + ); + tokio::spawn(voice_worker); + voice.start(); + server + .expect("session.update", |e| { + matches!(e, ClientEvent::SessionUpdate(_)) + }) + .await; + server.send(ServerEvent::SessionCreated); + Harness { + sessions, + voice, + server, + audio, + _dir: dir, + } +} + +fn response_created() -> ServerEvent { + ServerEvent::ResponseCreated { + response: ResponseInfo { + id: "r".into(), + status: None, + }, + } +} + +fn response_done() -> ServerEvent { + ServerEvent::ResponseDone { + response: ResponseInfo { + id: "r".into(), + status: Some("completed".into()), + }, + } +} + +fn audio_delta(item: &str, samples: usize) -> ServerEvent { + ServerEvent::AudioDelta { + item_id: item.into(), + delta: encode_pcm16(&vec![0; samples]), + } +} + +fn function_call(name: &str, call_id: &str, arguments: serde_json::Value) -> ServerEvent { + ServerEvent::OutputItemDone { + item: ConversationItem { + id: format!("item-{call_id}"), + kind: "function_call".into(), + name: Some(name.into()), + call_id: Some(call_id.into()), + arguments: Some(arguments.to_string()), + }, + } +} + +async fn wait_idle(sessions: &SessionService, id: &str) { + tokio::time::timeout(WAIT, async { + while sessions.is_session_busy(id.to_string()).await.unwrap() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("session finished"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_finish_while_the_model_speaks_waits_for_playback_and_silence() { + let mut h = harness().await; + let id = h + .sessions + .create_session(Some("Fix flaky test".into()), None) + .await + .unwrap(); + + // The model is talking: one second of audio is queued. + h.server.send(response_created()); + h.server.send(audio_delta("speech", 24_000)); + h.audio + .wait_for("the queued audio", |a| a.queued == 24_000) + .await; + h.sessions + .send_user_message(id.clone(), "go".into(), Vec::new(), None) + .await + .unwrap(); + wait_idle(&h.sessions, &id).await; + + // Generation ends, the speakers still play. + h.server.send(response_done()); + h.server.expect_quiet(Duration::from_millis(200)).await; + + h.audio.play(24_000); + let note = h + .server + .expect("notification", |e| { + matches!(e, ClientEvent::SystemMessage(_)) + }) + .await; + let ClientEvent::SystemMessage(text) = note else { + unreachable!() + }; + assert!(text.contains("Fix flaky test"), "{text}"); + assert!(text.contains("finished"), "{text}"); + h.server + .expect("response.create", |e| *e == ClientEvent::CreateResponse) + .await; + h.voice.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn message_conversation_starts_a_turn_and_its_end_is_announced() { + let mut h = harness().await; + // A session in "proj" makes the project known. + h.sessions + .create_session(None, Some("proj".into())) + .await + .unwrap(); + + h.server.send(response_created()); + h.server.send(function_call( + "message_conversation", + "c1", + serde_json::json!({"project": "proj", "title": "Docs", "message": "update the docs"}), + )); + h.server.send(response_done()); + + let output = h + .server + .expect("tool output", |e| { + matches!(e, ClientEvent::FunctionCallOutput { .. }) + }) + .await; + let ClientEvent::FunctionCallOutput { call_id, output } = output else { + unreachable!() + }; + assert_eq!(call_id, "c1"); + let output: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert_eq!(output["status"], "started"); + let new_id = output["conversation_id"].as_str().unwrap().to_string(); + // The output arrived on a free floor: the model answers at once. + h.server + .expect("response.create", |e| *e == ClientEvent::CreateResponse) + .await; + h.server.send(response_created()); + h.server.send(response_done()); + + wait_idle(&h.sessions, &new_id).await; + let note = h + .server + .expect("notification", |e| { + matches!(e, ClientEvent::SystemMessage(_)) + }) + .await; + let ClientEvent::SystemMessage(text) = note else { + unreachable!() + }; + assert!(text.contains(&new_id), "{text}"); + h.voice.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn reading_a_finished_conversation_returns_its_last_answer_and_drops_the_notification() { + let mut h = harness().await; + let id = h.sessions.create_session(None, None).await.unwrap(); + + // The user talks while the conversation finishes. + h.server.send(ServerEvent::SpeechStarted); + h.sessions + .send_user_message(id.clone(), "go".into(), Vec::new(), None) + .await + .unwrap(); + wait_idle(&h.sessions, &id).await; + h.server.send(ServerEvent::SpeechStopped); + h.server.send(response_created()); + h.server.send(function_call( + "get_conversation", + "c1", + serde_json::json!({"conversation_id": id, "full": true}), + )); + h.server.send(response_done()); + + let output = h + .server + .expect("tool output", |e| { + matches!(e, ClientEvent::FunctionCallOutput { .. }) + }) + .await; + let ClientEvent::FunctionCallOutput { output, .. } = output else { + unreachable!() + }; + let output: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert_eq!(output["state"], "idle"); + assert_eq!(output["turns"][0]["user"], "go"); + assert_eq!(output["turns"][0]["agent"], "done"); + + h.server + .expect("response.create", |e| *e == ClientEvent::CreateResponse) + .await; + h.server.send(response_created()); + h.server.send(response_done()); + // The model knows the result; no notification follows. + h.server.expect_quiet(Duration::from_millis(300)).await; + h.voice.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn barge_in_cancels_and_truncates_to_what_was_heard() { + let mut h = harness().await; + h.server.send(response_created()); + h.server.send(audio_delta("speech", 24_000)); + h.audio + .wait_for("the queued audio", |a| a.queued == 24_000) + .await; + // Half a second played when the user interrupts. + h.audio.play(12_000); + h.server.send(ServerEvent::SpeechStarted); + + h.server + .expect("response.cancel", |e| *e == ClientEvent::CancelResponse) + .await; + let truncate = h + .server + .expect("truncate", |e| { + matches!(e, ClientEvent::TruncateItem { .. }) + }) + .await; + assert_eq!( + truncate, + ClientEvent::TruncateItem { + item_id: "speech".into(), + audio_end_ms: 500 + } + ); + // The agent clears the speakers right after it sent the truncate. + h.audio + .wait_for("the cleared speakers", |a| a.clears == 1) + .await; + h.voice.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_response_cancelled_before_its_audio_stays_silent() { + let mut h = harness().await; + let id = h.sessions.create_session(None, None).await.unwrap(); + h.sessions + .send_user_message(id.clone(), "go".into(), Vec::new(), None) + .await + .unwrap(); + wait_idle(&h.sessions, &id).await; + // The finish is announced on the free floor... + h.server + .expect("response.create", |e| *e == ClientEvent::CreateResponse) + .await; + // ...and the user speaks up before that response exists. + h.server.send(ServerEvent::SpeechStarted); + h.server.send(response_created()); + h.server + .expect("response.cancel", |e| *e == ClientEvent::CancelResponse) + .await; + // Audio generated before the server handled the cancel. + h.server.send(audio_delta("notification", 2_400)); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(h.audio.0.lock().unwrap().queued, 0, "played over the user"); + h.voice.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn status_events_follow_the_floor() { + let h = harness().await; + let mut events = h.sessions.subscribe(); + h.server.send(response_created()); + let status = + wait_for_status(&mut events, WAIT, |s| s.activity == VoiceActivity::Speaking).await; + assert!(!status.muted); + h.voice.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn stop_ends_a_connection_attempt_that_hangs() { + let dir = tempfile::tempdir().unwrap(); + let (voice, mut statuses) = hanging_voice(&dir); + voice.start(); + wait_for_status(&mut statuses, WAIT, |s| { + s.activity == VoiceActivity::Connecting + }) + .await; + voice.stop(); + wait_for_status(&mut statuses, WAIT, |s| s.activity == VoiceActivity::Off).await; +} + +async fn wait_for_status( + events: &mut crate::session::event_stream::Subscription, + within: Duration, + pred: impl Fn(&VoiceStatus) -> bool, +) -> VoiceStatus { + tokio::time::timeout(within, async { + loop { + if let Ok(event) = events.recv().await + && let crate::session::event_stream::EventPayload::Ui(UiEvent::VoiceStatusChanged { + status, + }) = event.payload + && pred(&status) + { + return status; + } + } + }) + .await + .expect("voice status") +} + +#[tokio::test(start_paused = true)] +async fn a_connection_attempt_that_hangs_times_out() { + let dir = tempfile::tempdir().unwrap(); + let (voice, mut statuses) = hanging_voice(&dir); + voice.start(); + // Paused time runs ahead to the connect timeout. + let status = wait_for_status(&mut statuses, Duration::from_secs(60), |s| { + matches!(s.activity, VoiceActivity::Failed(_)) + }) + .await; + let VoiceActivity::Failed(message) = status.activity else { + unreachable!() + }; + assert!(message.contains("No realtime connection"), "{message}"); +} diff --git a/crates/code_assistant_core/src/voice/tools.rs b/crates/code_assistant_core/src/voice/tools.rs new file mode 100644 index 00000000..a04abe93 --- /dev/null +++ b/crates/code_assistant_core/src/voice/tools.rs @@ -0,0 +1,498 @@ +//! The voice agent's tools: they act on conversations (sessions) and do +//! nothing else. +//! +//! Every tool returns at once. `message_conversation` starts or queues a +//! turn and leaves it running; the agent hears about its end through a +//! notification. + +use super::source::OpenRequests; +use crate::persistence::ChatMetadata; +use crate::session::SessionService; +use crate::session::instance::SessionActivityState; +use crate::session_query::{ContentItem, ContentKind, ContentPart, ContentProjection}; +use anyhow::{Context, Result, anyhow, bail}; +use llm::realtime::ToolDefinition; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::collections::BTreeSet; +use std::time::SystemTime; + +/// Turns `get_conversation` returns with `full`. +pub const FULL_TURNS: usize = 10; +/// Characters per message `get_conversation` returns. +pub const MAX_MESSAGE_CHARS: usize = 2_000; +/// Characters of all messages of one `get_conversation` call. +pub const MAX_TOTAL_CHARS: usize = 16_000; +const DEFAULT_LIST_LIMIT: usize = 20; +const MAX_LIST_LIMIT: usize = 50; + +/// What a tool call means for the voice agent beyond its output. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ToolEffect { + /// The model read the conversation: its notification is known. + Read(String), + /// The model created or messaged the conversation. + Touched(String), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ToolOutcome { + /// JSON text for the function call output. + pub output: String, + pub effects: Vec, +} + +impl ToolOutcome { + pub fn error(error: anyhow::Error) -> Self { + Self { + output: json!({ "error": format!("{error:#}") }).to_string(), + effects: Vec::new(), + } + } +} + +#[derive(Clone)] +pub struct VoiceTools { + service: SessionService, + open_requests: OpenRequests, +} + +impl VoiceTools { + pub fn new(service: SessionService, open_requests: OpenRequests) -> Self { + Self { + service, + open_requests, + } + } + + pub fn definitions() -> Vec { + vec![ + ToolDefinition { + name: "list_conversations".into(), + description: "List the user's conversations with coding agents, most recently \ + active first, with their state (running, idle, errored, or waiting for the \ + user)." + .into(), + parameters: json!({ + "type": "object", + "properties": { + "project": { "type": "string", "description": "Only conversations of this project." }, + "include_settled": { "type": "boolean", "description": "Also list settled (archived) conversations. Default false." }, + "limit": { "type": "integer", "description": "Maximum number of conversations, default 20." } + }, + }), + }, + ToolDefinition { + name: "list_projects".into(), + description: "List the projects a new conversation can be started in.".into(), + parameters: json!({ "type": "object", "properties": {} }), + }, + ToolDefinition { + name: "message_conversation".into(), + description: "Send a message to a coding agent conversation. Without \ + conversation_id a new conversation is started in `project`. If the agent is \ + busy the message is queued for its next turn. Returns immediately; you get a \ + notification when the agent finishes." + .into(), + parameters: json!({ + "type": "object", + "properties": { + "conversation_id": { "type": "string", "description": "The conversation to message; empty to start a new one." }, + "project": { "type": "string", "description": "Project of a new conversation (see list_projects)." }, + "title": { "type": "string", "description": "Short title of a new conversation." }, + "message": { "type": "string", "description": "The message, written as the user's instruction to the agent." } + }, + "required": ["message"], + }), + }, + ToolDefinition { + name: "get_conversation".into(), + description: "Read a conversation. By default returns its state and the agent's \ + last message. With full=true returns the last 10 turns, each with the user's \ + message and the agent's final message of that turn." + .into(), + parameters: json!({ + "type": "object", + "properties": { + "conversation_id": { "type": "string" }, + "full": { "type": "boolean", "description": "Return the last 10 turns. Default false." } + }, + "required": ["conversation_id"], + }), + }, + ] + } + + /// Run a tool. Failures become an `{"error": …}` output the model can + /// read; they never end the voice session. + pub async fn call(&self, name: &str, arguments: &str) -> ToolOutcome { + let arguments: Value = if arguments.trim().is_empty() { + json!({}) + } else { + match serde_json::from_str(arguments) { + Ok(value) => value, + Err(e) => return ToolOutcome::error(anyhow!("invalid arguments: {e}")), + } + }; + let result = match name { + "list_conversations" => self.list_conversations(arguments).await, + "list_projects" => self.list_projects().await, + "message_conversation" => self.message_conversation(arguments).await, + "get_conversation" => self.get_conversation(arguments).await, + other => Err(anyhow!("unknown tool {other}")), + }; + result.unwrap_or_else(ToolOutcome::error) + } + + async fn list_conversations(&self, arguments: Value) -> Result { + #[derive(Deserialize)] + struct Args { + project: Option, + #[serde(default)] + include_settled: bool, + limit: Option, + } + let args: Args = serde_json::from_value(arguments)?; + let mut sessions = self.service.list_sessions().await?; + let lifecycles = self.service.list_session_lifecycles().await?; + let states = self.service.session_activity_states().await?; + let now = SystemTime::now(); + + sessions.sort_by_key(|s| std::cmp::Reverse(s.updated_at)); + let limit = args + .limit + .unwrap_or(DEFAULT_LIST_LIMIT) + .clamp(1, MAX_LIST_LIMIT); + let conversations: Vec = sessions + .iter() + .filter(|s| { + args.project + .as_deref() + .is_none_or(|p| s.initial_project.eq_ignore_ascii_case(p)) + }) + .filter(|s| { + args.include_settled || !lifecycles.get(&s.id).is_some_and(|l| l.is_settled()) + }) + .take(limit) + .map(|s| { + json!({ + "id": s.id, + "title": s.name, + "project": s.initial_project, + "branch": s.branch, + "state": self.state_label(&s.id, states.get(&s.id)), + "last_active": relative_time(now, s.updated_at), + }) + }) + .collect(); + Ok(ToolOutcome { + output: json!({ "conversations": conversations }).to_string(), + effects: Vec::new(), + }) + } + + async fn list_projects(&self) -> Result { + let projects = self.project_names().await?; + Ok(ToolOutcome { + output: json!({ "projects": projects }).to_string(), + effects: Vec::new(), + }) + } + + async fn project_names(&self) -> Result> { + let mut names: BTreeSet = crate::config::load_projects() + .unwrap_or_default() + .into_keys() + .collect(); + for session in self.service.list_sessions().await? { + if !session.initial_project.is_empty() { + names.insert(session.initial_project); + } + } + Ok(names.into_iter().collect()) + } + + async fn message_conversation(&self, arguments: Value) -> Result { + #[derive(Deserialize)] + struct Args { + conversation_id: Option, + project: Option, + title: Option, + message: String, + } + let args: Args = serde_json::from_value(arguments)?; + if args.message.trim().is_empty() { + bail!("message is empty"); + } + let existing = args.conversation_id.filter(|id| !id.trim().is_empty()); + + let (conversation_id, status) = match existing { + Some(id) => { + let busy = self.service.is_session_busy(id.clone()).await?; + self.service + .send_or_queue_user_message(id.clone(), args.message, Vec::new()) + .await?; + (id, if busy { "queued" } else { "started" }) + } + None => { + let project = args + .project + .filter(|p| !p.trim().is_empty()) + .context("project is required to start a new conversation")?; + let known = self.project_names().await?; + let project = known + .iter() + .find(|name| name.eq_ignore_ascii_case(&project)) + .cloned() + .ok_or_else(|| { + anyhow!( + "unknown project '{project}'; known projects: {}", + known.join(", ") + ) + })?; + let title = args.title.filter(|t| !t.trim().is_empty()); + let id = self.service.create_session(title, Some(project)).await?; + self.service + .send_user_message(id.clone(), args.message, Vec::new(), None) + .await?; + (id, "started") + } + }; + Ok(ToolOutcome { + output: json!({ "conversation_id": conversation_id, "status": status }).to_string(), + effects: vec![ToolEffect::Touched(conversation_id)], + }) + } + + async fn get_conversation(&self, arguments: Value) -> Result { + #[derive(Deserialize)] + struct Args { + conversation_id: String, + #[serde(default)] + full: bool, + } + let args: Args = serde_json::from_value(arguments)?; + let id = args.conversation_id; + let content = self + .service + .session_content( + id.clone(), + ContentProjection { + parts: vec![ContentPart::UserText, ContentPart::AssistantText], + ..Default::default() + }, + ) + .await?; + let states = self.service.session_activity_states().await?; + let turns = group_turns(&content.items); + + let mut output = json!({ + "id": id, + "title": content.name, + "project": content.project, + "state": self.state_label(&id, states.get(&id)), + }); + if args.full { + output["turns"] = json!(last_turns(&turns, FULL_TURNS)); + } else { + let last = turns + .iter() + .rev() + .find_map(|turn| turn.agent.as_deref()) + .map(|text| truncate(text, MAX_MESSAGE_CHARS)); + output["last_agent_message"] = json!(last); + } + Ok(ToolOutcome { + output: output.to_string(), + effects: vec![ToolEffect::Read(id)], + }) + } + + fn state_label(&self, id: &str, state: Option<&SessionActivityState>) -> String { + if let Some(waiting) = self.open_requests.waiting_for(id) { + return waiting; + } + match state { + None | Some(SessionActivityState::Idle) => "idle".into(), + Some(SessionActivityState::Errored { message }) => format!("errored: {message}"), + Some(SessionActivityState::RunningExternally) => "running in another window".into(), + Some( + SessionActivityState::AgentRunning + | SessionActivityState::WaitingForResponse + | SessionActivityState::RateLimited { .. }, + ) => "running".into(), + } + } +} + +/// One exchange: a user message and the agent's last text before the next +/// user message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Turn { + pub user: Option, + pub agent: Option, +} + +pub fn group_turns(items: &[ContentItem]) -> Vec { + let mut turns: Vec = Vec::new(); + for item in items { + match item.kind { + ContentKind::UserText => turns.push(Turn { + user: Some(item.text.clone()), + agent: None, + }), + ContentKind::AssistantText => { + if item.text.trim().is_empty() { + continue; + } + match turns.last_mut() { + Some(turn) => turn.agent = Some(item.text.clone()), + None => turns.push(Turn { + user: None, + agent: Some(item.text.clone()), + }), + } + } + _ => {} + } + } + turns +} + +/// The last `count` turns as JSON, each message truncated, oldest dropped +/// first when the total exceeds [`MAX_TOTAL_CHARS`]. +fn last_turns(turns: &[Turn], count: usize) -> Vec { + let start = turns.len().saturating_sub(count); + let mut selected: Vec<(Option, Option)> = turns[start..] + .iter() + .map(|turn| { + ( + turn.user.as_deref().map(|t| truncate(t, MAX_MESSAGE_CHARS)), + turn.agent + .as_deref() + .map(|t| truncate(t, MAX_MESSAGE_CHARS)), + ) + }) + .collect(); + let size = |turns: &[(Option, Option)]| -> usize { + turns + .iter() + .map(|(u, a)| u.as_ref().map_or(0, |t| t.len()) + a.as_ref().map_or(0, |t| t.len())) + .sum() + }; + while selected.len() > 1 && size(&selected) > MAX_TOTAL_CHARS { + selected.remove(0); + } + selected + .into_iter() + .map(|(user, agent)| json!({ "user": user, "agent": agent })) + .collect() +} + +fn truncate(text: &str, max_chars: usize) -> String { + if text.chars().count() <= max_chars { + return text.to_string(); + } + let cut: String = text.chars().take(max_chars).collect(); + format!("{cut}… [truncated]") +} + +fn relative_time(now: SystemTime, then: SystemTime) -> String { + let seconds = now.duration_since(then).map(|d| d.as_secs()).unwrap_or(0); + match seconds { + 0..60 => "just now".into(), + 60..3600 => format!("{} minutes ago", seconds / 60), + 3600..86_400 => format!("{} hours ago", seconds / 3600), + _ => format!("{} days ago", seconds / 86_400), + } +} + +/// The metadata of a conversation, for notifications. +pub(super) fn find_metadata<'a>( + sessions: &'a [ChatMetadata], + id: &str, +) -> Option<&'a ChatMetadata> { + sessions.iter().find(|s| s.id == id) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::persistence::NodeId; + use crate::session_query::Role; + + fn item(kind: ContentKind, text: &str) -> ContentItem { + ContentItem { + message_index: 0, + node_id: NodeId::default(), + role: if kind == ContentKind::UserText { + Role::User + } else { + Role::Assistant + }, + kind, + text: text.into(), + tool_name: None, + tool_input: None, + is_error: None, + truncated: false, + } + } + + #[test] + fn turns_keep_only_the_last_agent_text() { + let items = vec![ + item(ContentKind::UserText, "fix it"), + item(ContentKind::AssistantText, "looking"), + item(ContentKind::AssistantText, "fixed"), + item(ContentKind::UserText, "thanks"), + ]; + assert_eq!( + group_turns(&items), + vec![ + Turn { + user: Some("fix it".into()), + agent: Some("fixed".into()) + }, + Turn { + user: Some("thanks".into()), + agent: None + }, + ] + ); + } + + #[test] + fn last_turns_are_limited_in_count_and_size() { + let turns: Vec = (0..15) + .map(|i| Turn { + user: Some(format!("q{i}")), + agent: Some("a".repeat(5_000)), + }) + .collect(); + let selected = last_turns(&turns, FULL_TURNS); + // 2 000 chars per message (+ marker) caps at 7 turns of 16 000. + assert!(selected.len() <= FULL_TURNS); + assert_eq!(selected.last().unwrap()["user"], "q14"); + let total: usize = selected + .iter() + .map(|t| t["agent"].as_str().unwrap().len() + t["user"].as_str().unwrap().len()) + .sum(); + assert!(total <= MAX_TOTAL_CHARS); + assert!( + selected[0]["agent"] + .as_str() + .unwrap() + .ends_with("[truncated]") + ); + + let few = last_turns(&turns[..3], FULL_TURNS); + assert_eq!(few.len(), 3); + } + + #[test] + fn truncation_counts_characters() { + assert_eq!(truncate("äöü", 3), "äöü"); + assert_eq!(truncate("äöüß", 3), "äöü… [truncated]"); + } +} diff --git a/crates/llm/src/factory.rs b/crates/llm/src/factory.rs index 0794786c..29a7dadf 100644 --- a/crates/llm/src/factory.rs +++ b/crates/llm/src/factory.rs @@ -366,12 +366,12 @@ pub async fn create_llm_client_from_configs( /// } /// ``` #[derive(Debug)] -struct AiCoreDeployment { - deployment_uuid: String, - api_type: AiCoreApiType, +pub(crate) struct AiCoreDeployment { + pub(crate) deployment_uuid: String, + pub(crate) api_type: AiCoreApiType, } -fn parse_aicore_deployment(model_id: &str, value: &Value) -> Result { +pub(crate) fn parse_aicore_deployment(model_id: &str, value: &Value) -> Result { // Try simple string format first (backwards compatible) if let Some(uuid) = value.as_str() { return Ok(AiCoreDeployment { diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index 016de370..a7d4abbc 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -32,6 +32,7 @@ pub mod openai_responses_ws; pub mod openrouter; pub mod prompt_caching; pub mod provider_config; +pub mod realtime; pub mod recording; pub mod streaming; pub mod types; diff --git a/crates/llm/src/realtime/aicore.rs b/crates/llm/src/realtime/aicore.rs new file mode 100644 index 00000000..7d102ddd --- /dev/null +++ b/crates/llm/src/realtime/aicore.rs @@ -0,0 +1,175 @@ +//! Realtime sessions through AI Core: OAuth client credentials from the +//! provider config, the deployment from its `models` map. +//! +//! The WebSocket endpoint is the deployment's `deploymentUrl` (read from the +//! deployment resource on every connect, so a token refresh or redeployment +//! is picked up on reconnect) plus `/v1/realtime`. + +use super::transport::{RealtimeEndpoint, WsConnector}; +use super::{RealtimeConnection, RealtimeConnector}; +use crate::auth::TokenManager; +use anyhow::{Context, Result, anyhow}; +use std::sync::Arc; +use tokio::sync::OnceCell; +use tracing::info; + +const RESOURCE_GROUP: &str = "default"; + +pub struct AiCoreConnector { + client_id: String, + client_secret: String, + token_url: String, + /// `https://…/v2/inference`, as in the provider config. + api_base_url: String, + deployment_id: String, + /// Created on the first connect (it fetches a token right away). + token_manager: OnceCell>, +} + +impl AiCoreConnector { + pub fn new( + client_id: String, + client_secret: String, + token_url: String, + api_base_url: String, + deployment_id: String, + ) -> Self { + Self { + client_id, + client_secret, + token_url, + api_base_url, + deployment_id, + token_manager: OnceCell::new(), + } + } + + async fn token(&self) -> Result { + let manager = self + .token_manager + .get_or_try_init(|| { + TokenManager::new( + self.client_id.clone(), + self.client_secret.clone(), + self.token_url.clone(), + ) + }) + .await + .context("AI Core authentication failed")?; + manager.get_valid_token().await + } + + /// The deployment resource's `deploymentUrl`. + async fn deployment_url(&self, token: &str) -> Result { + let url = format!( + "{}/deployments/{}", + lm_base_url(&self.api_base_url), + self.deployment_id + ); + let response: serde_json::Value = reqwest::Client::new() + .get(&url) + .bearer_auth(token) + .header("AI-Resource-Group", RESOURCE_GROUP) + .send() + .await + .with_context(|| format!("Failed to read the AI Core deployment {url}"))? + .error_for_status() + .with_context(|| format!("AI Core deployment {} not readable", self.deployment_id))? + .json() + .await?; + response + .get("deploymentUrl") + .and_then(|v| v.as_str()) + .filter(|url| !url.is_empty()) + .map(str::to_string) + .ok_or_else(|| { + anyhow!( + "AI Core deployment {} has no URL (status: {})", + self.deployment_id, + response + .get("status") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + ) + }) + } +} + +#[async_trait::async_trait] +impl RealtimeConnector for AiCoreConnector { + async fn connect(&self) -> Result { + let token = self.token().await?; + let deployment_url = self.deployment_url(&token).await?; + let url = realtime_url(&deployment_url); + info!("AI Core realtime endpoint: {url}"); + WsConnector::new(RealtimeEndpoint { + url, + headers: vec![ + ("Authorization".to_string(), format!("Bearer {token}")), + ("AI-Resource-Group".to_string(), RESOURCE_GROUP.to_string()), + ], + }) + .connect() + .await + } + + fn declares_model(&self) -> bool { + false + } +} + +/// Where deployment resources live: `:///v2/lm`, whatever +/// path the configured inference URL has (`…/v2/inference`, `…/inference`). +fn lm_base_url(api_base_url: &str) -> String { + let base = api_base_url.trim_end_matches('/'); + let host_end = base + .find("://") + .map(|scheme| { + let rest = scheme + 3; + base[rest..] + .find('/') + .map_or(base.len(), |slash| rest + slash) + }) + .unwrap_or(base.len()); + format!("{}/v2/lm", &base[..host_end]) +} + +fn realtime_url(deployment_url: &str) -> String { + let base = deployment_url.trim_end_matches('/'); + let base = if let Some(rest) = base.strip_prefix("https://") { + format!("wss://{rest}") + } else if let Some(rest) = base.strip_prefix("http://") { + format!("ws://{rest}") + } else { + base.to_string() + }; + format!("{base}/v1/realtime") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deployment_resources_live_under_v2_lm_on_the_api_host() { + for base in [ + "https://api.ai.example.com/v2/inference/", + "https://api.ai.example.com/inference", + "https://api.ai.example.com", + ] { + assert_eq!( + lm_base_url(base), + "https://api.ai.example.com/v2/lm", + "{base}" + ); + } + } + + #[test] + fn realtime_url_appends_the_path_on_a_websocket_scheme() { + assert_eq!( + realtime_url("https://api.ai.example.com/v2/inference/deployments/d1"), + "wss://api.ai.example.com/v2/inference/deployments/d1/v1/realtime" + ); + } +} diff --git a/crates/llm/src/realtime/events.rs b/crates/llm/src/realtime/events.rs new file mode 100644 index 00000000..dd0b3eb2 --- /dev/null +++ b/crates/llm/src/realtime/events.rs @@ -0,0 +1,304 @@ +//! Wire types of the OpenAI Realtime protocol: the client events we send and +//! the subset of server events a voice agent reacts to. +//! +//! Server events not listed here parse as [`ServerEvent::Other`]; the +//! protocol sends many informational events (rate limits, content part +//! boundaries, buffer commits) a client can ignore. + +use base64::Engine; +use serde::Deserialize; +use serde_json::{Value, json}; + +/// Sample rate of the PCM16 mono audio both directions use. +pub const SAMPLE_RATE: u32 = 24_000; + +/// A function the model may call, as declared in the session. +#[derive(Debug, Clone, PartialEq)] +pub struct ToolDefinition { + pub name: String, + pub description: String, + /// JSON schema of the arguments object. + pub parameters: Value, +} + +/// The session settings sent with `session.update`. +#[derive(Debug, Clone, PartialEq)] +pub struct SessionSettings { + /// `None` when the endpoint routes to a fixed model. + pub model: Option, + pub instructions: String, + pub voice: String, + pub tools: Vec, + /// The `turn_detection` object, e.g. `{"type": "semantic_vad"}`. + pub turn_detection: Value, + /// Model transcribing the user's speech; `None` turns transcription off. + pub transcription_model: Option, +} + +/// An event the client sends. +#[derive(Debug, Clone, PartialEq)] +pub enum ClientEvent { + SessionUpdate(SessionSettings), + /// Microphone samples (PCM16, 24 kHz, mono). + AppendAudio(Vec), + /// Ask the model to respond to the conversation as it stands. + CreateResponse, + /// Stop the response being generated. + CancelResponse, + /// Cut an assistant audio item down to what the user actually heard. + TruncateItem { + item_id: String, + audio_end_ms: u64, + }, + /// The result of a function call. + FunctionCallOutput { + call_id: String, + output: String, + }, + /// A text item from the system: context the model did not hear. + SystemMessage(String), +} + +impl ClientEvent { + pub fn to_json(&self) -> Value { + match self { + ClientEvent::SessionUpdate(settings) => { + let tools: Vec = settings + .tools + .iter() + .map(|tool| { + json!({ + "type": "function", + "name": tool.name, + "description": tool.description, + "parameters": tool.parameters, + }) + }) + .collect(); + let mut input = json!({ + "format": { "type": "audio/pcm", "rate": SAMPLE_RATE }, + "turn_detection": settings.turn_detection, + }); + if let Some(model) = &settings.transcription_model { + input["transcription"] = json!({ "model": model }); + } + let mut event = json!({ + "type": "session.update", + "session": { + "type": "realtime", + "instructions": settings.instructions, + "output_modalities": ["audio"], + "audio": { + "input": input, + "output": { + "format": { "type": "audio/pcm", "rate": SAMPLE_RATE }, + "voice": settings.voice, + }, + }, + "tools": tools, + "tool_choice": "auto", + }, + }); + if let Some(model) = &settings.model { + event["session"]["model"] = json!(model); + } + event + } + ClientEvent::AppendAudio(samples) => json!({ + "type": "input_audio_buffer.append", + "audio": encode_pcm16(samples), + }), + ClientEvent::CreateResponse => json!({ "type": "response.create" }), + ClientEvent::CancelResponse => json!({ "type": "response.cancel" }), + ClientEvent::TruncateItem { + item_id, + audio_end_ms, + } => json!({ + "type": "conversation.item.truncate", + "item_id": item_id, + "content_index": 0, + "audio_end_ms": audio_end_ms, + }), + ClientEvent::FunctionCallOutput { call_id, output } => json!({ + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": call_id, + "output": output, + }, + }), + ClientEvent::SystemMessage(text) => json!({ + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "system", + "content": [{ "type": "input_text", "text": text }], + }, + }), + } + } +} + +/// An item of the conversation as the server reports it. +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct ConversationItem { + #[serde(default)] + pub id: String, + #[serde(rename = "type")] + pub kind: String, + #[serde(default)] + pub name: Option, + #[serde(default)] + pub call_id: Option, + #[serde(default)] + pub arguments: Option, +} + +/// The response a `response.created` / `response.done` event is about. +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct ResponseInfo { + #[serde(default)] + pub id: String, + /// `completed`, `cancelled`, `failed` or `incomplete` once done. + #[serde(default)] + pub status: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct ErrorInfo { + #[serde(default)] + pub message: String, + #[serde(default)] + pub code: Option, +} + +/// A server event. Event names follow the GA protocol; the beta names of +/// the audio events are accepted as aliases. +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(tag = "type")] +pub enum ServerEvent { + #[serde(rename = "session.created")] + SessionCreated, + #[serde(rename = "session.updated")] + SessionUpdated, + #[serde(rename = "response.created")] + ResponseCreated { response: ResponseInfo }, + #[serde(rename = "response.done")] + ResponseDone { response: ResponseInfo }, + #[serde(rename = "response.output_item.added")] + OutputItemAdded { item: ConversationItem }, + #[serde(rename = "response.output_item.done")] + OutputItemDone { item: ConversationItem }, + #[serde(rename = "response.output_audio.delta", alias = "response.audio.delta")] + AudioDelta { item_id: String, delta: String }, + #[serde( + rename = "response.output_audio_transcript.done", + alias = "response.audio_transcript.done" + )] + AudioTranscriptDone { item_id: String, transcript: String }, + #[serde(rename = "input_audio_buffer.speech_started")] + SpeechStarted, + #[serde(rename = "input_audio_buffer.speech_stopped")] + SpeechStopped, + #[serde(rename = "conversation.item.input_audio_transcription.completed")] + InputTranscription { item_id: String, transcript: String }, + #[serde(rename = "error")] + Error { error: ErrorInfo }, + #[serde(other)] + Other, +} + +impl ServerEvent { + /// Parse a text frame. A known event with an unexpected shape is + /// reported as an error rather than silently dropped. + pub fn parse(text: &str) -> anyhow::Result { + Ok(serde_json::from_str(text)?) + } +} + +pub fn encode_pcm16(samples: &[i16]) -> String { + let mut bytes = Vec::with_capacity(samples.len() * 2); + for sample in samples { + bytes.extend_from_slice(&sample.to_le_bytes()); + } + base64::engine::general_purpose::STANDARD.encode(bytes) +} + +pub fn decode_pcm16(data: &str) -> anyhow::Result> { + let bytes = base64::engine::general_purpose::STANDARD.decode(data)?; + Ok(bytes + .as_chunks::<2>() + .0 + .iter() + .map(|pair| i16::from_le_bytes(*pair)) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pcm16_round_trips() { + let samples = vec![0, 1, -1, i16::MAX, i16::MIN, 1234]; + assert_eq!(decode_pcm16(&encode_pcm16(&samples)).unwrap(), samples); + } + + #[test] + fn parses_audio_delta_under_both_names() { + for name in ["response.output_audio.delta", "response.audio.delta"] { + let text = + format!(r#"{{"type":"{name}","item_id":"i1","delta":"AAA=","response_id":"r"}}"#); + assert_eq!( + ServerEvent::parse(&text).unwrap(), + ServerEvent::AudioDelta { + item_id: "i1".into(), + delta: "AAA=".into() + } + ); + } + } + + #[test] + fn unknown_events_parse_as_other() { + let event = ServerEvent::parse(r#"{"type":"rate_limits.updated","rate_limits":[]}"#); + assert_eq!(event.unwrap(), ServerEvent::Other); + } + + #[test] + fn parses_function_call_items() { + let text = r#"{"type":"response.output_item.done","item":{"id":"it","type":"function_call", + "name":"list_conversations","call_id":"c1","arguments":"{}","status":"completed"}}"#; + let ServerEvent::OutputItemDone { item } = ServerEvent::parse(text).unwrap() else { + panic!("wrong event"); + }; + assert_eq!(item.kind, "function_call"); + assert_eq!(item.call_id.as_deref(), Some("c1")); + assert_eq!(item.name.as_deref(), Some("list_conversations")); + } + + #[test] + fn session_update_declares_tools_and_audio_formats() { + let event = ClientEvent::SessionUpdate(SessionSettings { + model: Some("gpt-realtime".into()), + instructions: "be brief".into(), + voice: "marin".into(), + tools: vec![ToolDefinition { + name: "t".into(), + description: "d".into(), + parameters: json!({"type": "object"}), + }], + turn_detection: json!({"type": "semantic_vad"}), + transcription_model: Some("gpt-4o-mini-transcribe".into()), + }) + .to_json(); + assert_eq!(event["session"]["tools"][0]["name"], "t"); + assert_eq!(event["session"]["model"], "gpt-realtime"); + assert_eq!(event["session"]["audio"]["input"]["format"]["rate"], 24000); + assert_eq!(event["session"]["audio"]["output"]["voice"], "marin"); + assert_eq!( + event["session"]["audio"]["input"]["transcription"]["model"], + "gpt-4o-mini-transcribe" + ); + } +} diff --git a/crates/llm/src/realtime/mod.rs b/crates/llm/src/realtime/mod.rs new file mode 100644 index 00000000..52148854 --- /dev/null +++ b/crates/llm/src/realtime/mod.rs @@ -0,0 +1,128 @@ +//! Realtime (speech-to-speech) model sessions over the OpenAI Realtime +//! protocol. +//! +//! A realtime session is one long-lived WebSocket: audio streams both ways, +//! the server detects turns and keeps the conversation state. This module +//! only moves typed events; deciding *when* to send what is the caller's job +//! (see the voice agent in `code_assistant_core::voice`). +//! +//! [`RealtimeConnector`] is the seam: the WebSocket implementation is +//! [`WsConnector`], tests hand out channel pairs of their own. + +mod aicore; +mod events; +mod transport; + +pub use aicore::AiCoreConnector; +pub use events::{ + ClientEvent, ConversationItem, ErrorInfo, ResponseInfo, SAMPLE_RATE, ServerEvent, + SessionSettings, ToolDefinition, decode_pcm16, encode_pcm16, +}; +pub use transport::{RealtimeEndpoint, WsConnector}; + +use crate::provider_config::ProviderConfig; +use anyhow::{Context, Result, anyhow, bail}; +use serde_json::Value; +use std::sync::Arc; +use tokio::sync::mpsc; + +/// What arrives from an open session. +#[derive(Debug, Clone, PartialEq)] +pub enum Incoming { + Event(ServerEvent), + /// The connection ended; carries the reason when one is known. Nothing + /// follows. + Closed(Option), +} + +/// An open realtime session as a pair of channels. Dropping `outgoing` +/// closes the connection. +pub struct RealtimeConnection { + pub outgoing: mpsc::UnboundedSender, + pub incoming: mpsc::UnboundedReceiver, +} + +/// Opens realtime sessions; called again to reconnect. +#[async_trait::async_trait] +pub trait RealtimeConnector: Send + Sync { + async fn connect(&self) -> Result; + + /// Whether `session.update` names the model. A gateway that routes to a + /// deployment (AI Core) knows the model already. + fn declares_model(&self) -> bool { + true + } +} + +/// Provider types that can serve realtime sessions. +pub const REALTIME_PROVIDER_TYPES: &[&str] = &[ + "openai", + "openai-responses", + "openai-responses-ws", + "ai-core", +]; + +const DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; + +/// Whether a provider can serve realtime sessions. +pub fn supports_realtime(provider: &ProviderConfig) -> bool { + REALTIME_PROVIDER_TYPES.contains(&provider.provider.as_str()) +} + +/// The connector for `model` on a provider from `providers.json`. +/// +/// - OpenAI types: `api_key` auth, URL derived from `base_url`. +/// - `ai-core`: client credentials; `model` must be a key of the provider's +/// `models` map, whose deployment serves the session. +/// +/// `url_override` replaces the derived URL (OpenAI types only). +pub fn connector_for_provider( + provider: &ProviderConfig, + model: &str, + url_override: Option<&str>, +) -> Result> { + let config = &provider.config; + let text = |key: &str| -> Result { + config + .get(key) + .and_then(Value::as_str) + .map(str::to_string) + .with_context(|| format!("The provider '{}' has no {key}", provider.label)) + }; + match provider.provider.as_str() { + "openai" | "openai-responses" | "openai-responses-ws" => { + let api_key = text("api_key")?; + let base_url = text("base_url").unwrap_or_else(|_| DEFAULT_OPENAI_BASE_URL.into()); + let mut endpoint = RealtimeEndpoint::openai(&base_url, &api_key, model); + if let Some(url) = url_override.filter(|url| !url.is_empty()) { + endpoint.url = url.to_string(); + } + Ok(Arc::new(WsConnector::new(endpoint))) + } + "ai-core" => { + let deployment = config + .get("models") + .and_then(|models| models.get(model)) + .ok_or_else(|| { + anyhow!( + "The AI Core provider '{}' has no deployment for model '{model}' \ + (add it to the provider's models)", + provider.label + ) + })?; + let deployment = crate::factory::parse_aicore_deployment(model, deployment)?; + Ok(Arc::new(AiCoreConnector::new( + text("client_id")?, + text("client_secret")?, + text("token_url")?, + text("api_base_url")?, + deployment.deployment_uuid, + ))) + } + other => bail!( + "Provider '{}' ({other}) has no realtime API; voice mode needs an OpenAI or AI Core \ + provider", + provider.label + ), + } +} diff --git a/crates/llm/src/realtime/transport.rs b/crates/llm/src/realtime/transport.rs new file mode 100644 index 00000000..e569b343 --- /dev/null +++ b/crates/llm/src/realtime/transport.rs @@ -0,0 +1,159 @@ +//! The WebSocket transport of a realtime session. + +use super::{ClientEvent, Incoming, RealtimeConnection, RealtimeConnector, ServerEvent}; +use anyhow::{Context, Result}; +use futures_util::{SinkExt, StreamExt}; +use tokio::sync::mpsc; +use tokio_tungstenite::{ + connect_async_with_config, + tungstenite::{ + client::IntoClientRequest, + http::{HeaderName, HeaderValue}, + protocol::Message as WsMessage, + }, +}; +use tracing::{debug, info, warn}; + +/// Where and how to open a realtime session. +#[derive(Clone)] +pub struct RealtimeEndpoint { + pub url: String, + pub headers: Vec<(String, String)>, +} + +impl std::fmt::Debug for RealtimeEndpoint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Header values carry credentials. + f.debug_struct("RealtimeEndpoint") + .field("url", &self.url) + .field( + "headers", + &self.headers.iter().map(|(k, _)| k).collect::>(), + ) + .finish() + } +} + +impl RealtimeEndpoint { + /// The endpoint of an OpenAI-compatible API: `base_url` is the HTTP base + /// (`https://api.openai.com/v1`), the session lives at + /// `wss://…/v1/realtime?model=`. + pub fn openai(base_url: &str, api_key: &str, model: &str) -> Self { + let base = base_url.trim_end_matches('/'); + let ws_base = if let Some(rest) = base.strip_prefix("https://") { + format!("wss://{rest}") + } else if let Some(rest) = base.strip_prefix("http://") { + format!("ws://{rest}") + } else { + base.to_string() + }; + Self { + url: format!("{ws_base}/realtime?model={}", urlencoding::encode(model)), + headers: vec![("Authorization".to_string(), format!("Bearer {api_key}"))], + } + } +} + +/// Opens realtime sessions over WebSocket. +pub struct WsConnector { + endpoint: RealtimeEndpoint, +} + +impl WsConnector { + pub fn new(endpoint: RealtimeEndpoint) -> Self { + Self { endpoint } + } +} + +#[async_trait::async_trait] +impl RealtimeConnector for WsConnector { + async fn connect(&self) -> Result { + let mut request = self + .endpoint + .url + .as_str() + .into_client_request() + .context("Failed to build the realtime WebSocket request")?; + for (key, value) in &self.endpoint.headers { + request.headers_mut().insert( + HeaderName::from_bytes(key.as_bytes()).context("Invalid header name")?, + HeaderValue::from_str(value).context("Invalid header value")?, + ); + } + info!("Connecting realtime session to {}", self.endpoint.url); + let (stream, _response) = connect_async_with_config(request, None, false) + .await + .with_context(|| format!("Realtime connection to {} failed", self.endpoint.url))?; + let (mut sink, mut source) = stream.split(); + + let (outgoing_tx, mut outgoing_rx) = mpsc::unbounded_channel::(); + let (incoming_tx, incoming_rx) = mpsc::unbounded_channel::(); + let (pong_tx, mut pong_rx) = mpsc::unbounded_channel::>(); + + // Writer: client events and pongs. Ends when the caller drops its + // sender, which closes the socket. + tokio::spawn(async move { + loop { + let message = tokio::select! { + event = outgoing_rx.recv() => match event { + Some(event) => WsMessage::Text(event.to_json().to_string().into()), + None => break, + }, + Some(payload) = pong_rx.recv() => WsMessage::Pong(payload.into()), + }; + if let Err(e) = sink.send(message).await { + warn!("Realtime send failed: {e}"); + return; + } + } + let _ = sink.send(WsMessage::Close(None)).await; + }); + + // Reader: server events until the socket closes. + tokio::spawn(async move { + let reason = loop { + match source.next().await { + Some(Ok(WsMessage::Text(text))) => match ServerEvent::parse(&text) { + Ok(event) => { + if incoming_tx.send(Incoming::Event(event)).is_err() { + return; + } + } + Err(e) => debug!("Unparsed realtime event ({e}): {text}"), + }, + Some(Ok(WsMessage::Ping(payload))) => { + let _ = pong_tx.send(payload.to_vec()); + } + Some(Ok(WsMessage::Close(frame))) => { + break frame.map(|f| format!("{} {}", f.code, f.reason)); + } + Some(Ok(_)) => {} + Some(Err(e)) => break Some(e.to_string()), + None => break None, + } + }; + let _ = incoming_tx.send(Incoming::Closed(reason)); + }); + + Ok(RealtimeConnection { + outgoing: outgoing_tx, + incoming: incoming_rx, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn openai_endpoint_derives_the_websocket_url() { + let endpoint = RealtimeEndpoint::openai("https://api.openai.com/v1/", "k", "gpt-realtime"); + assert_eq!( + endpoint.url, + "wss://api.openai.com/v1/realtime?model=gpt-realtime" + ); + assert_eq!(endpoint.headers[0].1, "Bearer k"); + assert!(!format!("{endpoint:?}").contains("Bearer")); + } +} diff --git a/crates/ui_acp/src/ui.rs b/crates/ui_acp/src/ui.rs index a4f784ba..dff2745d 100644 --- a/crates/ui_acp/src/ui.rs +++ b/crates/ui_acp/src/ui.rs @@ -997,6 +997,9 @@ impl UserInterface for ACPUserUI { UiEvent::ConfigChanged => { // Config file changes not relevant in ACP mode } + UiEvent::VoiceStatusChanged { .. } | UiEvent::VoiceTranscript { .. } => { + // Voice mode is GPUI-specific. + } } Ok(()) } diff --git a/crates/ui_gpui/assets/icons/audio_lines.svg b/crates/ui_gpui/assets/icons/audio_lines.svg new file mode 100644 index 00000000..da57dfcc --- /dev/null +++ b/crates/ui_gpui/assets/icons/audio_lines.svg @@ -0,0 +1 @@ + diff --git a/crates/ui_gpui/assets/icons/mic.svg b/crates/ui_gpui/assets/icons/mic.svg new file mode 100644 index 00000000..a23c62b2 --- /dev/null +++ b/crates/ui_gpui/assets/icons/mic.svg @@ -0,0 +1 @@ + diff --git a/crates/ui_gpui/assets/icons/mic_off.svg b/crates/ui_gpui/assets/icons/mic_off.svg new file mode 100644 index 00000000..27d951da --- /dev/null +++ b/crates/ui_gpui/assets/icons/mic_off.svg @@ -0,0 +1 @@ + diff --git a/crates/ui_gpui/src/app/event_loop.rs b/crates/ui_gpui/src/app/event_loop.rs index 3684301b..3b8bfa3b 100644 --- a/crates/ui_gpui/src/app/event_loop.rs +++ b/crates/ui_gpui/src/app/event_loop.rs @@ -1113,8 +1113,19 @@ impl Gpui { cx.refresh(); } + UiEvent::VoiceStatusChanged { status } => { + // A failure shows in the title bar's voice controls, not in + // the error popover: that belongs to the viewed session. + self.apply_voice_status(status); + cx.refresh(); + } + UiEvent::VoiceTranscript { entry } => { + self.append_voice_transcript(entry); + cx.refresh(); + } UiEvent::ConfigChanged => { debug!("UI: ConfigChanged event — config files modified on disk"); + self.refresh_voice_configured(); self.config_generation .fetch_add(1, std::sync::atomic::Ordering::Relaxed); diff --git a/crates/ui_gpui/src/app/mod.rs b/crates/ui_gpui/src/app/mod.rs index ff53f70a..d8d6fa11 100644 --- a/crates/ui_gpui/src/app/mod.rs +++ b/crates/ui_gpui/src/app/mod.rs @@ -9,3 +9,4 @@ pub(super) mod commands; pub(super) mod drafts; pub(super) mod event_bridge; pub(super) mod event_loop; +pub(super) mod voice; diff --git a/crates/ui_gpui/src/app/voice.rs b/crates/ui_gpui/src/app/voice.rs new file mode 100644 index 00000000..aaea2b59 --- /dev/null +++ b/crates/ui_gpui/src/app/voice.rs @@ -0,0 +1,81 @@ +//! Voice mode state mirrored from the core, and the commands to drive it. + +use crate::Gpui; +use code_assistant_core::voice::{TranscriptEntry, VoiceConfig, VoiceService, VoiceStatus}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +/// Transcript lines kept for the title-bar popover. +const TRANSCRIPT_LIMIT: usize = 200; + +#[derive(Clone, Default)] +pub struct VoiceState { + service: Arc>>, + status: Arc>, + transcript: Arc>>, + /// Whether `voice.json` names a provider and model. + configured: Arc, +} + +impl Gpui { + /// Install the voice service; without one the title bar shows no voice + /// control. + pub fn set_voice_service(&self, service: VoiceService) { + *self.voice.service.lock().unwrap() = Some(service); + self.refresh_voice_configured(); + } + + pub fn voice_available(&self) -> bool { + self.voice.service.lock().unwrap().is_some() + } + + /// Whether a voice provider and model are configured. + pub fn voice_configured(&self) -> bool { + self.voice.configured.load(Ordering::Relaxed) + } + + /// Re-read `voice.json` (after the settings or the file changed). + pub fn refresh_voice_configured(&self) { + self.voice + .configured + .store(VoiceConfig::load().is_configured(), Ordering::Relaxed); + } + + pub fn voice_status(&self) -> VoiceStatus { + self.voice.status.lock().unwrap().clone() + } + + pub fn voice_transcript(&self) -> Vec { + self.voice.transcript.lock().unwrap().clone() + } + + /// Start voice mode, or stop it when it runs. + pub fn toggle_voice(&self) { + let Some(service) = self.voice.service.lock().unwrap().clone() else { + return; + }; + if self.voice_status().activity.is_active() { + service.stop(); + } else { + self.voice.transcript.lock().unwrap().clear(); + service.start(); + } + } + + pub fn set_voice_muted(&self, muted: bool) { + if let Some(service) = self.voice.service.lock().unwrap().as_ref() { + service.set_muted(muted); + } + } + + pub(crate) fn apply_voice_status(&self, status: VoiceStatus) { + *self.voice.status.lock().unwrap() = status; + } + + pub(crate) fn append_voice_transcript(&self, entry: TranscriptEntry) { + let mut transcript = self.voice.transcript.lock().unwrap(); + transcript.push(entry); + let excess = transcript.len().saturating_sub(TRANSCRIPT_LIMIT); + transcript.drain(..excess); + } +} diff --git a/crates/ui_gpui/src/lib.rs b/crates/ui_gpui/src/lib.rs index 1ea9e0b0..7e307d3b 100644 --- a/crates/ui_gpui/src/lib.rs +++ b/crates/ui_gpui/src/lib.rs @@ -345,6 +345,10 @@ pub struct Gpui { /// input-area completion and submit-time invocation. Refreshed on /// session load via [`Gpui::refresh_skills`]. skills: Arc>>, + + /// Voice mode: the handle (absent when the build has no audio), its + /// status and the transcript of the current voice conversation. + voice: app::voice::VoiceState, } /// State for a pending message edit (for branching) @@ -655,6 +659,8 @@ impl Gpui { config_generation: Arc::new(std::sync::atomic::AtomicU64::new(0)), skills: Arc::new(Mutex::new(Vec::new())), + + voice: app::voice::VoiceState::default(), } } diff --git a/crates/ui_gpui/src/main_screen/mod.rs b/crates/ui_gpui/src/main_screen/mod.rs index 016506ce..77c8d81e 100644 --- a/crates/ui_gpui/src/main_screen/mod.rs +++ b/crates/ui_gpui/src/main_screen/mod.rs @@ -3,6 +3,7 @@ pub mod project_dialog; mod question_prompt; pub mod right_panel; mod status_popover; +mod voice_control; use crate::shared::{frame_profile, ui_state}; use crate::sidebar::{SessionSidebar, SessionSidebarEvent}; @@ -208,6 +209,8 @@ pub struct MainScreen { /// Form for the viewed session's oldest open `ask_question` request. question_prompt: Option>, + /// Whether the voice transcript popover below the title bar is open. + voice_transcript_open: bool, /// Whether the sidebar got the first session listing. Without it, an /// empty listing equals the initial `sessions` and never reaches the /// sidebar, which then shows no project folders. @@ -350,6 +353,7 @@ impl MainScreen { ui_scale: initial_scale, right_sidebar_width: px(initial_sidebar_width), right_sidebar_resizing: false, + voice_transcript_open: false, sidebar_sessions_synced: false, resize_start_x: 0.0, resize_start_width: 0.0, @@ -2102,6 +2106,8 @@ impl Render for MainScreen { ), ) }) + // Voice mode, next to the right panel views + .children(self.render_voice_controls(cx)) .child( div() .id("about-btn") @@ -2300,6 +2306,8 @@ impl Render for MainScreen { } }), ) + // Voice transcript popover below the title bar + .children(self.render_voice_transcript(cx)) // Modal dialog overlay for new project creation .when_some(new_project_dialog, |el, dialog| el.child(dialog)) // Modal "About" dialog overlay diff --git a/crates/ui_gpui/src/main_screen/voice_control.rs b/crates/ui_gpui/src/main_screen/voice_control.rs new file mode 100644 index 00000000..b6d828da --- /dev/null +++ b/crates/ui_gpui/src/main_screen/voice_control.rs @@ -0,0 +1,243 @@ +//! Voice mode in the title bar: start/stop, mute, the voice agent's state, +//! and a popover with the voice conversation's transcript. + +use super::MainScreen; +use crate::Gpui; +use code_assistant_core::voice::{TranscriptRole, VoiceActivity}; +use gpui_kit::component::tooltip::Tooltip; +use gpui_kit::component::{ActiveTheme, Icon, Sizable, Size}; +use gpui_kit::{AnyElement, Context, SharedString, div, prelude::*, px}; + +impl MainScreen { + /// The title-bar controls, or `None` when the build has no voice mode. + pub(super) fn render_voice_controls(&self, cx: &mut Context) -> Option { + let gpui = cx.try_global::()?; + if !gpui.voice_available() { + return None; + } + let status = gpui.voice_status(); + let active = status.activity.is_active(); + let failure = match &status.activity { + VoiceActivity::Failed(message) => Some(SharedString::from(message.clone())), + _ => None, + }; + let failed = failure.is_some(); + // Without a voice model there is nothing to start; a running voice + // session stays controllable either way. + if !gpui.voice_configured() && !active && !failed { + return None; + } + + let mic_color = if failed { + cx.theme().danger + } else if active { + cx.theme().primary + } else { + cx.theme().muted_foreground + }; + let mic_tooltip = if active { + "Stop voice mode" + } else { + "Start voice mode" + }; + let mic = div() + .id("voice-toggle-btn") + .size(px(28.)) + .rounded_sm() + .flex() + .items_center() + .justify_center() + .cursor_pointer() + .hover(|s| s.bg(cx.theme().muted)) + .when(active, |el| el.bg(cx.theme().muted)) + .child( + Icon::default() + .path(SharedString::from("icons/mic.svg")) + .with_size(Size::Small) + .text_color(mic_color), + ) + .tooltip(move |window, cx| Tooltip::new(mic_tooltip).build(window, cx)) + .on_click(cx.listener(|this, _, _, cx| { + if let Some(gpui) = cx.try_global::() { + gpui.toggle_voice(); + } + this.voice_transcript_open = false; + cx.notify(); + })); + + let mut row = div().flex().items_center().gap_1().mr_1(); + if let Some(message) = failure { + row = row.child( + div() + .id("voice-failure-chip") + .h(px(24.)) + .px_2() + .rounded_md() + .flex() + .items_center() + .text_xs() + .text_color(cx.theme().danger) + .child("Voice stopped") + .tooltip(move |window, cx| Tooltip::new(message.clone()).build(window, cx)), + ); + } + if active { + let muted = status.muted; + let label = match status.activity { + VoiceActivity::Connecting => "Connecting…", + VoiceActivity::UserSpeaking => "Hearing you", + VoiceActivity::Speaking => "Speaking", + _ if muted => "Muted", + _ => "Listening", + }; + let label = if status.queued_notifications > 0 { + format!("{label} · {} waiting", status.queued_notifications) + } else { + label.to_string() + }; + let speaking = status.activity == VoiceActivity::Speaking; + row = row + .child( + div() + .id("voice-status-chip") + .h(px(24.)) + .px_2() + .rounded_md() + .flex() + .items_center() + .gap_1() + .cursor_pointer() + .text_xs() + .text_color(cx.theme().muted_foreground) + .hover(|s| s.bg(cx.theme().muted)) + .when(self.voice_transcript_open, |el| el.bg(cx.theme().muted)) + .child( + Icon::default() + .path(SharedString::from("icons/audio_lines.svg")) + .with_size(Size::XSmall) + .text_color(if speaking { + cx.theme().primary + } else { + cx.theme().muted_foreground + }), + ) + .child(SharedString::from(label)) + .tooltip(|window, cx| Tooltip::new("Show the transcript").build(window, cx)) + .on_click(cx.listener(|this, _, _, cx| { + this.voice_transcript_open = !this.voice_transcript_open; + cx.notify(); + })), + ) + .child( + div() + .id("voice-mute-btn") + .size(px(28.)) + .rounded_sm() + .flex() + .items_center() + .justify_center() + .cursor_pointer() + .hover(|s| s.bg(cx.theme().muted)) + .child( + Icon::default() + .path(SharedString::from(if muted { + "icons/mic_off.svg" + } else { + "icons/mic.svg" + })) + .with_size(Size::Small) + .text_color(if muted { + cx.theme().danger + } else { + cx.theme().muted_foreground + }), + ) + .tooltip(move |window, cx| { + Tooltip::new(if muted { "Unmute" } else { "Mute" }).build(window, cx) + }) + .on_click(move |_, _, cx| { + if let Some(gpui) = cx.try_global::() { + gpui.set_voice_muted(!muted); + } + }), + ); + } + Some(row.child(mic).into_any_element()) + } + + /// The transcript popover below the title bar, while open. + pub(super) fn render_voice_transcript(&self, cx: &mut Context) -> Option { + if !self.voice_transcript_open { + return None; + } + let gpui = cx.try_global::()?; + if !gpui.voice_status().activity.is_active() { + return None; + } + let transcript = gpui.voice_transcript(); + + let entries = transcript.into_iter().map(|entry| { + let (label, color) = match entry.role { + TranscriptRole::User => ("You", cx.theme().foreground), + TranscriptRole::Assistant => ("Assistant", cx.theme().primary), + TranscriptRole::Tool => ("Tool", cx.theme().muted_foreground), + TranscriptRole::Notification => ("Notification", cx.theme().muted_foreground), + }; + // A notification's first lines name the conversations; the + // instructions to the model below them are noise here. + let text = match entry.role { + TranscriptRole::Notification => entry + .text + .lines() + .skip(1) + .take_while(|line| !line.trim().is_empty()) + .collect::>() + .join("\n"), + _ => entry.text, + }; + div() + .flex() + .flex_col() + .gap(px(2.)) + .child(div().text_xs().text_color(color).child(label)) + .child( + div() + .text_sm() + .text_color(cx.theme().foreground) + .child(SharedString::from(text)), + ) + }); + + let empty = div() + .text_sm() + .text_color(cx.theme().muted_foreground) + .child("Say something — the transcript appears here."); + + Some( + div() + .absolute() + .top(px(52.)) + .right(px(16.)) + .w(px(380.)) + .max_h(px(440.)) + .bg(cx.theme().popover) + .border_1() + .border_color(cx.theme().border) + .rounded_lg() + .shadow_lg() + .child( + div() + .id("voice-transcript") + .max_h(px(440.)) + .overflow_y_scroll() + .p_3() + .flex() + .flex_col() + .gap_3() + .children(entries) + .when(gpui.voice_transcript().is_empty(), |el| el.child(empty)), + ) + .into_any_element(), + ) + } +} diff --git a/crates/ui_gpui/src/settings_screen/mod.rs b/crates/ui_gpui/src/settings_screen/mod.rs index c06130e3..0488eb64 100644 --- a/crates/ui_gpui/src/settings_screen/mod.rs +++ b/crates/ui_gpui/src/settings_screen/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod provider_forms; pub(crate) mod provider_suggestions; mod providers_section; mod skills_section; +mod voice_section; use gpui_kit::component::scroll::ScrollableElement; use gpui_kit::component::{ActiveTheme, Icon, Sizable, Size}; @@ -23,6 +24,7 @@ pub enum SettingsSection { Models, Skills, McpServers, + Voice, } /// Events emitted by the settings screen. @@ -43,6 +45,7 @@ pub struct SettingsScreen { general_section: Entity, skills_section: Entity, mcp_section: Entity, + voice_section: Entity, } impl SettingsScreen { @@ -53,6 +56,7 @@ impl SettingsScreen { let general_section = cx.new(|cx| general_section::GeneralSection::new(window, cx)); let skills_section = cx.new(|cx| skills_section::SkillsSection::new(window, cx)); let mcp_section = cx.new(|cx| mcp_section::McpSection::new(window, cx)); + let voice_section = cx.new(|cx| voice_section::VoiceSection::new(window, cx)); Self { focus_handle: cx.focus_handle(), @@ -62,6 +66,7 @@ impl SettingsScreen { general_section, skills_section, mcp_section, + voice_section, } } @@ -104,6 +109,9 @@ impl SettingsScreen { SettingsSection::McpServers => { self.mcp_section.update(cx, |s, _cx| s.reload()); } + SettingsSection::Voice => { + self.voice_section.update(cx, |s, _cx| s.reload()); + } SettingsSection::General => {} } @@ -282,6 +290,12 @@ impl Render for SettingsScreen { "MCP Servers", "icons/braces.svg", cx, + )) + .child(self.render_nav_item( + SettingsSection::Voice, + "Voice", + "icons/mic.svg", + cx, )), ), ) @@ -304,6 +318,7 @@ impl Render for SettingsScreen { SettingsSection::McpServers => { self.mcp_section.clone().into_any_element() } + SettingsSection::Voice => self.voice_section.clone().into_any_element(), }, )), ) diff --git a/crates/ui_gpui/src/settings_screen/voice_section.rs b/crates/ui_gpui/src/settings_screen/voice_section.rs new file mode 100644 index 00000000..15249d78 --- /dev/null +++ b/crates/ui_gpui/src/settings_screen/voice_section.rs @@ -0,0 +1,366 @@ +//! Voice settings: the realtime model voice mode talks to, and when +//! conversations report back. Stored in `voice.json`. + +use code_assistant_core::voice::config::supports_realtime; +use code_assistant_core::voice::{NotifyScope, VoiceConfig}; +use gpui_kit::component::input::{Input, InputEvent, InputState}; +use gpui_kit::component::switch::Switch; +use gpui_kit::component::{ActiveTheme, Sizable, Size}; +use gpui_kit::{ + App, Context, Entity, FocusHandle, Focusable, SharedString, Subscription, div, prelude::*, px, +}; +use tracing::warn; + +const EAGERNESS: &[&str] = &["auto", "low", "medium", "high"]; + +/// Which text field an input edits. +#[derive(Clone, Copy)] +enum Field { + Model, + Voice, + Transcription, + Cooling, +} + +pub struct VoiceSection { + focus_handle: FocusHandle, + config: VoiceConfig, + /// (provider id, label) of the providers with a realtime API. + providers: Vec<(String, String)>, + model_input: Entity, + voice_input: Entity, + transcription_input: Entity, + cooling_input: Entity, + _subscriptions: Vec, +} + +impl VoiceSection { + pub fn new(window: &mut gpui_kit::Window, cx: &mut Context) -> Self { + let config = VoiceConfig::load(); + let mut subscriptions = Vec::new(); + let mut input = |value: String, placeholder: &str, field: Field, cx: &mut Context| { + let placeholder = placeholder.to_string(); + let state = cx.new(|cx| { + InputState::new(window, cx) + .placeholder(placeholder) + .default_value(value) + }); + subscriptions.push(cx.subscribe_in( + &state, + window, + move |this, input, event: &InputEvent, _window, cx| { + if matches!(event, InputEvent::Change) { + let value = input.read(cx).value().trim().to_string(); + this.on_field_changed(field, value, cx); + } + }, + )); + state + }; + let model_input = input(config.model.clone(), "gpt-realtime", Field::Model, cx); + let voice_input = input(config.voice.clone(), "marin", Field::Voice, cx); + let transcription_input = input( + config.transcription_model.clone(), + "gpt-4o-mini-transcribe", + Field::Transcription, + cx, + ); + let cooling_input = input(config.cooling_ms.to_string(), "1500", Field::Cooling, cx); + Self { + focus_handle: cx.focus_handle(), + config, + providers: realtime_providers(), + model_input, + voice_input, + transcription_input, + cooling_input, + _subscriptions: subscriptions, + } + } + + /// Re-read the providers (they may have changed in another section). + pub fn reload(&mut self) { + self.providers = realtime_providers(); + self.config = VoiceConfig::load(); + } + + fn on_field_changed(&mut self, field: Field, value: String, cx: &App) { + match field { + Field::Model => self.config.model = value, + Field::Voice => self.config.voice = value, + Field::Transcription => self.config.transcription_model = value, + Field::Cooling => { + let Ok(ms) = value.parse() else { return }; + self.config.cooling_ms = ms; + } + } + self.save(cx); + } + + fn update(&mut self, change: impl FnOnce(&mut VoiceConfig), cx: &mut Context) { + change(&mut self.config); + self.save(cx); + cx.notify(); + } + + fn save(&self, cx: &App) { + if let Err(e) = self.config.save() { + warn!("Failed to save the voice settings: {e:#}"); + } + // The title bar shows voice mode only once it is configured. + if let Some(gpui) = cx.try_global::() { + gpui.refresh_voice_configured(); + } + } + + fn render_choice( + &self, + id: String, + label: String, + selected: bool, + on_click: impl Fn(&mut Self, &mut Context) + 'static, + cx: &mut Context, + ) -> impl IntoElement { + div() + .id(SharedString::from(id)) + .px_3() + .py_1() + .rounded_md() + .border_1() + .cursor_pointer() + .text_sm() + .border_color(if selected { + cx.theme().primary + } else { + cx.theme().border + }) + .text_color(if selected { + cx.theme().foreground + } else { + cx.theme().muted_foreground + }) + .when(selected, |el| el.bg(cx.theme().muted)) + .hover(|s| s.bg(cx.theme().muted)) + .child(SharedString::from(label)) + .on_click(cx.listener(move |this, _, _, cx| on_click(this, cx))) + } + + fn render_field( + &self, + label: &str, + input: &Entity, + hint: &str, + cx: &mut Context, + ) -> impl IntoElement { + div() + .flex() + .flex_col() + .gap_1() + .child(caption(label, cx)) + .child(div().w(px(260.)).child(Input::new(input))) + .child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child(SharedString::from(hint.to_string())), + ) + } +} + +fn realtime_providers() -> Vec<(String, String)> { + let mut providers: Vec<(String, String)> = + llm::provider_config::ConfigurationSystem::load_providers_config(None) + .unwrap_or_default() + .into_iter() + .filter(|(_, provider)| supports_realtime(provider)) + .map(|(id, provider)| (id, provider.label)) + .collect(); + providers.sort_by(|a, b| a.1.cmp(&b.1)); + providers +} + +fn caption(text: &str, cx: &App) -> impl IntoElement { + div() + .text_xs() + .font_weight(gpui_kit::FontWeight::MEDIUM) + .text_color(cx.theme().muted_foreground) + .child(SharedString::from(text.to_string())) +} + +impl Focusable for VoiceSection { + fn focus_handle(&self, _: &App) -> FocusHandle { + self.focus_handle.clone() + } +} + +impl Render for VoiceSection { + fn render( + &mut self, + _window: &mut gpui_kit::Window, + cx: &mut Context, + ) -> impl IntoElement { + let provider_choices: Vec<_> = self + .providers + .iter() + .map(|(id, label)| { + let selected = self.config.provider == *id; + let provider = id.clone(); + self.render_choice( + format!("voice-provider-{id}"), + label.clone(), + selected, + move |this, cx| { + let provider = provider.clone(); + this.update(|config| config.provider = provider, cx) + }, + cx, + ) + .into_any_element() + }) + .collect(); + let eagerness_choices: Vec<_> = EAGERNESS + .iter() + .map(|value| { + let selected = self.config.vad_eagerness == *value; + self.render_choice( + format!("voice-eagerness-{value}"), + value.to_string(), + selected, + move |this, cx| { + this.update(|config| config.vad_eagerness = value.to_string(), cx) + }, + cx, + ) + .into_any_element() + }) + .collect(); + let touched_only = self.config.notify == NotifyScope::Touched; + + div() + .flex() + .flex_col() + .gap_5() + .w_full() + .max_w(px(700.)) + .child( + div() + .text_xs() + .font_weight(gpui_kit::FontWeight::SEMIBOLD) + .text_color(cx.theme().muted_foreground) + .child("VOICE"), + ) + .child( + div() + .p_4() + .rounded_lg() + .border_1() + .border_color(cx.theme().border) + .bg(cx.theme().secondary) + .text_sm() + .text_color(cx.theme().foreground) + .child( + "Voice mode is a hands-free assistant for all your conversations: ask \ + what is running, start a conversation, or send one a message. Start it \ + with the microphone in the title bar. It tells you when a conversation \ + finishes, but never talks over you.", + ), + ) + .child( + div() + .flex() + .flex_col() + .gap_2() + .child(caption("Realtime provider", cx)) + .child(if provider_choices.is_empty() { + div() + .text_sm() + .text_color(cx.theme().muted_foreground) + .child( + "No provider with a realtime API yet. Add an OpenAI or AI Core \ + provider under Providers.", + ) + .into_any_element() + } else { + div() + .flex() + .flex_wrap() + .gap_2() + .children(provider_choices) + .into_any_element() + }), + ) + .child(self.render_field( + "Model", + &self.model_input, + "A speech-to-speech model of the provider. For AI Core: the name in the \ + provider's models map whose deployment to use.", + cx, + )) + .child(self.render_field( + "Voice", + &self.voice_input, + "The model's voice, e.g. marin, cedar, alloy.", + cx, + )) + .child(self.render_field( + "Transcription model", + &self.transcription_input, + "Transcribes your speech for the transcript; empty turns it off.", + cx, + )) + .child( + div() + .flex() + .flex_col() + .gap_2() + .child(caption("Turn detection", cx)) + .child(div().flex().gap_2().children(eagerness_choices)) + .child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child( + "How soon the model answers after you pause. Low waits \ + longer, high answers sooner.", + ), + ), + ) + .child(self.render_field( + "Quiet time before notifications (ms)", + &self.cooling_input, + "How long it must be silent after the model spoke before it tells you about \ + finished conversations.", + cx, + )) + .child( + div() + .flex() + .items_center() + .gap_2() + .child( + Switch::new("voice-notify-touched") + .checked(touched_only) + .with_size(Size::Small) + .on_click({ + let view = cx.entity(); + move |enabled, _window, app| { + let scope = if *enabled { + NotifyScope::Touched + } else { + NotifyScope::All + }; + view.update(app, |this, cx| { + this.update(|config| config.notify = scope, cx) + }); + } + }), + ) + .child( + div() + .text_sm() + .text_color(cx.theme().foreground) + .child("Only report conversations started or messaged by voice"), + ), + ) + } +} diff --git a/docs/voice-mode.md b/docs/voice-mode.md new file mode 100644 index 00000000..15174a5f --- /dev/null +++ b/docs/voice-mode.md @@ -0,0 +1,330 @@ +# Voice mode + +Status: implemented (2026-10-08). Code: `code_assistant_core::voice`, +`llm::realtime`, `crates/audio`, GPUI title bar and *Settings → Voice*. + +## Goal + +A global voice assistant, separate from any single session. The user +switches it on once (title bar) and then talks to it about all of their +conversations: "what is running?", "start a conversation in project X that +fixes the flaky test", "tell the refactoring session to also update the +docs", "what did the build session end with?". + +- **Global, not per session.** One voice agent per app process. It never + lives inside a session's transcript and is not tied to the session being + viewed. +- **Realtime model, configured in settings.** A speech-to-speech model + (OpenAI Realtime protocol first) chosen in a new *Voice* settings section. +- **Its own small agent loop.** It has no file, shell or browser tools. + Its only tools act on conversations, through `SessionService`. +- **Voice first.** When a background conversation finishes, the voice agent + is told, but the notification never interrupts the user or the voice + agent while either is speaking. It waits for real silence. + +Non-goals for v1: voice input into a normal session's composer (dictation), +answering permission prompts or questions by voice (see Follow-ups), the +terminal and ACP frontends. + +## Why not `agent_core::runtime` + +The existing agent loop is request/response: build the history, call +`LLMProvider::send_message`, run the tools, repeat. A realtime model works +differently. One long-lived WebSocket session keeps the conversation state +on the server, audio streams both ways all the time, the server detects +turns (VAD), and a response can be cancelled halfway through playback. So +the voice agent gets its own event loop. It reuses: + +- `SessionService` and `EventStream`, as its only link to the rest of the + app, exactly like a frontend; +- `session_query` — for reading transcripts. + +Its four tools are a small dispatch of their own (`voice::tools`), not a +`tools_core` registry: they need none of the tool context (command executor, +permissions, rendering) and are declared to the realtime session as plain +function schemas. + +## Architecture + +``` + ┌──────────── ui_gpui ─────────────┐ + │ title-bar toggle, voice indicator │ + │ Voice settings section │ + └──────▲──────────────┬────────────┘ + UiEvent::Voice* VoiceService::start/stop + │ │ +┌──────────────── code_assistant_core::voice ─────────────────┐ +│ VoiceService (start/stop, owns one VoiceAgent task) │ +│ │ +│ VoiceAgent event loop ── select! over: │ +│ • realtime server events (RealtimeConnection) │ +│ • audio events (captured, drained) │ +│ • tool completions (spawned tool futures) │ +│ • EventStream subscription → NotificationSource │ +│ • timers (cooling, playback fallback)│ +│ every input → Floor::handle(event) → Vec │ +│ │ +│ Floor (pure state machine) NotificationQueue voice tools │ +└───────▲───────────────────────────────▲──────────────────────┘ + │ │ + llm::realtime (protocol, audio (voice-processing I/O on + WebSocket transport) macOS, cpal elsewhere) +``` + +### New pieces by layer + +| Layer | Piece | Responsibility | +|---|---|---| +| 0 | `llm::realtime` | Protocol types (`ClientEvent`, `ServerEvent`), `WsConnector` over `tokio-tungstenite`, `RealtimeEndpoint::openai` (API-key auth). A connection is a pair of channels; the `RealtimeConnector` trait lets tests script the server. | +| 0 | crate `audio` | `AudioIo`: default microphone and speakers, PCM16 mono 24 kHz, a playback queue that counts played samples and reports `Drained`, `clear_playback()` for barge-in. macOS: one `VoiceProcessingIO` unit (echo cancellation, format conversion). Elsewhere: `cpal` with linear resampling and half-duplex gating. | +| 3 | `code_assistant_core::voice` | `VoiceConfig`, `VoiceService`, `VoiceAgent` loop, `Floor` state machine, `NotificationQueue`, `NotificationSource`, voice tools, instructions. Audio comes in through the `VoiceAudio` trait / `AudioFactory`. | +| 4 | `ui_gpui` | Title-bar toggle, mute button, status chip, transcript popover, *Voice* settings section. | +| 5 | `code_assistant` | Feature `voice` (default). Adapts `audio::AudioIo` to `VoiceAudio` and runs the `VoiceService` worker next to the `SessionService` worker. | + +The `audio` crate sits behind the `voice` feature, so headless builds +(`--no-default-features`) do not link CoreAudio, ALSA or WASAPI. The core +does not depend on it. + +`cargo run -p audio --example devices [-- --silence | --tone]` opens the +devices for two seconds and reports captured chunks, playback and drains. + +## Configuration + +Stored in `/voice.json` and edited in a new *Voice* settings +section. It follows the pattern of `handoff.json` and `lifecycle.json`. + +```json +{ + "provider": "openai", // provider id from providers.json + "model": "gpt-realtime", + "voice": "marin", + "url": "wss://…", // optional, overrides the derived URL + "transcription_model": "gpt-4o-mini-transcribe", // "" = no transcript + "vad_eagerness": "auto", // semantic VAD: low | medium | high | auto + "cooling_ms": 1500, + "notify": "all" // "all" | "touched" (see Notifications) +} +``` + +Credentials come from the referenced `providers.json` entry, so nothing is +duplicated. Realtime models stay out of `models.json` on purpose: they cannot +run a normal session, and adding them there would put them into every model +picker. The settings section offers the providers that can serve realtime +sessions (`llm::realtime::connector_for_provider`): + +- `openai`, `openai-responses`, `openai-responses-ws` with an `api_key`: the + URL is `wss:///…/realtime?model=…` (or `url`). +- `ai-core`: the provider's client credentials and token manager; `model` + names an entry of the provider's `models` map whose deployment serves the + session. On every connect the connector reads the deployment resource + (`…/v2/lm/deployments/{id}`), takes its `deploymentUrl`, appends + `/v1/realtime` and connects with the bearer token and + `AI-Resource-Group: default`. `session.update` then omits `model`. + +The devices are the system defaults. + +## Voice tools + +All tools are thin adapters over `SessionService` / `session_query` +(`SessionService::session_content` reads a stored session without making +it the active one; `session_activity_states` gives the loaded sessions' +states). Every tool returns at once. None waits for a conversation's agent to finish. + +| Tool | Parameters | Behaviour | +|---|---|---| +| `list_conversations` | `project?`, `include_settled?` (default false), `limit?` (default 20) | `list_sessions` + lifecycle + activity state → `[{id, title, project, branch, state, last_active}]`, where `state` is `running`, `idle`, `errored: …` or `waiting for …` (permission or answers). Sorted by recent activity. | +| `list_projects` | — | Project names, so `message_conversation` can create sessions in the right place. Can be merged into `list_conversations` later if it turns out to be noise. | +| `message_conversation` | `conversation_id?`, `project?`, `title?`, `message` | With no id, `create_session(title, project)` (the project must be known) and then send. With an id, `send_or_queue_user_message`. Returns `{conversation_id, status}` with `status` `started` or `queued`. Marks the session as *touched*. | +| `get_conversation` | `conversation_id`, `full?` (default false) | Default: `{state, last_agent_message}`, the final assistant text of the latest turn. With `full: true`: the last 10 turns, each with the user message and the turn's **last** assistant text only (no thinking, no tool calls, no tool results). Every text is truncated (about 2 000 characters each, with a total cap). Built on `session_query::get_session_content` with `UserText` + `AssistantText` plus grouping by turn. Calling it acknowledges any pending notification for that conversation. | + +`message_conversation` replaces separate *create* and *comment* tools, as +suggested: an empty `conversation_id` means create. The voice agent never +reads the visible session or switches it. A later option is a +`show_conversation` tool that asks the GPUI frontend to open a session. + +## Notifications + +`NotificationSource` subscribes to the `EventStream` and watches +`UiEvent::UpdateSessionActivityState`: + +- running → `Idle`: *finished* +- running → `Errored`: *failed* +- `RequestToolPermission` / `RequestUserQuestions`: *waiting for you* + (v1 only reports it; it cannot answer) + +With `notify: "all"` (the default), every session in this process is +reported. `"touched"` limits it to sessions the voice agent created or +messaged during this voice session. Sessions running in another process +(`RunningExternally`) send no events and are not covered. + +On `StreamError::Lagged` the source gets back in sync by diffing +`list_sessions` activity against its last known states. + +Each notification enters the `NotificationQueue`, keyed by session id. A +newer event for the same session replaces the older one, so a session that +finishes twice before it is flushed is reported once. A `get_conversation` +call for that session drops its entry: the model already knows. + +When flushed, the queued entries are put into **one** text item. One +`response.create` follows: + +> [background notification] +> - Conversation "Fix flaky test" in project proj-x (id …) finished its turn. +> - Conversation "Refactor parser" (id …) is waiting for permission to run +> `execute_command`. +> +> The user is not speaking right now. If it is useful to them, mention this +> briefly. Call get_conversation before you summarise a result. Otherwise +> say nothing. + +A "waiting for you" entry is withdrawn when the request is answered in the +UI before the floor frees up. + +## Two-level priority: the `Floor` state machine + +This is the core of the feature and the part to get right first. It is a +**pure** state machine. Inputs are events, outputs are commands; it has no +I/O and no clock (timers are commands the loop runs and reports back as +events). That makes it fully unit-testable. + +Priority levels: + +1. **Voice**: user speech, the model's answer to the user, and tool calls + and results that belong to that answer. +2. **Background**: conversation notifications. They are only ever delivered + when the floor is free, and never cancel or delay level 1. + +The model generates audio much faster than it plays, so `response.done` +does **not** mean "stopped talking". Because we own the speakers natively, +the playback queue reports `Drained` exactly, with no browser round trip. +A fallback timer of `audio duration + 5 s` still guards against a stuck +sink. + +### States + +| State | Meaning | +|---|---| +| `Speaking` | a response is being generated **or** its audio is still playing | +| `UserTurn` | the user is speaking, or has spoken and is owed an answer | +| `Cooling` | playback drained; silence timer (`cooling_ms`) running | +| `Idle` | silent, nothing pending | + +### Transitions + +| Input | Effect | +|---|---| +| server `response.created` | cancel timers → `Speaking` | +| server `response.output_audio.delta` | (agent) queue for the speakers; remember the item and the sample offset its audio starts at | +| server `response.output_item.done` (function call) | (agent) spawn the tool → `ToolStarted` | +| server `response.done` / cancelled | attach held tool outputs; in `Speaking`, wait for the drain (playback timer: pending audio + 5 s), or act as drained if nothing is pending | +| server `input_audio_buffer.speech_started` (barge-in) | cancel timers; if `Speaking`: `response.cancel` (deferred until `response.created` when our create is still in flight) and `StopPlayback` (agent: clear the queue, `conversation.item.truncate` to the samples played). Audio deltas play only in `Speaking`, so late audio of the cancelled response is dropped → `UserTurn` | +| server `input_audio_buffer.speech_stopped` | stays `UserTurn`; the server's VAD creates the response. If none follows within 8 s, the model answers unanswered tool outputs, or else the floor is free again | +| server `error` | when our `response.create` was in flight: give the floor back (no wedge) | +| sink `Drained` (or fallback timer) | if tool outputs are unanswered and no tool runs any more → `response.create`; otherwise → `Cooling` + start the cooling timer | +| cooling timer fires | `Cooling` → `Idle`; flush notifications if any are queued | +| tool completed | response still open → hold until `response.done`; otherwise attach, and the output counts as unanswered until the next response starts. `Speaking` → answered after `Drained`; `UserTurn` → **no** `response.create` (the model uses it in its next answer, or answers when the user's turn ends without one); `Cooling`/`Idle` → `response.create` once the last running tool is done, so several calls of one response get one answer | +| notification arrives | always queue; `Speaking`/`UserTurn` → nothing more; `Cooling` → restart the cooling timer (groups finishes that land close together); `Idle` → flush now | +| `get_conversation` read | drop that conversation's queued notification | + +While a tool call runs, notifications are not flushed: the tool's result +makes the model speak anyway, and the notification follows after that +answer. + +Invariants, each with its own test: + +1. No `response.create` while a response is open. +2. No `response.create` in `UserTurn` caused by a tool result or a + notification. +3. A notification is only flushed after `Drained` **and** `cooling_ms` of + silence, or straight from `Idle`. +4. A notification never causes `response.cancel` or `flush()`. +5. Barge-in always truncates the server-side item to what was actually + heard, so the model does not believe it said things the user never heard. + +### Function calls inside a response + +Voice tools run concurrently as spawned futures. Their outputs go through +the floor as *tool completed* events, never straight to the socket. So the +rules above apply equally to slow calls (`get_conversation` on a large +session) and to fast ones. + +## Session lifetime and reconnect + +- Start: open the realtime session → `session.update` (instructions, tools, + voice, turn detection, input transcription on) → start capture. +- Realtime sessions have a maximum duration and can drop. The agent keeps a + local text transcript (user transcripts + assistant transcripts + tool + calls). On a closed connection it reconnects (three attempts, backing + off), re-sends `session.update`, seeds the last 30 transcript lines as one + system message, and keeps the notification queue (flushed after a cooling + period). Tool results of the old connection are dropped. +- Stop: the toggle, or an error that cannot be retried. Capture and + playback stop, the socket closes, and the queue is dropped. +- Mute: captured audio is dropped (the device stays open). A user turn in + progress ends as if the user had stopped speaking, since the server never + hears the end of it. Notifications are still delivered: they speak from + `Idle`, which a muted user is in. +- Voice runs keep no transcript on disk in v1. The live transcript is kept + in memory for the UI popover and for reconnect. + +## UI (GPUI) + +- Title bar: a microphone toggle (start/stop), and while active a status + chip (Connecting… / Listening / Hearing you / Speaking / Muted, plus + "· N waiting" for queued notifications) and a mute button. Events arrive + as app-scoped `UiEvent::VoiceStatusChanged` and `UiEvent::VoiceTranscript` + (`session_id: None` on the `EventStream`); `Gpui` mirrors them + (`app/voice.rs`). A failure shows as a "Voice stopped" chip next to the + microphone, with the reason in its tooltip, until voice mode starts again. +- The status chip opens the transcript popover. +- Settings → *Voice*: provider, model, voice, transcription model, VAD + eagerness, cooling time, notify scope. +- macOS bundle: `NSMicrophoneUsageDescription` in `Info.plist` and the + `com.apple.security.device.audio-input` entitlement for the hardened + runtime. +- A microphone the user denied records silence instead of failing, so on + macOS the `audio` crate checks the permission first: voice mode does not + start and the failure names the setting. An undecided permission lets the + system ask on the first capture. +- The voice agent posts into sessions like a user would, so those sessions + show the message as a normal user message. Later, a small "via voice" + marker on the message could be added. + +## Decisions + +1. **Echo cancellation: native.** On macOS the `audio` crate drives a + `VoiceProcessingIO` audio unit (the system's echo canceller, the one + FaceTime uses) for capture and playback, so the user can interrupt the + model while it speaks. Other platforms fall back to `cpal` with + half-duplex gating: the microphone is dropped while audio plays, so there + is no barge-in there. +2. **Provider: OpenAI Realtime protocol**, directly (`openai*` provider + types) or through AI Core (`ai-core`, deployment from the provider's + `models` map). `voice.json` can override the OpenAI URL for compatible + gateways. +3. **Notify scope: all conversations** by default. + +## Tests + +- `voice::floor` — every transition and invariant, table-style. +- `voice::notifications`, `voice::source`, `voice::tools` (turn grouping + and limits), `voice::config` (endpoint derivation). +- `voice::tests` — end to end: scripted realtime server, fake audio devices + and a real `SessionService` with a mock LLM. A conversation finishing while + the model speaks is announced only after drain plus cooling; a + `message_conversation` call starts a session whose end is announced; + `get_conversation` returns the turns and suppresses the notification; + barge-in cancels and truncates to what was heard. +- `llm::realtime` — event parsing and serialization, PCM16 encoding. +- `audio` — playback queue accounting and drain reports, resampler. + +## Follow-ups + +- Answer permission prompts and questions by voice + (`respond_permission`, `answer_questions` as voice tools, gated by an + explicit confirmation step). +- `stop_conversation` (`request_stop`) and `show_conversation` (open it in + the UI). +- Dictation into the normal composer, reusing the `audio` crate. +- Gemini Live transport. +- Choosing input and output devices.