From 20e38894f3a588752162f6f84066bccfce53691f Mon Sep 17 00:00:00 2001 From: Kevin Boos Date: Wed, 9 Sep 2026 16:59:22 -0700 Subject: [PATCH 1/4] Add `robius-speech`, a new crate for platform-native speech-to-text input `robius-speech` streams speech-to-text from the OS's own recognizer, with no model downloads and no third-party services. A recognition "session" returns multiple partial transcripts as they stream in and a single final transcript from a user's "utterances" (chunks of spoken words). It also provides microphone levels for a sound level meter or waveform animation. Backends: * macOS/iOS: SFSpeechRecognizer + AVAudioEngine, via a Swift bridge * Android: SpeechRecognizer, via Java compiled to an embedded DEX * Windows: SAPI dictation, chosen over WinRT because it works in a plain unpackaged desktop app * Linux, etc: unavailable, just a stub whose `is_supported()` returns false Permissions are automatically requested by the crate on every platform, just like all other robius crates, so your app doesn't need any platform-specific code. --- .github/workflows/speech.yml | 51 ++ Cargo.lock | 140 +++- README.md | 4 + crates/speech/.gitignore | 2 + crates/speech/Cargo.toml | 35 + crates/speech/LICENSE | 21 + crates/speech/README.md | 119 +++ crates/speech/android_build.rs | 67 ++ crates/speech/build.rs | 104 +++ .../java/dev/robius/speech/NativeSpeech.java | 380 +++++++++ .../speech/SpeechPermissionFragment.java | 168 ++++ crates/speech/src/android.rs | 120 +++ crates/speech/src/apple.rs | 43 + crates/speech/src/lib.rs | 316 ++++++++ crates/speech/src/windows.rs | 440 +++++++++++ crates/speech/swift/NativeSpeech.swift | 568 ++++++++++++++ crates/speech/swift/tests/SpeechTests.swift | 269 +++++++ crates/speech/swift/tests/run.sh | 31 + .../android_retry_test.cpython-313.pyc | Bin 0 -> 20480 bytes crates/speech/tests/android_retry_test.py | 734 ++++++++++++++++++ 20 files changed, 3602 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/speech.yml create mode 100644 crates/speech/.gitignore create mode 100644 crates/speech/Cargo.toml create mode 100644 crates/speech/LICENSE create mode 100644 crates/speech/README.md create mode 100644 crates/speech/android_build.rs create mode 100644 crates/speech/build.rs create mode 100644 crates/speech/java/dev/robius/speech/NativeSpeech.java create mode 100644 crates/speech/java/dev/robius/speech/SpeechPermissionFragment.java create mode 100644 crates/speech/src/android.rs create mode 100644 crates/speech/src/apple.rs create mode 100644 crates/speech/src/lib.rs create mode 100644 crates/speech/src/windows.rs create mode 100644 crates/speech/swift/NativeSpeech.swift create mode 100644 crates/speech/swift/tests/SpeechTests.swift create mode 100755 crates/speech/swift/tests/run.sh create mode 100644 crates/speech/tests/__pycache__/android_retry_test.cpython-313.pyc create mode 100644 crates/speech/tests/android_retry_test.py diff --git a/.github/workflows/speech.yml b/.github/workflows/speech.yml new file mode 100644 index 0000000..fed4311 --- /dev/null +++ b/.github/workflows/speech.yml @@ -0,0 +1,51 @@ +name: Native speech + +on: + push: + paths: + - crates/speech/** + - Cargo.toml + - Cargo.lock + - .github/workflows/speech.yml + pull_request: + paths: + - crates/speech/** + - Cargo.toml + - Cargo.lock + - .github/workflows/speech.yml + +jobs: + native: + name: Speech (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-14, macos-15-intel, windows-2022, ubuntu-latest] + env: + MACOSX_DEPLOYMENT_TARGET: '11.0' + steps: + - uses: actions/checkout@v5 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + - run: cargo clippy -p robius-speech --all-targets -- -D warnings + - run: cargo test -p robius-speech + - name: Apple native lifecycle regressions + if: runner.os == 'macOS' + run: bash crates/speech/swift/tests/run.sh + - name: Compile iOS backends + if: matrix.os == 'macos-14' + run: | + rustup target add aarch64-apple-ios aarch64-apple-ios-sim + cargo check -p robius-speech --target aarch64-apple-ios + cargo check -p robius-speech --target aarch64-apple-ios-sim + - uses: actions/setup-java@v4 + if: matrix.os == 'macos-14' + with: + distribution: temurin + java-version: '17' + - name: Android speech lifecycle regressions + if: matrix.os == 'macos-14' + run: python3 crates/speech/tests/android_retry_test.py diff --git a/Cargo.lock b/Cargo.lock index 44f682f..da4d5e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -992,8 +992,8 @@ dependencies = [ "objc2-local-authentication", "retry", "robius-android-env", - "windows", - "windows-core", + "windows 0.56.0", + "windows-core 0.56.0", "zbus", "zbus_polkit", ] @@ -1045,7 +1045,7 @@ dependencies = [ "objc2-foundation", "robius-android-env", "tokio", - "windows", + "windows 0.56.0", "zbus", ] @@ -1063,7 +1063,7 @@ dependencies = [ "objc2-foundation", "objc2-ui-kit", "robius-android-env", - "windows", + "windows 0.56.0", ] [[package]] @@ -1081,7 +1081,16 @@ dependencies = [ "objc2-ui-kit", "robius-android-env", "robius-common", - "windows", + "windows 0.56.0", +] + +[[package]] +name = "robius-speech" +version = "0.3.1" +dependencies = [ + "jni", + "robius-android-env", + "windows 0.61.3", ] [[package]] @@ -1561,22 +1570,68 @@ version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1de69df01bdf1ead2f4ac895dc77c9351aefff65b2f3db429a343f9cbf05e132" dependencies = [ - "windows-core", + "windows-core 0.56.0", "windows-targets 0.52.5", ] +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core 0.61.2", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.2", +] + [[package]] name = "windows-core" version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4698e52ed2d08f8658ab0c39512a7c00ee5fe2688c65f8c0a4f06750d729f2a6" dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", + "windows-implement 0.56.0", + "windows-interface 0.56.0", + "windows-result 0.1.1", "windows-targets 0.52.5", ] +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.56.0" @@ -1588,6 +1643,17 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-interface" version = "0.56.0" @@ -1599,12 +1665,39 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.1.1" @@ -1614,6 +1707,24 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -1638,7 +1749,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -1672,6 +1783,15 @@ dependencies = [ "windows_x86_64_msvc 0.52.5", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" diff --git a/README.md b/README.md index 4a4a18a..d7f61db 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,9 @@ There's also a [status table](#crate--platform-status-table) that shows what eac * [`robius-share`](crates/share/): opens the native system share sheet so you can share a file or other content with a different app on your system. * This is implemented using `Intent.createChooser` on Android, `UIActivityViewController` on iOS, `NSSharingServicePicker` on macOS, and the WinRT Share UI on Windows. * Linux doesn't really have a system share sheet, so we wrote a custom XDG portal connection that still supports every possible type of share payload. +* [`robius-speech`](crates/speech/): streams native speech-to-text results and microphone levels on macOS, iOS, Android, and Windows. + * Provides recording sessions with partial/final transcripts, graceful stop, and cancellation, independently of the UI toolkit. + * Linux doesn't offer any OS-provided speech-to-text functionality, so this crate can't do anything on Linux. * [`robius-web-auth-session`](crates/web_auth_session/): runs an OAuth/SSO login process in the OS's own in-app browser session, and then sends the result back to your app. * Currently this is for iOS only, based on `ASWebAuthenticationSession`. There's no other safe/supported way to do web login on iOS, because otherwise iOS will suspend your app while showing the browser. @@ -49,6 +52,7 @@ Symbol legend: ✅ fully supported · ⚠️ partial, or has issues · 🚧 unde | [`robius-location`](crates/location/) | ✅ `CLLocationManager` (CoreLocation) | ✅ `CLLocationManager` (CoreLocation) | ✅ `LocationManager` | ✅ `Geolocator` (`Windows.Devices.Geolocation`, WinRT) | ✅ XDG Location portal, with a `GeoClue` fallback | | [`robius-open`](crates/open/) | ✅ `NSWorkspace.openURL` | ✅ `UIApplication.openURL` | ✅ `Intent` (`ACTION_VIEW`) | ✅ `Launcher.LaunchUriAsync` (WinRT) | ✅ `xdg-open` | | [`robius-share`](crates/share/) | ✅ `NSSharingServicePicker` | ✅ `UIActivityViewController` | ✅ `ACTION_SEND` / `ACTION_SEND_MULTIPLE` via `Intent.createChooser` | ✅ WinRT Share UI (`DataTransferManager`) | ✅ XDG portal "Open With" chooser (`OpenURI` / `OpenFile`), or its `SaveFiles` dialog for multi-item payloads; `xdg-open` fallback | +| [`robius-speech`](crates/speech/) | ⚠️ `SFSpeechRecognizer` + `AVAudioEngine` | ⚠️ `SFSpeechRecognizer` + `AVAudioEngine` | ⚠️ `SpeechRecognizer` | ⚠️ SAPI dictation | ❌ not supported | | [`robius-web-auth-session`](crates/web_auth_session/) | ❌ not supported | ✅ `ASWebAuthenticationSession` | 🚧 planned (custom chrome tabs) | ❌ not supported | ❌ not supported | diff --git a/crates/speech/.gitignore b/crates/speech/.gitignore new file mode 100644 index 0000000..4fffb2f --- /dev/null +++ b/crates/speech/.gitignore @@ -0,0 +1,2 @@ +/target +/Cargo.lock diff --git a/crates/speech/Cargo.toml b/crates/speech/Cargo.toml new file mode 100644 index 0000000..177ab25 --- /dev/null +++ b/crates/speech/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "robius-speech" +version.workspace = true +edition.workspace = true +authors.workspace = true +description = "Native streaming speech-to-text input and microphone capture for Rust apps" +documentation = "https://docs.rs/robius-speech" +homepage.workspace = true +keywords = ["robius", "speech", "dictation", "microphone", "STT"] +categories.workspace = true +license.workspace = true +repository.workspace = true +readme = "README.md" + +[lints.rust] +keyword_idents_2024 = "forbid" +non_ascii_idents = "forbid" +non_local_definitions = "forbid" +unsafe_op_in_unsafe_fn = "forbid" +unnameable_types = "warn" +unused_import_braces = "warn" + +[lints.clippy] +collapsible_if = "allow" +collapsible_else_if = "allow" +uninlined_format_args = "allow" + +[dependencies] + +[target.'cfg(target_os = "windows")'.dependencies] +windows = { version = "0.61.3", default-features = false, features = ["Win32_Media_Speech", "Win32_System_Com", "Win32_Globalization"] } + +[target.'cfg(target_os = "android")'.dependencies] +jni.workspace = true +robius-android-env.workspace = true diff --git a/crates/speech/LICENSE b/crates/speech/LICENSE new file mode 100644 index 0000000..ea5839f --- /dev/null +++ b/crates/speech/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Project Robius Developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/speech/README.md b/crates/speech/README.md new file mode 100644 index 0000000..2b72652 --- /dev/null +++ b/crates/speech/README.md @@ -0,0 +1,119 @@ +# Robius speech + +`robius-speech` provides native streaming speech-to-text input and microphone capture for Rust apps. + +* This crate currently only allows access to speech recognition (speech-to-text) services, not text-to-speech. +* This crate doesn't offer integration with third-party transcription services or model downloads. + +| Platform | Native service | Requirements | +| --- | --- | --- | +| macOS | ✅ SFSpeechRecognizer and AVAudioEngine | macOS 11+, microphone and speech permission | +| iOS | ✅ SFSpeechRecognizer and AVAudioEngine | iOS 13+, microphone and speech permission | +| Android | ✅ SpeechRecognizer | API 26+, installed recognition service, microphone permission | +| Windows | ✅ SAPI dictation and native audio input | Installed system speech language and microphone access | +| Linux, Web | ❌ Unavailable | `is_supported()` returns false | + +* Apple and Android prefer on-device recognition when supported, but each platform may use +the system provider's network service when a local recognizer is unavailable. +* Windows SAPI uses entirely on-device recognizers that are already installed. +* Linux simply doesn't offer a built-in platform-native speech to text service, so there's nothing we can do. + + +## Usage + +```rust,no_run +use robius_speech::{NativeSpeechEvent, NativeSpeechOptions, NativeSpeechSession, SpeechError}; + +fn start_dictation() -> Result { + NativeSpeechSession::start(NativeSpeechOptions::default(), |event| { + match event { + NativeSpeechEvent::Transcript { text, is_final } => { + // Forward to your UI thread: replace the current utterance on + // partial results and commit it when is_final is true. + println!("{text} (final: {is_final})"); + } + NativeSpeechEvent::Error(error) => eprintln!("{} ({:?})", error, error.kind()), + _ => {} + } + }) +} +``` + +The session (`NativeSpeechSession` object) must stay alive for as long as you are dictating; +dropping it releases the microphone and ends recognition. +The `start()` function returns immediately rather than blocking on the permission prompt, +and the `Started` event tells you when permission has been granted and the microphone is actually recording. + +Each `AudioLevel` event includes an amplitude level that's normalized between 0 and 1, +which can be used to display a level meter or similar sound wave animation. +Callbacks arrive on platform-dependent native OS threads, so you'll want to forward them +to your main UI thread or event loop. + +Within a session, partial transcripts replace the current "utterance", +but final transcripts commit the full utterance, with recognition continuing across +as many utterances as the user speaks. An "utterance" is just a chunk of words spoken by the user. + +Calling `stop()` closes the microphone but still allows the final result to arrive, +whereas `cancel()` (or just dropping the session) discards anything that's still pending. +Either way, the session ends with exactly one terminal event: `Stopped` or `Error`. +Note that it's possible for an in-progress callback to continue executing after you cancel it. +This crate also provides `cancel_all()` for things like suspending/quitting the app. + +Importantly, only one session can be active at a time. + +Errors have a `SpeechErrorKind` so that you can handle the exact cause. +For ex, `PermissionDenied` usually means that you should inform the user that they need +to enable audio and/or speech permissions in system settings. +Similarly, `Unavailable` means that the audio input or speech recognition service +just doesn't exist and retrying it will never work. + + +## Platform integration + +**Apple.** Your application must declare `NSMicrophoneUsageDescription` and +`NSSpeechRecognitionUsageDescription` in its Info.plist. +This crate will request both permissions by itself, if needed. +Sandboxed or hardened macOS applications (which is typical for any distributed app bundle) +will also need the `com.apple.security.device.audio-input` entitlement. + +Note that running a raw executable (e.g., `cargo run`) won't work with audio/speech, +you need an app bundle that has an embedded plist. +The apple backend in this crate will detect that case and report an error. + +**Android.** Declare `android.permission.RECORD_AUDIO` and a `queries` intent for +`android.speech.RecognitionService` in your manifest. The runtime permission +request is handled for you, through a headless fragment whose result routes back +to the crate rather than to your activity's `onRequestPermissionsResult`, so no +permission plumbing of your own is needed. This works just like other robius crates. + +## Building and validation + +Apple builds always compile the Swift bridge, and a missing toolchain will fail the build. +You need to install the Xcode command line tools (`xcode-select --install`), which are +already required by Rust itself... so you probably have that already taken care of. +On iOS, you need a full Xcode installation just like every other crate/app. + +For macOS builds, set the `MACOSX_DEPLOYMENT_TARGET` env var to 11.0 or newer. +This mostly just matters on Intel x86 macs, not Apple silicon (ARM aarch64), +but it doesn't hurt to always set it. + +Android builds need an SDK platform jar, `d8`, and a JDK, which the build script +locates through `ANDROID_HOME` (or `ANDROID_SDK_ROOT`), `ANDROID_PLATFORM`, +`ANDROID_BUILD_TOOLS_VERSION`, and `JAVA_HOME`. When those tools are absent the +build still succeeds, emitting a warning and compiling a backend that reports +itself as unavailable, which keeps plain `cargo check` and rust-analyzer working. +If you install them afterwards, run `cargo clean -p robius-speech` so that the +build script looks for them again. + +For this we recommend using the [`android-build`](https://crates.io/crates/android-build) crate +like all Makepad + Robius apps do. + +```sh +MACOSX_DEPLOYMENT_TARGET=11.0 cargo test -p robius-speech +MACOSX_DEPLOYMENT_TARGET=11.0 cargo clippy -p robius-speech --all-targets -- -D warnings +bash crates/speech/swift/tests/run.sh +python3 crates/speech/tests/android_retry_test.py +``` + +The Swift test harness requires macOS, and the Android harness requires a JDK and a C compiler +on either macOS or Linux. diff --git a/crates/speech/android_build.rs b/crates/speech/android_build.rs new file mode 100644 index 0000000..a6ab76c --- /dev/null +++ b/crates/speech/android_build.rs @@ -0,0 +1,67 @@ +use std::{env, fs, path::{Path, PathBuf}, process::Command}; + +fn run(command: &mut Command) { + let output = command.output().expect("failed to launch Android speech bridge compiler"); + assert!(output.status.success(), "Android speech bridge compilation failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); +} + +fn newest_child(root: &Path, required: &str) -> Option { + let mut paths: Vec<_> = fs::read_dir(root).ok()? + .filter_map(Result::ok).map(|entry| entry.path()) + .filter(|path| path.join(required).is_file()).collect(); + paths.sort_by_key(|path| { + path.file_name().unwrap().to_string_lossy().split(|c: char| !c.is_ascii_digit()) + .filter_map(|part| part.parse::().ok()).collect::>() + }); + paths.pop() +} + +pub fn build() { + println!("cargo:rerun-if-changed=java"); + for name in ["ANDROID_HOME", "ANDROID_SDK_ROOT", "ANDROID_PLATFORM", "ANDROID_BUILD_TOOLS_VERSION", "JAVA_HOME", "PATH"] { + println!("cargo:rerun-if-env-changed={name}"); + } + let unavailable = || println!("cargo:warning=Android native dictation disabled: an Android SDK, platform jar, d8, and JDK are required. Configure ANDROID_HOME and JAVA_HOME to enable it."); + let Some(sdk) = env::var_os("ANDROID_HOME").or_else(|| env::var_os("ANDROID_SDK_ROOT")).map(PathBuf::from) else { + unavailable(); + return; + }; + let platform = env::var_os("ANDROID_PLATFORM").map(|name| sdk.join("platforms").join(name)) + .or_else(|| newest_child(&sdk.join("platforms"), "android.jar")); + let build_tools = env::var_os("ANDROID_BUILD_TOOLS_VERSION").map(|name| sdk.join("build-tools").join(name)) + .or_else(|| newest_child(&sdk.join("build-tools"), "lib/d8.jar")); + let java_bin = |name: &str| { + let name = if cfg!(windows) { format!("{name}.exe") } else { name.to_owned() }; + env::var_os("JAVA_HOME").map(|root| PathBuf::from(root).join("bin").join(&name)) + .unwrap_or_else(|| PathBuf::from(name)) + }; + let (Some(platform), Some(build_tools)) = (platform, build_tools) else { + unavailable(); + return; + }; + if !platform.join("android.jar").is_file() || !build_tools.join("lib/d8.jar").is_file() + || ["javac", "java"].iter().any(|name| { + !Command::new(java_bin(name)).arg("-version").output().is_ok_and(|output| output.status.success()) + }) + { + unavailable(); + return; + } + let output = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + let classes = output.join("speech-java"); + if classes.exists() { fs::remove_dir_all(&classes).unwrap(); } + fs::create_dir_all(&classes).unwrap(); + run(Command::new(java_bin("javac")).args(["-source", "8", "-target", "8", "-classpath"]) + .arg(platform.join("android.jar")).arg("-d").arg(&classes) + .arg("java/dev/robius/speech/NativeSpeech.java") + .arg("java/dev/robius/speech/SpeechPermissionFragment.java")); + let mut class_files: Vec<_> = fs::read_dir(classes.join("dev/robius/speech")).unwrap() + .filter_map(Result::ok).map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "class")).collect(); + class_files.sort(); + run(Command::new(java_bin("java")).arg("-cp").arg(build_tools.join("lib/d8.jar")) + .args(["com.android.tools.r8.D8", "--min-api", "26", "--lib"]) + .arg(platform.join("android.jar")).arg("--output").arg(&output).args(class_files)); + println!("cargo:rustc-cfg=native_speech_android"); +} diff --git a/crates/speech/build.rs b/crates/speech/build.rs new file mode 100644 index 0000000..d68a894 --- /dev/null +++ b/crates/speech/build.rs @@ -0,0 +1,104 @@ +use std::{env, path::PathBuf, process::Command}; + +mod android_build; + +fn main() { + println!("cargo:rustc-check-cfg=cfg(native_speech_android)"); + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); + if target_os == "android" { + android_build::build(); + return; + } + if target_os != "macos" && target_os != "ios" { return; } + apple_build(&target_os); +} + +/// Compiles and links the Swift bridge. Dictation ships in every Apple build, so +/// a missing Swift toolchain fails the build rather than quietly dropping the +/// feature on you. +fn apple_build(target_os: &str) { + // The iOS SDKs only come with full Xcode, whereas macOS needs nothing more + // than the command line tools. Suggest whichever one fits the target. + let install_hint = if target_os == "macos" { + "Install the Xcode command line tools with `xcode-select --install`." + } else { + "The iOS SDK ships only with full Xcode. Install it, then select it with \ + `sudo xcode-select -s /Applications/Xcode.app`." + }; + // The toolchain is missing or misconfigured, so installing it will fix things. + let missing_toolchain = |reason: &str| -> ! { + panic!("robius-speech could not build its Swift bridge: {reason}.\n\ + Native dictation is part of every Apple build. {install_hint}\n\ + `xcode-select -p` prints which developer directory is currently selected."); + }; + // The toolchain ran fine and rejected our own source, so installing + // something won't help. Don't send anyone off to the App Store for this. + let bridge_failed = |reason: &str| -> ! { + panic!("robius-speech could not build its Swift bridge: {reason}"); + }; + + println!("cargo:rerun-if-changed=swift/NativeSpeech.swift"); + for key in ["MACOSX_DEPLOYMENT_TARGET", "IPHONEOS_DEPLOYMENT_TARGET", "IPHONESIMULATOR_DEPLOYMENT_TARGET", "DEVELOPER_DIR", "SDKROOT"] { + println!("cargo:rerun-if-env-changed={key}"); + } + let out = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + let arch = match env::var("CARGO_CFG_TARGET_ARCH").unwrap().as_str() { + "aarch64" => "arm64", + "x86_64" => "x86_64", + other => bridge_failed(&format!("unsupported Apple architecture {other}")), + }; + let simulator = target_os == "ios" && (env::var("CARGO_CFG_TARGET_ABI").as_deref() == Ok("sim") || arch == "x86_64"); + let (sdk, deployment_key, default_deployment, platform) = match (target_os, simulator) { + ("macos", _) => ("macosx", "MACOSX_DEPLOYMENT_TARGET", "11.0", "macosx"), + (_, true) => ("iphonesimulator", "IPHONESIMULATOR_DEPLOYMENT_TARGET", "13.0", "ios"), + _ => ("iphoneos", "IPHONEOS_DEPLOYMENT_TARGET", "13.0", "ios"), + }; + let configured_deployment = env::var(deployment_key).ok(); + if target_os == "macos" && arch == "x86_64" && configured_deployment.is_none() { + panic!("robius-speech requires macOS 11.0 or newer. Set MACOSX_DEPLOYMENT_TARGET=11.0 (or newer) for the entire Cargo invocation, including the final executable. Rust's default Intel deployment target is too old for this Swift bridge."); + } + let deployment = configured_deployment.unwrap_or_else(|| default_deployment.into()); + if target_os == "macos" { + let major = deployment.split('.').next().and_then(|part| part.parse::().ok()).unwrap_or(0); + assert!(major >= 11, "robius-speech requires MACOSX_DEPLOYMENT_TARGET=11.0 or newer for the entire Cargo invocation"); + } + let target = format!("{arch}-apple-{platform}{deployment}{}", if simulator { "-simulator" } else { "" }); + let Ok(sdk_result) = Command::new("xcrun").args(["--sdk", sdk, "--show-sdk-path"]).output() else { + missing_toolchain("xcrun is not installed"); + }; + if !sdk_result.status.success() { + missing_toolchain(&format!("cannot locate the {sdk} SDK: {}", String::from_utf8_lossy(&sdk_result.stderr).trim())); + } + let sdk_path = String::from_utf8(sdk_result.stdout).unwrap(); + let sdk_path = sdk_path.trim(); + let library = out.join("librobius_speech.a"); + let result = Command::new("xcrun").args([ + "--sdk", sdk, "swiftc", "-swift-version", "5", "-O", "-emit-library", "-static", "-parse-as-library", + "-module-name", "RobiusSpeech", "-target", &target, "-sdk", sdk_path, + "-module-cache-path", + ]).arg(out.join("swift-module-cache")).arg("swift/NativeSpeech.swift").arg("-o").arg(&library).output(); + let Ok(result) = result else { missing_toolchain("the Swift compiler is not installed") }; + if !result.status.success() { + bridge_failed(&format!("the Swift bridge did not compile:\n{}", String::from_utf8_lossy(&result.stderr).trim())); + } + println!("cargo:rustc-link-search=native={}", out.display()); + println!("cargo:rustc-link-search=native={sdk_path}/usr/lib/swift"); + println!("cargo:rustc-link-search=native=/usr/lib/swift"); + println!("cargo:rustc-link-lib=static=robius_speech"); + for framework in ["Speech", "AVFoundation", "Foundation"] { + println!("cargo:rustc-link-lib=framework={framework}"); + } + // Swift's compiler supplies compatibility libraries from its toolchain. + let Ok(info) = Command::new("xcrun").args(["--sdk", sdk, "swiftc", "-target", &target, "-sdk", sdk_path, "-print-target-info"]).output() else { + missing_toolchain("the Swift compiler is not installed"); + }; + if !info.status.success() { + missing_toolchain("cannot query the Swift runtime search paths"); + } + for line in String::from_utf8_lossy(&info.stdout).lines() { + let path = line.trim().trim_end_matches(',').trim_matches('"'); + if path.starts_with('/') && path.contains("/lib/swift") { + println!("cargo:rustc-link-search=native={path}"); + } + } +} diff --git a/crates/speech/java/dev/robius/speech/NativeSpeech.java b/crates/speech/java/dev/robius/speech/NativeSpeech.java new file mode 100644 index 0000000..906fd35 --- /dev/null +++ b/crates/speech/java/dev/robius/speech/NativeSpeech.java @@ -0,0 +1,380 @@ +package dev.robius.speech; + +import android.Manifest; +import android.app.Activity; +import android.app.Application; +import android.content.Context; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.os.Build; +import android.os.Bundle; +import android.os.Handler; +import android.os.Looper; +import android.speech.RecognitionListener; +import android.speech.RecognizerIntent; +import android.speech.SpeechRecognizer; +import java.util.ArrayList; +import java.util.HashMap; + +/** + * One dictation session, driving the system SpeechRecognizer. + * + * This is loaded from the crate's embedded DEX, so an app doesn't need a custom + * Activity or any Java of its own. Everything here runs on the main looper. + */ +public final class NativeSpeech implements RecognitionListener, Application.ActivityLifecycleCallbacks { + private static final Handler MAIN = new Handler(Looper.getMainLooper()); + private static final HashMap SESSIONS = new HashMap<>(); + private static native void event(long id, int kind, String text, float level); + // Error kinds, matching `codes` in lib.rs. + private static final int ERR_OTHER = 5; + private static final int ERR_PERMISSION = 6; + private static final int ERR_UNAVAILABLE = 7; + private static final int ERR_LANGUAGE = 8; + private static final int ERR_AUDIO = 9; + + private final Activity activity; + private final long id; + private final String locale; + private final boolean preferOnDevice; + private SpeechRecognizer recognizer; + private SpeechPermissionFragment permissionRequest; + private boolean listening = true; + private boolean awaitingResult; + private boolean started; + private boolean disposed; + private boolean triedSystemFallback; + private boolean askedForPermission; + private int recognizerGeneration; + private int transientFailures; + private static final int MAX_TRANSIENT_RETRIES = 3; + private String pending = ""; + private final Runnable restart = () -> listen(); + private final Runnable finalTimeout = () -> finish(); + + private NativeSpeech(Activity activity, long id, String locale, boolean preferOnDevice) { + this.activity = activity; + this.id = id; + this.locale = locale; + this.preferOnDevice = preferOnDevice; + } + + public static boolean supported(Activity activity) { + // Querying the service does not start it or request any permission. + return SpeechRecognizer.isRecognitionAvailable(activity) || onDeviceAvailable(activity); + } + + private static boolean onDeviceAvailable(Context context) { + if (Build.VERSION.SDK_INT < 31) return false; + try { + return (Boolean) SpeechRecognizer.class.getMethod("isOnDeviceRecognitionAvailable", Context.class) + .invoke(null, context); + } catch (ReflectiveOperationException | RuntimeException error) { + return false; + } + } + + public static void start(Activity activity, long id, String locale, boolean preferOnDevice) { + MAIN.post(() -> { + NativeSpeech session = new NativeSpeech(activity, id, locale, preferOnDevice); + SESSIONS.put(id, session); + session.begin(); + }); + } + + public static void stop(long id, boolean cancel) { + MAIN.post(() -> { + NativeSpeech session = SESSIONS.get(id); + if (session == null) return; + session.listening = false; + MAIN.removeCallbacks(session.restart); + if (cancel) { + session.dispose(); + } else if (!session.awaitingResult) { + session.finish(); + } else { + try { + session.recognizer.stopListening(); + MAIN.postDelayed(session.finalTimeout, 3000); + } catch (RuntimeException error) { + session.finish(); + } + } + }); + } + + private void begin() { + if (activity.isFinishing() || activity.isDestroyed()) { + // Bound to this discarded Activity, not to the device. + fail("The application is no longer active."); + return; + } + if (activity.checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + // Ask once, then resume from the top. The guard also stops a recognizer + // that reports the permission as missing even after a grant from looping. + if (askedForPermission) { + fail("Allow microphone access in Android settings to use speech input.", ERR_PERMISSION); + return; + } + askedForPermission = true; + permissionRequest = SpeechPermissionFragment.request(activity, outcome -> { + if (disposed) return; + permissionRequest = null; + if (outcome == SpeechPermissionFragment.Outcome.GRANTED) { + begin(); + } else if (outcome == SpeechPermissionFragment.Outcome.CANCELLED) { + finish(); + } else { + fail("Microphone access is needed for speech input.", ERR_PERMISSION); + } + }); + return; + } + try { + activity.getApplication().registerActivityLifecycleCallbacks(this); + listen(); + } catch (RuntimeException error) { + fail("Unable to start the system speech recognition service.", ERR_UNAVAILABLE); + } + } + + private SpeechRecognizer createRecognizer() { + if (preferOnDevice && !triedSystemFallback && onDeviceAvailable(activity)) { + try { + SpeechRecognizer deviceRecognizer = (SpeechRecognizer) SpeechRecognizer.class + .getMethod("createOnDeviceSpeechRecognizer", Context.class).invoke(null, activity); + if (deviceRecognizer != null) return deviceRecognizer; + } catch (ReflectiveOperationException | RuntimeException error) { + // Some vendor services advertise on-device support without providing it. + } + } + return SpeechRecognizer.isRecognitionAvailable(activity) + ? SpeechRecognizer.createSpeechRecognizer(activity) : null; + } + + private void listen() { + if (disposed || !listening) return; + pending = ""; + Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); + intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM); + intent.putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true); + intent.putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1); + if (locale != null && !locale.isEmpty()) intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE, locale); + // On older devices this is a preference: the installed system service may ignore it. + intent.putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, preferOnDevice && !triedSystemFallback); + try { + if (recognizer == null) recognizer = createRecognizer(); + if (recognizer == null) { + fail("No speech recognition service is installed on this device.", ERR_UNAVAILABLE); + return; + } + // Each attempt gets a new listener generation. Late callbacks from + // an utterance or cancelled client must not affect its successor. + installListener(); + awaitingResult = true; + recognizer.startListening(intent); + } catch (RuntimeException error) { + fail("Unable to start microphone recording for speech input.", ERR_AUDIO); + } + } + + private void installListener() { + // Destroyed services can still have Binder callbacks queued. Keep those + // from committing an old hypothesis after switching to the fallback. + final int generation = ++recognizerGeneration; + recognizer.setRecognitionListener(new RecognitionListener() { + private boolean current() { return !disposed && generation == recognizerGeneration; } + @Override public void onReadyForSpeech(Bundle params) { + if (current()) NativeSpeech.this.onReadyForSpeech(params); + } + @Override public void onBeginningOfSpeech() {} + @Override public void onRmsChanged(float rms) { + if (current()) NativeSpeech.this.onRmsChanged(rms); + } + @Override public void onBufferReceived(byte[] buffer) {} + @Override public void onEndOfSpeech() { + if (current()) NativeSpeech.this.onEndOfSpeech(); + } + @Override public void onError(int error) { + if (current()) NativeSpeech.this.onError(error); + } + @Override public void onResults(Bundle results) { + if (current()) NativeSpeech.this.onResults(results); + } + @Override public void onPartialResults(Bundle results) { + if (current()) NativeSpeech.this.onPartialResults(results); + } + @Override public void onEvent(int type, Bundle params) {} + }); + } + + private String transcript(Bundle results) { + ArrayList values = results.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION); + return values == null || values.isEmpty() || values.get(0) == null ? "" : values.get(0); + } + + private void commitPending() { + if (!pending.isEmpty()) { + event(id, 2, pending, 0); + pending = ""; + } + } + + private void nextUtterance() { + awaitingResult = false; + event(id, 3, null, 0); + if (listening) MAIN.postDelayed(restart, 150); + else finish(); + } + + private void retryTransientError(int error) { + commitPending(); + if (transientFailures >= MAX_TRANSIENT_RETRIES) { + fail("The system speech service is still unavailable. Try again in a moment."); + return; + } + ++recognizerGeneration; + // A client error can mean its Binder connection has died. Recreate + // that client after releasing the old one, preserving on-device choice. + if (error == SpeechRecognizer.ERROR_CLIENT) { + try { recognizer.cancel(); } catch (RuntimeException ignored) {} + try { recognizer.destroy(); } catch (RuntimeException ignored) {} + recognizer = null; + } + event(id, 3, null, 0); + MAIN.removeCallbacks(restart); + // Do not reset this on onReadyForSpeech: a broken service can report + // ready and fail repeatedly without ever completing an utterance. + MAIN.postDelayed(restart, 500L << transientFailures++); + } + + private void finish() { + if (disposed) return; + commitPending(); + dispose(); + event(id, 4, null, 0); + } + + private void fail(String message) { + fail(message, ERR_OTHER); + } + + private void fail(String message, int kind) { + if (disposed) return; + dispose(); + event(id, kind, message, 0); + } + + private void dispose() { + if (disposed) return; + disposed = true; + listening = false; + MAIN.removeCallbacks(restart); + MAIN.removeCallbacks(finalTimeout); + SESSIONS.remove(id); + if (permissionRequest != null) { + permissionRequest.cancel(); + permissionRequest = null; + } + activity.getApplication().unregisterActivityLifecycleCallbacks(this); + if (recognizer != null) { + try { recognizer.cancel(); } catch (RuntimeException ignored) {} + try { recognizer.destroy(); } catch (RuntimeException ignored) {} + recognizer = null; + } + } + + @Override public void onReadyForSpeech(Bundle params) { + if (disposed || !listening || !awaitingResult) return; + if (!started) { + started = true; + event(id, 0, null, 0); + } + } + @Override public void onBeginningOfSpeech() {} + @Override public void onRmsChanged(float rms) { + if (!disposed && listening) event(id, 3, null, Float.isNaN(rms) ? 0 : Math.max(0, Math.min(1, rms / 10.0f))); + } + @Override public void onBufferReceived(byte[] buffer) {} + @Override public void onEndOfSpeech() { + if (!disposed) event(id, 3, null, 0); + } + @Override public void onPartialResults(Bundle results) { + if (disposed || !awaitingResult || results == null) return; + String text = transcript(results); + if (!text.isEmpty() && !text.equals(pending)) { + pending = text; + event(id, 1, text, 0); + } + } + @Override public void onResults(Bundle results) { + if (disposed || !awaitingResult) return; + awaitingResult = false; + transientFailures = 0; + String text = results == null ? "" : transcript(results); + if (!text.isEmpty()) pending = text; + commitPending(); + nextUtterance(); + } + @Override public void onError(int error) { + if (disposed || !awaitingResult) return; + awaitingResult = false; + if (error == SpeechRecognizer.ERROR_NO_MATCH || error == SpeechRecognizer.ERROR_SPEECH_TIMEOUT) { + transientFailures = 0; + commitPending(); + nextUtterance(); + return; + } + if (!listening) { + finish(); + return; + } + if (error == SpeechRecognizer.ERROR_RECOGNIZER_BUSY || error == SpeechRecognizer.ERROR_CLIENT) { + retryTransientError(error); + return; + } + // On-device recognition may exist without a model for the requested + // language. Let the regular native service handle it when available. + if ((error == 12 || error == 13) && preferOnDevice && !triedSystemFallback + && SpeechRecognizer.isRecognitionAvailable(activity)) { + triedSystemFallback = true; + try { + ++recognizerGeneration; + recognizer.destroy(); + recognizer = SpeechRecognizer.createSpeechRecognizer(activity); + installListener(); + commitPending(); + nextUtterance(); + return; + } catch (RuntimeException ignored) { + // Continue with the actionable language error below. + } + } + switch (error) { + case SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS: + fail("Allow microphone access in Android settings to use speech input.", ERR_PERMISSION); break; + case SpeechRecognizer.ERROR_AUDIO: + fail("The microphone is unavailable. Check whether another app is using it.", ERR_AUDIO); break; + case SpeechRecognizer.ERROR_NETWORK: + case SpeechRecognizer.ERROR_NETWORK_TIMEOUT: + fail("The system speech service needs a network connection. Check your connection and try again."); break; + case 12: // ERROR_LANGUAGE_NOT_SUPPORTED (API 31) + case 13: // ERROR_LANGUAGE_UNAVAILABLE (API 31) + fail("The system speech service does not have recognition support for this language.", ERR_LANGUAGE); break; + default: + fail("The system speech service stopped (error " + error + "). Try again."); + } + } + @Override public void onEvent(int type, Bundle params) {} + @Override public void onActivityPaused(Activity other) { + if (other == activity) finish(); + } + @Override public void onActivityDestroyed(Activity other) { + if (other == activity) finish(); + } + @Override public void onActivityCreated(Activity activity, Bundle state) {} + @Override public void onActivityStarted(Activity activity) {} + @Override public void onActivityResumed(Activity activity) {} + @Override public void onActivityStopped(Activity activity) {} + @Override public void onActivitySaveInstanceState(Activity activity, Bundle state) {} +} diff --git a/crates/speech/java/dev/robius/speech/SpeechPermissionFragment.java b/crates/speech/java/dev/robius/speech/SpeechPermissionFragment.java new file mode 100644 index 0000000..741d514 --- /dev/null +++ b/crates/speech/java/dev/robius/speech/SpeechPermissionFragment.java @@ -0,0 +1,168 @@ +package dev.robius.speech; + +import android.app.Activity; +import android.app.Fragment; +import android.app.FragmentManager; +import android.content.pm.PackageManager; +import android.os.Bundle; + +/** + * Headless fragment that asks for RECORD_AUDIO. `Fragment.requestPermissions` routes the + * result back here rather than to the activity's own `onRequestPermissionsResult`, so an + * application needs no permission plumbing of its own. The request belongs to the current + * Activity and is cancelled if that Activity is recreated or destroyed. + */ +public final class SpeechPermissionFragment extends Fragment { + public enum Outcome { GRANTED, DENIED, CANCELLED } + /** Delivered once on the UI thread, unless the caller cancels first. */ + public interface Result { void onResult(Outcome outcome); } + + // Must be <= 0xffff: android.app.Fragment encodes its index in the upper 16 bits. + private static final int REQUEST_CODE = 0x5350; + private static final String HOST_TAG = "dev.robius.speech.PermissionHost"; + private static final String TAG = "dev.robius.speech.SpeechPermissionFragment"; + private static final String[] PERMISSIONS = { android.Manifest.permission.RECORD_AUDIO }; + + // Null after delivery or cancellation; the OS dialog can outlive its caller. + private Result callback; + private boolean launched; + private boolean completed; + private Fragment host; + private FragmentManager manager; + private FragmentManager.FragmentLifecycleCallbacks lifecycle; + + /** Required for Fragment transactions; this helper is never saved for restoration. */ + public SpeechPermissionFragment() {} + + private SpeechPermissionFragment(Result callback) { + this.callback = callback; + } + + /** Asks without blocking. Called on the UI thread by NativeSpeech. */ + public static SpeechPermissionFragment request(Activity activity, Result callback) { + SpeechPermissionFragment fragment = null; + try { + if (activity.isFinishing() || activity.isDestroyed()) { + callback.onResult(Outcome.CANCELLED); + return null; + } + FragmentManager manager = activity.getFragmentManager(); + Fragment existing = manager.findFragmentByTag(HOST_TAG); + if (existing != null) { + Fragment child = existing.getChildFragmentManager().findFragmentByTag(TAG); + if (child instanceof SpeechPermissionFragment) { + SpeechPermissionFragment pending = (SpeechPermissionFragment) child; + if (!pending.completed) { + if (pending.callback != null) { + callback.onResult(Outcome.CANCELLED); + return null; + } + // Cancelling a session cannot dismiss the OS dialog. Transfer + // its pending result instead of launching a duplicate request. + pending.callback = callback; + return pending; + } + } + // Includes an empty framework host restored after process death. + manager.beginTransaction().remove(existing).commitNowAllowingStateLoss(); + } + fragment = new SpeechPermissionFragment(callback); + fragment.manager = manager; + fragment.host = new Fragment(); + fragment.lifecycle = fragment.hostLifecycle(); + manager.registerFragmentLifecycleCallbacks(fragment.lifecycle, false); + manager.beginTransaction().add(fragment.host, HOST_TAG).commitNowAllowingStateLoss(); + fragment.host.getChildFragmentManager().beginTransaction() + .add(fragment, TAG).commitNowAllowingStateLoss(); + return fragment.completed ? null : fragment; + } catch (RuntimeException error) { + if (fragment == null) { + callback.onResult(Outcome.CANCELLED); + } else { + fragment.deliver(Outcome.CANCELLED); + if (fragment.host.isAdded()) fragment.removeSelf(); + else fragment.unregisterLifecycle(); + } + return null; + } + } + + private FragmentManager.FragmentLifecycleCallbacks hostLifecycle() { + return new FragmentManager.FragmentLifecycleCallbacks() { + @Override public void onFragmentSaveInstanceState(FragmentManager fm, Fragment f, Bundle state) { + // The helper comes from an embedded child DEX, which Android's Activity + // class loader cannot restore. Persist only our plain framework host. + // This callback runs after child state is saved. The host is exclusively + // ours and has no other state; never modify another fragment's bundle. + if (f == host) state.clear(); + } + @Override public void onFragmentDetached(FragmentManager fm, Fragment f) { + if (f == host) unregisterLifecycle(); + } + }; + } + + private void unregisterLifecycle() { + if (lifecycle != null) { + manager.unregisterFragmentLifecycleCallbacks(lifecycle); + lifecycle = null; + } + } + + /** Release only this session's callback; a launched dialog may still return. */ + public void cancel() { + callback = null; + if (!launched) { + completed = true; + removeSelf(); + } + } + + @Override public void onResume() { + super.onResume(); + if (completed || (!launched && callback == null)) { + removeSelf(); + return; + } + if (!launched) { + launched = true; + try { + requestPermissions(PERMISSIONS, REQUEST_CODE); + } catch (RuntimeException error) { + deliver(Outcome.CANCELLED); + removeSelf(); + } + } + } + + @Override public void onRequestPermissionsResult(int requestCode, String[] permissions, int[] results) { + if (requestCode != REQUEST_CODE) return; + // An empty array means the request was interrupted, which counts as denied. + boolean granted = results.length > 0 && results[0] == PackageManager.PERMISSION_GRANTED; + deliver(granted ? Outcome.GRANTED : Outcome.DENIED); + removeSelf(); + } + + @Override public void onDestroy() { + super.onDestroy(); + // Neither fragment is retained. End the old Activity's session now; a + // later grant must never resume recognition against its destroyed Activity. + deliver(Outcome.CANCELLED); + } + + private void deliver(Outcome outcome) { + if (completed) return; + completed = true; + Result pending = callback; + callback = null; + if (pending != null) pending.onResult(outcome); + } + + private void removeSelf() { + // Removal can be requested from within a Fragment lifecycle callback. + // Keep the save-state guard until the host actually detaches. + if (host != null && host.isAdded()) { + manager.beginTransaction().remove(host).commitAllowingStateLoss(); + } + } +} diff --git a/crates/speech/src/android.rs b/crates/speech/src/android.rs new file mode 100644 index 0000000..5483b3c --- /dev/null +++ b/crates/speech/src/android.rs @@ -0,0 +1,120 @@ +//! Android dictation, bridged to the system `SpeechRecognizer`. +//! +//! The Java side in `java/dev/robius/speech/` does the real work, including the +//! runtime permission request; this module just loads it and calls into it. That +//! Java is compiled to DEX and embedded in the library, so an app doesn't need to +//! add anything to its own Activity. + +use std::sync::{Mutex, OnceLock}; +use jni::{JNIEnv, NativeMethod, objects::{GlobalRef, JClass, JObject, JString, JValue}, sys::{jfloat, jint, jlong}}; +use crate::{codes, error_kind_from_code, NativeSpeechEvent, NativeSpeechOptions, SpeechError, SpeechErrorKind}; + +static BRIDGE: OnceLock>> = OnceLock::new(); + +fn with_env(run: impl FnOnce(&mut JNIEnv, &JObject) -> jni::errors::Result) -> Result { + // The environment provider selects the host toolkit's current Activity. + // ndk-context can panic before initialization; report unavailable instead + // of unwinding through a caller merely checking for speech support. + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + robius_android_env::with_activity(|env, activity| { + let result = env.with_local_frame(32, |env| run(env, activity)); + if env.exception_check().unwrap_or(false) { + // Never leave an exception pending on the application's thread. + let _ = env.exception_clear(); + } + result + }) + })).map_err(|_| SpeechError::new( + SpeechErrorKind::Unavailable, + "Unable to access the Android application context. Initialize it before using speech recognition.", + ))?; + result.and_then(|inner| inner).map_err(|error| SpeechError::new( + SpeechErrorKind::Unavailable, + format!("Unable to access Android speech recognition: {error}"), + )) +} + +pub(super) const ENGINE: &str = "android-speechrecognizer"; + +fn bridge(env: &mut JNIEnv, activity: &JObject) -> jni::errors::Result { + let mut cached = BRIDGE.get_or_init(|| Mutex::new(None)).lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(class) = cached.as_ref() { + return Ok(class.clone()); + } + if activity.is_null() { + return Err(jni::errors::Error::NullPtr("Android Activity")); + } + // Borrow the current Activity without caching it: Android can replace it + // after rotation or recreation. Only the loaded bridge class is retained. + let parent = env.call_method(activity, "getClassLoader", "()Ljava/lang/ClassLoader;", &[])?.l()?; + let dex = env.byte_array_from_slice(include_bytes!(concat!(env!("OUT_DIR"), "/classes.dex")))?; + let bytes = env.call_static_method("java/nio/ByteBuffer", "wrap", "([B)Ljava/nio/ByteBuffer;", &[JValue::Object(&dex)])?.l()?; + let loader = env.new_object("dalvik/system/InMemoryDexClassLoader", "(Ljava/nio/ByteBuffer;Ljava/lang/ClassLoader;)V", + &[JValue::Object(&bytes), JValue::Object(&parent)])?; + let name = env.new_string("dev.robius.speech.NativeSpeech")?; + let class = JClass::from(env.call_method(loader, "loadClass", "(Ljava/lang/String;)Ljava/lang/Class;", &[JValue::Object(&name)])?.l()?); + env.register_native_methods(&class, &[NativeMethod { + name: "event".into(), + sig: "(JILjava/lang/String;F)V".into(), + fn_ptr: native_event as *mut std::ffi::c_void, + }])?; + let global = env.new_global_ref(class)?; + *cached = Some(global.clone()); + Ok(global) +} + +extern "system" fn native_event(mut env: JNIEnv, _: JClass, id: jlong, kind: jint, text: JString, level: jfloat) { + // Java invokes this on the main thread. Decoding/callback failures must not + // unwind through JNI, and app callbacks are isolated by crate::emit as well. + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let text = if text.is_null() { String::new() } + else { env.get_string(&text).map(String::from).unwrap_or_default() }; + let event = match kind { + codes::STARTED => NativeSpeechEvent::Started, + codes::PARTIAL | codes::FINAL => NativeSpeechEvent::Transcript { text, is_final: kind == codes::FINAL }, + codes::LEVEL => NativeSpeechEvent::AudioLevel(if level.is_finite() { level.clamp(0.0, 1.0) } else { 0.0 }), + codes::STOPPED => NativeSpeechEvent::Stopped, + codes::ERROR..=codes::ERROR_AUDIO => { + NativeSpeechEvent::Error(SpeechError::new(error_kind_from_code(kind), text)) + } + _ => return, + }; + crate::emit(id as u64, event); + })); + if env.exception_check().unwrap_or(false) { let _ = env.exception_clear(); } +} + +pub(super) fn is_supported() -> bool { + with_env(|env, activity| { + if activity.is_null() { return Ok(false); } + let class = bridge(env, activity)?; + let class: &JClass = class.as_obj().into(); + env.call_static_method(class, "supported", "(Landroid/app/Activity;)Z", &[JValue::Object(activity)])?.z() + }).unwrap_or(false) +} + +pub(super) fn start(id: u64, options: &NativeSpeechOptions) -> Result<(), SpeechError> { + with_env(|env, activity| { + if activity.is_null() { return Err(jni::errors::Error::NullPtr("Android Activity")); } + let class = bridge(env, activity)?; + let class: &JClass = class.as_obj().into(); + let locale = env.new_string(options.locale.as_deref().unwrap_or(""))?; + env.call_static_method(class, "start", "(Landroid/app/Activity;JLjava/lang/String;Z)V", &[ + JValue::Object(activity), JValue::Long(id as jlong), JValue::Object(&locale), + JValue::Bool(options.prefer_on_device.into()), + ])?; + Ok(()) + }) +} + +pub(super) fn stop(id: u64, cancel: bool) { + let result = with_env(|env, activity| { + let class = bridge(env, activity)?; + let class: &JClass = class.as_obj().into(); + env.call_static_method(class, "stop", "(JZ)V", &[JValue::Long(id as jlong), JValue::Bool(cancel.into())])?; + Ok(()) + }); + if let Err(error) = result { + crate::emit(id, NativeSpeechEvent::Error(error)); + } +} diff --git a/crates/speech/src/apple.rs b/crates/speech/src/apple.rs new file mode 100644 index 0000000..7ca4811 --- /dev/null +++ b/crates/speech/src/apple.rs @@ -0,0 +1,43 @@ +//! macOS and iOS dictation, bridged to `SFSpeechRecognizer` and `AVAudioEngine`. +//! +//! The Swift side in `swift/NativeSpeech.swift` does the real work, including +//! asking for both permissions; this module is just the C ABI between the two. + +use std::ffi::{c_char, CStr, CString}; +use crate::{codes, emit, error_kind_from_code, NativeSpeechEvent, NativeSpeechOptions, SpeechError}; + +extern "C" { + fn robius_speech_start(id: u64, locale: *const c_char, prefer_on_device: bool); + fn robius_speech_stop(id: u64, cancel: bool); +} + +pub(super) const ENGINE: &str = "apple-sfspeech"; + +pub(super) fn is_supported() -> bool { true } + +pub(super) fn start(id: u64, options: &NativeSpeechOptions) -> Result<(), SpeechError> { + let locale = CString::new(options.locale.as_deref().unwrap_or_default()).unwrap(); + unsafe { robius_speech_start(id, locale.as_ptr(), options.prefer_on_device); } + Ok(()) +} + +pub(super) fn stop(id: u64, cancel: bool) { + unsafe { robius_speech_stop(id, cancel); } +} + +#[no_mangle] +extern "C" fn robius_speech_event(id: u64, kind: i32, text: *const c_char, level: f32) { + let text = || { + if text.is_null() { String::new() } + else { unsafe { CStr::from_ptr(text) }.to_string_lossy().into_owned() } + }; + let event = match kind { + codes::STARTED => NativeSpeechEvent::Started, + codes::PARTIAL => NativeSpeechEvent::Transcript { text: text(), is_final: false }, + codes::FINAL => NativeSpeechEvent::Transcript { text: text(), is_final: true }, + codes::LEVEL => NativeSpeechEvent::AudioLevel(if level.is_finite() { level.clamp(0.0, 1.0) } else { 0.0 }), + codes::STOPPED => NativeSpeechEvent::Stopped, + _ => NativeSpeechEvent::Error(SpeechError::new(error_kind_from_code(kind), text())), + }; + emit(id, event); +} diff --git a/crates/speech/src/lib.rs b/crates/speech/src/lib.rs new file mode 100644 index 0000000..733fbfd --- /dev/null +++ b/crates/speech/src/lib.rs @@ -0,0 +1,316 @@ +//! Native streaming speech-to-text input and microphone capture for Rust apps. +//! +//! Recognition keeps going across as many utterances as the user speaks, until you +//! stop it. An "utterance" is just a chunk of words spoken by the user: partial +//! transcripts replace the current one, whereas a final transcript commits it. +//! +//! Note that speech always stays as plain text here. This crate never interprets +//! spoken works as commands or synthesizes key events from them, +//! so saying something like "enter" will just type the word "enter". +//! +//! Apple builds need the Xcode command line tools (`xcode-select --install`), and +//! support macOS 11+ and iOS 13+. For macOS builds, set the `MACOSX_DEPLOYMENT_TARGET` +//! env var to 11.0 or newer for the whole Cargo invocation, including the final app. +//! This mostly just matters on Intel x86 macs, but it doesn't hurt to always set it. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::atomic::{AtomicU64, Ordering}; + +#[cfg(any(target_os = "macos", target_os = "ios"))] +mod apple; +#[cfg(all(target_os = "android", native_speech_android))] +mod android; +#[cfg(target_os = "windows")] +mod windows; + +#[cfg(any(target_os = "macos", target_os = "ios"))] +use apple as backend; +#[cfg(all(target_os = "android", native_speech_android))] +use android as backend; +#[cfg(target_os = "windows")] +use windows as backend; + +#[derive(Clone, Debug)] +pub struct NativeSpeechOptions { + /// A BCP-47 locale (e.g., `"en-US"`), or the OS's default speech language. + pub locale: Option, + /// Prefer on-device recognition when the native service supports it. + /// Otherwise the system service may need a network connection. + pub prefer_on_device: bool, +} + +impl Default for NativeSpeechOptions { + fn default() -> Self { + Self { locale: None, prefer_on_device: true } + } +} + +/// The cause of a speech failure, so you can handle each one properly +/// instead of matching on message text. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +#[non_exhaustive] +pub enum SpeechErrorKind { + /// The user denied permission, or your app is missing the privacy + /// declaration it needs in order to even ask. + /// Usually you'll want to tell the user to enable audio and/or speech + /// permissions in their system settings. + PermissionDenied, + /// The audio input or speech recognition service just doesn't exist here, + /// so retrying will never work. + Unavailable, + /// The recognizer doesn't support the language you asked for. + Language, + /// The microphone is missing, busy, or changed mid-session. + /// Starting things again will likely work. + Audio, + /// Another recording is already active; only one session can be active at a time. + Busy, + /// Anything else that the OS reported. + Other, +} + +/// A [`SpeechErrorKind`] that you can match on, plus the message the OS gave us. +/// `Display` writes just that message, which is already worded for end users. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SpeechError { + kind: SpeechErrorKind, + message: String, +} + +impl SpeechError { + pub fn new(kind: SpeechErrorKind, message: impl Into) -> Self { + Self { kind, message: message.into() } + } + + pub fn kind(&self) -> SpeechErrorKind { + self.kind + } + + pub fn message(&self) -> &str { + &self.message + } +} + +impl std::fmt::Display for SpeechError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for SpeechError {} + +#[derive(Clone, Debug, PartialEq)] +pub enum NativeSpeechEvent { + /// Permissions were granted and the microphone is actually recording. + Started, + /// Recognized text is available. + /// + /// If `is_final` is `false`, the text is a partial transcript that may be updated/changed later. + /// If `is_final` is `true`, the text is a final transcript that is fully "committed". + Transcript { + text: String, + is_final: bool, + }, + /// Microphone amplitude, normalized between 0 and 1, which you can use to + /// display a level meter or similar sound wave animation. + AudioLevel(f32), + /// Capture and final recognition are done. This is a final event. + Stopped, + /// A final error, e.g., denied permissions or unavailable hardware. + Error(SpeechError), +} + +/// Event codes shared with the Swift and Java bridges. The error kinds are just +/// extra values of the same `kind` argument, so classifying them needs no ABI change. +#[cfg(any(target_os = "macos", target_os = "ios", all(target_os = "android", native_speech_android)))] +#[allow(dead_code)] // Not every backend uses every code. +pub(crate) mod codes { + pub(crate) const STARTED: i32 = 0; + pub(crate) const PARTIAL: i32 = 1; + pub(crate) const FINAL: i32 = 2; + pub(crate) const LEVEL: i32 = 3; + pub(crate) const STOPPED: i32 = 4; + pub(crate) const ERROR: i32 = 5; + pub(crate) const ERROR_PERMISSION: i32 = 6; + pub(crate) const ERROR_UNAVAILABLE: i32 = 7; + pub(crate) const ERROR_LANGUAGE: i32 = 8; + pub(crate) const ERROR_AUDIO: i32 = 9; +} + +/// Unknown codes fall back to `Other`, so a newer bridge still reports its message. +#[cfg(any(target_os = "macos", target_os = "ios", all(target_os = "android", native_speech_android)))] +pub(crate) fn error_kind_from_code(code: i32) -> SpeechErrorKind { + match code { + codes::ERROR_PERMISSION => SpeechErrorKind::PermissionDenied, + codes::ERROR_UNAVAILABLE => SpeechErrorKind::Unavailable, + codes::ERROR_LANGUAGE => SpeechErrorKind::Language, + codes::ERROR_AUDIO => SpeechErrorKind::Audio, + _ => SpeechErrorKind::Other, + } +} + +type Callback = Arc; +static SESSIONS: OnceLock>> = OnceLock::new(); +static NEXT_SESSION: AtomicU64 = AtomicU64::new(1); + +fn sessions() -> &'static Mutex> { + SESSIONS.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// Owns the system microphone and recognition task; dropping it cancels both. +/// +/// Importantly, only one session can be active at a time. +/// +/// Callbacks arrive on platform-dependent native OS threads, so you'll want to +/// forward them to your main UI thread or event loop. Note that it's possible for +/// an in-progress callback to continue executing after you cancel the session. +pub struct NativeSpeechSession { + id: u64, +} + +/// The short, stable name of the recognizer this build uses, e.g., `"apple-sfspeech"`. +/// Handy for logs and bug reports. +pub fn engine_name() -> &'static str { + backend::ENGINE +} + +impl NativeSpeechSession { + /// Whether this platform offers a native speech recognizer at all. + /// + /// This is false on Linux and the web, so you'll usually want to hide any + /// dictation button entirely rather than let it fail when pressed. + pub fn is_supported() -> bool { + backend::is_supported() + } + + /// Starts dictating, requesting microphone (and on Apple, speech recognition) + /// permission if needed, so you don't need any platform-specific permission code. + /// + /// This returns immediately rather than blocking on the permission prompt. + /// The `Started` event tells you when the microphone is actually recording, + /// and a refusal arrives as [`SpeechErrorKind::PermissionDenied`]. + pub fn start( + options: NativeSpeechOptions, + callback: impl Fn(NativeSpeechEvent) + Send + Sync + 'static, + ) -> Result { + if !Self::is_supported() { + return Err(SpeechError::new( + SpeechErrorKind::Unavailable, + "Native speech recognition is not available on this platform.", + )); + } + if options.locale.as_ref().is_some_and(|locale| locale.contains('\0')) { + return Err(SpeechError::new( + SpeechErrorKind::Language, + "The speech recognition language is invalid.", + )); + } + let id = NEXT_SESSION.fetch_add(1, Ordering::Relaxed); + { + let mut sessions = sessions().lock().unwrap(); + if !sessions.is_empty() { + return Err(SpeechError::new( + SpeechErrorKind::Busy, + "Another speech recording is already active.", + )); + } + sessions.insert(id, Arc::new(callback)); + } + if let Err(error) = backend::start(id, &options) { + sessions().lock().unwrap().remove(&id); + return Err(error); + } + Ok(Self { id }) + } + + /// Closes the microphone, but still lets the final recognition result arrive. + pub fn stop(&self) { + backend::stop(self.id, false); + } + + /// Stops immediately and discards anything that's still pending. + pub fn cancel(&self) { + sessions().lock().unwrap().remove(&self.id); + backend::stop(self.id, true); + } +} + +impl Drop for NativeSpeechSession { + fn drop(&mut self) { + self.cancel(); + } +} + +/// Cancels every active session, for things like suspending or quitting the app. +pub fn cancel_all() { + let ids: Vec<_> = sessions().lock().unwrap().drain().map(|(id, _)| id).collect(); + for id in ids { + backend::stop(id, true); + } +} + +#[cfg(any(test, target_os = "macos", target_os = "ios", all(target_os = "android", native_speech_android), target_os = "windows"))] +pub(crate) fn emit(id: u64, event: NativeSpeechEvent) { + let callback = { + let mut sessions = sessions().lock().unwrap(); + if matches!(event, NativeSpeechEvent::Stopped | NativeSpeechEvent::Error(_)) { + sessions.remove(&id) + } else { + sessions.get(&id).cloned() + } + }; + if let Some(callback) = callback { + // Never unwind through a native callback boundary. + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| callback(event))); + } +} + +#[cfg(not(any(target_os = "macos", target_os = "ios", all(target_os = "android", native_speech_android), target_os = "windows")))] +mod backend { + use super::{NativeSpeechOptions, SpeechError, SpeechErrorKind}; + pub(super) const ENGINE: &str = "none"; + pub(super) fn is_supported() -> bool { false } + pub(super) fn start(_: u64, _: &NativeSpeechOptions) -> Result<(), SpeechError> { + Err(SpeechError::new( + SpeechErrorKind::Unavailable, + "Native speech recognition is not available on this platform.", + )) + } + pub(super) fn stop(_: u64, _: bool) {} +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn terminal_and_cancelled_sessions_discard_late_native_results() { + let received = Arc::new(Mutex::new(Vec::new())); + let output = received.clone(); + sessions().lock().unwrap().insert(1001, Arc::new(move |event| { + output.lock().unwrap().push(event); + })); + emit(1001, NativeSpeechEvent::Transcript { text: "draft".into(), is_final: false }); + emit(1001, NativeSpeechEvent::Stopped); + emit(1001, NativeSpeechEvent::Transcript { text: "stale".into(), is_final: true }); + assert_eq!(*received.lock().unwrap(), vec![ + NativeSpeechEvent::Transcript { text: "draft".into(), is_final: false }, + NativeSpeechEvent::Stopped, + ]); + + let output = received.clone(); + sessions().lock().unwrap().insert(1002, Arc::new(move |event| { + output.lock().unwrap().push(event); + })); + // A previous session's delayed callback must not be delivered to its + // successor, even when recognition is restarted immediately. + emit(1001, NativeSpeechEvent::Error(SpeechError::new(SpeechErrorKind::Other, "late failure"))); + NativeSpeechSession { id: 1002 }.cancel(); + emit(1002, NativeSpeechEvent::Started); + assert_eq!(received.lock().unwrap().len(), 2); + let sessions = sessions().lock().unwrap(); + assert!(!sessions.contains_key(&1001)); + assert!(!sessions.contains_key(&1002)); + } +} diff --git a/crates/speech/src/windows.rs b/crates/speech/src/windows.rs new file mode 100644 index 0000000..ee0a8f5 --- /dev/null +++ b/crates/speech/src/windows.rs @@ -0,0 +1,440 @@ +//! Windows dictation through SAPI and its native microphone input. +//! +//! We use SAPI rather than the WinRT `SpeechRecognizer` because SAPI works in a +//! plain unpackaged desktop app, whereas WinRT dictation needs package identity. +//! SAPI also uses entirely on-device recognizers that are already installed. +//! +//! Note that every COM interface here stays on one dedicated MTA thread, +//! including when it gets released. +//! See + +use crate::{emit, NativeSpeechEvent, NativeSpeechOptions, SpeechError, SpeechErrorKind}; +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock, mpsc}; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::time::{Duration, Instant}; +use windows::core::{Interface, IUnknown, PCWSTR, PWSTR, HRESULT}; +use windows::Win32::Globalization::LocaleNameToLCID; +use windows::Win32::Media::Speech::*; +use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CoTaskMemFree, CoUninitialize, + CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, +}; + +const RECORDING: u8 = 0; +const STOPPING: u8 = 1; +const CANCELLED: u8 = 2; + +struct Control { + state: AtomicU8, + wake: mpsc::Sender<()>, +} + +static CONTROLS: OnceLock>>> = OnceLock::new(); +// Cancellation can release the public session before its native worker has +// closed the microphone. Serialize native ownership across that short interval. +static MICROPHONE: Mutex<()> = Mutex::new(()); + +fn controls() -> &'static Mutex>> { + CONTROLS.get_or_init(|| Mutex::new(HashMap::new())) +} + +pub(super) fn is_supported() -> bool { + static SUPPORTED: OnceLock = OnceLock::new(); + *SUPPORTED.get_or_init(|| { + // Only query installed engine tokens: do not open the microphone or + // load a recognition model while checking whether to show the control. + // A separate apartment also avoids changing the UI thread's COM model. + std::thread::Builder::new().name("speech-availability".into()).spawn(|| { + let Ok(_apartment) = Apartment::new() else { return false }; + unsafe { + let Ok(category): Result = + CoCreateInstance(&SpObjectTokenCategory, None, CLSCTX_INPROC_SERVER) + else { return false }; + if category.SetId(SPCAT_RECOGNIZERS, false).is_err() { return false; } + let Ok(tokens) = category.EnumTokens(PCWSTR::null(), PCWSTR::null()) else { return false }; + let mut count = 0; + tokens.GetCount(&mut count).is_ok() && count > 0 + } + }).ok().and_then(|thread| thread.join().ok()).unwrap_or(false) + }) +} + +pub(super) fn start(id: u64, options: &NativeSpeechOptions) -> Result<(), SpeechError> { + let (wake, receiver) = mpsc::channel(); + let control = Arc::new(Control { state: AtomicU8::new(RECORDING), wake }); + controls().lock().unwrap_or_else(|poisoned| poisoned.into_inner()).insert(id, control.clone()); + let options = options.clone(); + let spawned = std::thread::Builder::new().name("native-speech".into()).spawn(move || { + worker(id, || { + if control.state.load(Ordering::Acquire) != RECORDING { + Ok(()) + } else { + run(id, &options, &control, &receiver) + } + }); + }); + if let Err(error) = spawned { + controls().lock().unwrap_or_else(|poisoned| poisoned.into_inner()).remove(&id); + return Err(SpeechError::new(SpeechErrorKind::Other, format!("Could not start Windows speech recognition: {error}"))); + } + Ok(()) +} + +fn worker(id: u64, run: impl FnOnce() -> Result<(), SpeechError>) { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _microphone = MICROPHONE.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + run() + })).unwrap_or_else(|_| Err(SpeechError::new(SpeechErrorKind::Other, "Windows speech recognition stopped unexpectedly. Please try again."))); + controls().lock().unwrap_or_else(|poisoned| poisoned.into_inner()).remove(&id); + // COM/audio resources unwind before this terminal callback releases the + // public session, including when an unexpected worker panic occurs. + emit(id, match result { + Ok(()) => NativeSpeechEvent::Stopped, + Err(error) => NativeSpeechEvent::Error(error), + }); +} + +pub(super) fn stop(id: u64, cancel: bool) { + if let Some(control) = controls().lock().unwrap_or_else(|poisoned| poisoned.into_inner()).get(&id) { + // A repeated graceful stop must never undo a cancellation. + control.state.fetch_max(if cancel { CANCELLED } else { STOPPING }, Ordering::AcqRel); + let _ = control.wake.send(()); + } +} + +struct Apartment; + +impl Apartment { + fn new() -> Result { + unsafe { CoInitializeEx(None, COINIT_MULTITHREADED).ok() } + .map_err(|error| failure("Could not initialize Windows speech recognition", error))?; + Ok(Self) + } +} + +impl Drop for Apartment { + fn drop(&mut self) { + unsafe { CoUninitialize() }; + } +} + +struct Recognition { + recognizer: ISpRecognizer, + context: ISpRecoContext, + _grammar: ISpRecoGrammar, +} + +impl Drop for Recognition { + fn drop(&mut self) { + // This is our in-process recognizer, so stopping it cannot affect + // another application's dictation or Windows voice controls. + let _ = unsafe { self.recognizer.SetRecoState(SPRST_INACTIVE_WITH_PURGE) }; + } +} + +pub(super) const ENGINE: &str = "windows-sapi"; + +fn failure(context: &str, error: windows::core::Error) -> SpeechError { + failure_kind(SpeechErrorKind::Other, context, error) +} + +fn failure_kind(kind: SpeechErrorKind, context: &str, error: windows::core::Error) -> SpeechError { + SpeechError::new(kind, format!("{context}. Check Windows microphone access and that a speech recognition language is installed. {error}")) +} + +fn run( + id: u64, + options: &NativeSpeechOptions, + control: &Control, + receiver: &mpsc::Receiver<()>, +) -> Result<(), SpeechError> { + // Declared first so COM is uninitialized after every interface is dropped. + let _apartment = Apartment::new()?; + let recognition = create_recognition(options)?; + if control.state.load(Ordering::Acquire) != RECORDING { + return Ok(()); + } + unsafe { recognition._grammar.SetDictationState(SPRS_ACTIVE) } + .map_err(|error| failure("Could not open the microphone for dictation", error))?; + if control.state.load(Ordering::Acquire) == CANCELLED { + return Ok(()); + } + emit(id, NativeSpeechEvent::Started); + let mut stop_deadline = None; + let mut partial = String::new(); + loop { + let state = control.state.load(Ordering::Acquire); + if state == CANCELLED { + return Ok(()); + } + if state == STOPPING && stop_deadline.is_none() { + // INACTIVE closes native capture and lets buffered audio finish. + // PURGE is reserved for cancellation and the cleanup guard. + unsafe { recognition.recognizer.SetRecoState(SPRST_INACTIVE) } + .map_err(|error| failure("Could not finish speech recognition", error))?; + stop_deadline = Some(Instant::now() + Duration::from_secs(5)); + } + while let Some(event) = next_event(&recognition.context)? { + if control.state.load(Ordering::Acquire) == CANCELLED { + return Ok(()); + } + if handle_event(id, &event, &mut partial, control, stop_deadline.is_some())? { + return Ok(()); + } + } + if stop_deadline.is_some_and(|deadline| Instant::now() >= deadline) { + // Some engines never send END_SR_STREAM on a silent recording. + // Preserve the visible hypothesis and always release the device. + finish_partial(id, &mut partial, control); + return Ok(()); + } + // SAPI queues events on its own worker threads. Poll only while this + // session exists; stop/cancel wakes this wait immediately. + let _ = receiver.recv_timeout(Duration::from_millis(20)); + } +} + +/// Returns true when SAPI has ended this stream. +fn handle_event(id: u64, event: &Event, partial: &mut String, control: &Control, stopping: bool) -> Result { + match SPEVENTENUM(event.0._bitfield & 0xffff) { + SPEI_HYPOTHESIS | SPEI_RECOGNITION => { + let text = event.text()?; + let is_final = event.0._bitfield & 0xffff == SPEI_RECOGNITION.0; + if is_final { + partial.clear(); + } else { + partial.clone_from(&text); + } + emit(id, NativeSpeechEvent::Transcript { text, is_final }); + } + SPEI_FALSE_RECOGNITION => { + if control.state.load(Ordering::Acquire) == STOPPING { + // Stopping mid-phrase can make an engine reject its buffered + // tail. Keep the words already shown instead of erasing them. + finish_partial(id, partial, control); + } else { + // Remove a rejected utterance while recording, without + // committing speech the engine did not recognize. + partial.clear(); + emit(id, NativeSpeechEvent::Transcript { text: String::new(), is_final: true }); + } + } + SPEI_SR_AUDIO_LEVEL => { + emit(id, NativeSpeechEvent::AudioLevel((event.0.wParam.0 as f32 / 100.0).clamp(0.0, 1.0))); + } + SPEI_END_SR_STREAM => { + // Finalize the displayed hypothesis even when the stream ended + // with an error, before the terminal callback releases the session. + finish_partial(id, partial, control); + let status = HRESULT(event.0.lParam.0 as i32); + if let Err(error) = status.ok() { + return Err(failure_kind(SpeechErrorKind::Audio, "Windows speech recognition lost its audio input", error)); + } + if !stopping { + return Err(SpeechError::new(SpeechErrorKind::Audio, "Windows speech recognition stopped receiving microphone audio. Check the microphone and try again.")); + } + return Ok(true); + } + _ => {} + } + Ok(false) +} + +fn finish_partial(id: u64, partial: &mut String, control: &Control) { + if !partial.is_empty() && control.state.load(Ordering::Acquire) != CANCELLED { + emit(id, NativeSpeechEvent::Transcript { text: std::mem::take(partial), is_final: true }); + } +} + +fn create_recognition(options: &NativeSpeechOptions) -> Result { + unsafe { + let recognizer: ISpRecognizer = CoCreateInstance(&SpInprocRecognizer, None, CLSCTX_INPROC_SERVER) + .map_err(|error| failure_kind(SpeechErrorKind::Unavailable, "Windows speech recognition is unavailable", error))?; + // SAPI's recognizers run on-device, also when prefer_on_device is false. + if let Some(locale) = options.locale.as_deref() { + let name: Vec = locale.encode_utf16().chain(Some(0)).collect(); + let language = LocaleNameToLCID(PCWSTR(name.as_ptr()), 0) & 0xffff; + if language == 0 { + return Err(SpeechError::new(SpeechErrorKind::Language, format!("Windows does not recognize the speech language {locale}."))); + } + let category: ISpObjectTokenCategory = CoCreateInstance(&SpObjectTokenCategory, None, CLSCTX_INPROC_SERVER) + .map_err(|error| failure("Could not find Windows speech recognition languages", error))?; + category.SetId(SPCAT_RECOGNIZERS, false) + .map_err(|error| failure("Could not find Windows speech recognition languages", error))?; + let attributes: Vec = format!("Language={language:x}").encode_utf16().chain(Some(0)).collect(); + let tokens = category.EnumTokens(PCWSTR(attributes.as_ptr()), PCWSTR::null()) + .map_err(|error| failure_kind(SpeechErrorKind::Language, "Could not find the requested speech recognition language", error))?; + let token = tokens.Item(0) + .map_err(|_| SpeechError::new(SpeechErrorKind::Language, format!("Install the Windows speech recognition language for {locale} before dictating in that language.")))?; + recognizer.SetRecognizer(&token) + .map_err(|error| failure_kind(SpeechErrorKind::Language, "Could not use the requested speech recognition language", error))?; + } + let input: ISpObjectToken = default_token(SPCAT_AUDIOIN) + .map_err(|error| failure_kind(SpeechErrorKind::Audio, "No Windows microphone is available", error))?; + recognizer.SetInput(&input, true) + .map_err(|error| failure_kind(SpeechErrorKind::Audio, "Could not select the Windows microphone", error))?; + let context = recognizer.CreateRecoContext() + .map_err(|error| failure("Could not create Windows dictation", error))?; + // SPFEI also includes these two reserved flags, as required by SAPI. + let interest = [SPEI_HYPOTHESIS, SPEI_RECOGNITION, SPEI_FALSE_RECOGNITION, + SPEI_SR_AUDIO_LEVEL, SPEI_END_SR_STREAM, SPEI_RESERVED1, SPEI_RESERVED2] + .into_iter().fold(0, |mask, event| mask | (1u64 << event.0)); + context.SetInterest(interest, interest) + .map_err(|error| failure("Could not subscribe to Windows dictation", error))?; + let grammar = context.CreateGrammar(1) + .map_err(|error| failure("Could not create the dictation grammar", error))?; + grammar.LoadDictation(PCWSTR::null(), SPLO_STATIC) + .map_err(|error| failure_kind(SpeechErrorKind::Language, "The installed Windows speech language does not support dictation", error))?; + Ok(Recognition { recognizer, context, _grammar: grammar }) + } +} + +unsafe fn default_token(category_id: PCWSTR) -> windows::core::Result { + unsafe { + let category: ISpObjectTokenCategory = CoCreateInstance(&SpObjectTokenCategory, None, CLSCTX_INPROC_SERVER)?; + category.SetId(category_id, false)?; + let id = ComText(category.GetDefaultTokenId()?); + let token: ISpObjectToken = CoCreateInstance(&SpObjectToken, None, CLSCTX_INPROC_SERVER)?; + token.SetId(PCWSTR::null(), PCWSTR(id.0.0), false)?; + Ok(token) + } +} + +struct ComText(PWSTR); + +impl Drop for ComText { + fn drop(&mut self) { + unsafe { CoTaskMemFree(Some(self.0.0.cast())) }; + } +} + +struct Event(SPEVENT); + +impl Event { + fn text(&self) -> Result { + let raw = self.0.lParam.0 as *mut std::ffi::c_void; + unsafe { + let result = ISpRecoResult::from_raw_borrowed(&raw) + .ok_or_else(|| SpeechError::new(SpeechErrorKind::Other, "Windows speech recognition returned an empty result."))?; + let mut text = ComText(PWSTR::null()); + result.GetText(0, u32::MAX, true, &mut text.0, None) + .map_err(|error| failure("Could not read Windows dictation text", error))?; + if text.0.is_null() { + return Ok(String::new()); + } + text.0.to_string().map_err(|_| SpeechError::new(SpeechErrorKind::Other, "Windows speech recognition returned invalid text.")) + } + } +} + +impl Drop for Event { + fn drop(&mut self) { + let pointer = self.0.lParam.0 as *mut std::ffi::c_void; + if pointer.is_null() { + return; + } + // SpClearEvent's ownership rules, including failed/ignored events. + match SPEVENTLPARAMTYPE((self.0._bitfield >> 16) & 0xffff) { + SPET_LPARAM_IS_TOKEN | SPET_LPARAM_IS_OBJECT => unsafe { + drop(IUnknown::from_raw(pointer)); + }, + SPET_LPARAM_IS_POINTER | SPET_LPARAM_IS_STRING => unsafe { + CoTaskMemFree(Some(pointer)); + }, + _ => {} + } + } +} + +fn next_event(context: &ISpRecoContext) -> Result, SpeechError> { + let mut event = Event(SPEVENT::default()); + let mut fetched = 0; + unsafe { context.GetEvents(1, &mut event.0, &mut fetched) } + .map_err(|error| failure("Could not receive Windows dictation", error))?; + Ok((fetched != 0).then_some(event)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn recording() -> (u64, Arc>>) { + let id = crate::NEXT_SESSION.fetch_add(1, Ordering::Relaxed) + 10_000; + let events = Arc::new(Mutex::new(Vec::new())); + let output = events.clone(); + crate::sessions().lock().unwrap().insert(id, Arc::new(move |event| { + output.lock().unwrap().push(event); + })); + (id, events) + } + + #[test] + fn a_panicking_worker_releases_the_session_and_does_not_block_its_successor() { + let (id, events) = recording(); + let (wake, _receiver) = mpsc::channel(); + controls().lock().unwrap().insert(id, Arc::new(Control { state: AtomicU8::new(RECORDING), wake })); + worker(id, || panic!("injected worker failure; no COM or microphone is used")); + assert!(matches!(events.lock().unwrap().as_slice(), [NativeSpeechEvent::Error(_)])); + assert!(!crate::sessions().lock().unwrap().contains_key(&id)); + assert!(!controls().lock().unwrap().contains_key(&id)); + + let (next, next_events) = recording(); + worker(next, || Ok(())); + assert_eq!(*next_events.lock().unwrap(), vec![NativeSpeechEvent::Stopped]); + } + + #[test] + fn rejecting_the_last_utterance_during_stop_preserves_it_exactly_once() { + let (id, events) = recording(); + let (wake, _receiver) = mpsc::channel(); + let control = Control { state: AtomicU8::new(STOPPING), wake }; + let mut pending = "last spoken words".to_owned(); + let mut event = Event(SPEVENT::default()); + worker(id, || { + event.0._bitfield = SPEI_FALSE_RECOGNITION.0; + assert!(!handle_event(id, &event, &mut pending, &control, true)?); + event.0._bitfield = SPEI_END_SR_STREAM.0; + assert!(handle_event(id, &event, &mut pending, &control, true)?); + Ok(()) + }); + assert_eq!(*events.lock().unwrap(), vec![ + NativeSpeechEvent::Transcript { text: "last spoken words".into(), is_final: true }, + NativeSpeechEvent::Stopped, + ]); + } + + #[test] + fn a_failed_stream_finalizes_visible_words_before_reporting_the_error() { + let (id, events) = recording(); + let (wake, _receiver) = mpsc::channel(); + let control = Control { state: AtomicU8::new(STOPPING), wake }; + let mut pending = "keep this interrupted utterance".to_owned(); + let mut event = Event(SPEVENT::default()); + event.0._bitfield = SPEI_END_SR_STREAM.0; + event.0.lParam.0 = 0x80004005u32 as i32 as isize; // E_FAIL, not an owned pointer. + worker(id, || handle_event(id, &event, &mut pending, &control, true).map(|_| ())); + let events = events.lock().unwrap(); + assert!(matches!(events.as_slice(), [ + NativeSpeechEvent::Transcript { text, is_final: true }, NativeSpeechEvent::Error(_) + ] if text == "keep this interrupted utterance")); + } + + #[test] + fn normal_rejection_clears_the_hypothesis_and_cancellation_does_not_commit_it() { + let (id, events) = recording(); + let (wake, _receiver) = mpsc::channel(); + let control = Control { state: AtomicU8::new(RECORDING), wake }; + let mut pending = "unrecognized words".to_owned(); + let mut event = Event(SPEVENT::default()); + event.0._bitfield = SPEI_FALSE_RECOGNITION.0; + assert!(!handle_event(id, &event, &mut pending, &control, false).unwrap()); + assert_eq!(*events.lock().unwrap(), vec![NativeSpeechEvent::Transcript { text: String::new(), is_final: true }]); + assert!(pending.is_empty()); + control.state.store(CANCELLED, Ordering::Release); + pending = "discard this queued result".to_owned(); + finish_partial(id, &mut pending, &control); + assert_eq!(events.lock().unwrap().len(), 1); + crate::sessions().lock().unwrap().remove(&id); + } +} diff --git a/crates/speech/swift/NativeSpeech.swift b/crates/speech/swift/NativeSpeech.swift new file mode 100644 index 0000000..377d210 --- /dev/null +++ b/crates/speech/swift/NativeSpeech.swift @@ -0,0 +1,568 @@ +// macOS and iOS dictation, using SFSpeechRecognizer plus AVAudioEngine. +// +// This side asks for both permissions, owns the microphone, and restarts the +// recognizer between utterances. Rust only sees the events sent to the callback +// below. Note that session state lives on the main queue; only the audio tap runs +// elsewhere, and it hands its buffers over through a synchronized capture sink. + +import Foundation +import AVFoundation +import Speech + +@_silgen_name("robius_speech_event") +private func nativeEvent(_ id: UInt64, _ kind: Int32, _ text: UnsafePointer?, _ level: Float) + +// Session/recognizer state lives on the main queue. Audio taps use a synchronized +// capture sink and marshal meter updates and errors back to that queue. +private var sessions: [UInt64: NativeSpeech] = [:] + +// Error kinds, matching `codes` in lib.rs. Not private: the regression test +// asserts against these rather than repeating the numbers. +let errOther: Int32 = 5 +let errPermission: Int32 = 6 +let errUnavailable: Int32 = 7 +let errLanguage: Int32 = 8 +let errAudio: Int32 = 9 + +#if os(iOS) +// The HFP spelling was introduced by the iOS 26 SDK. This older spelling has +// the identical value and also compiles with older supported Xcode releases. +private let speechAudioOptions: AVAudioSession.CategoryOptions = [.defaultToSpeaker, .allowBluetooth, .mixWithOthers] +#endif + +// NotificationCenter waits for an observer even when it targets OperationQueue.main. +// Audio-engine notifications originate on an internal queue that engine teardown +// also waits for. Return to that queue before touching the engine on the main queue. +func observeAudioNotification(_ name: Notification.Name, object: Any?, handler: @escaping () -> Void) -> NSObjectProtocol { + NotificationCenter.default.addObserver(forName: name, object: object, queue: nil) { _ in + DispatchQueue.main.async { handler() } + } +} + +struct SpeechRetryPolicy { + private(set) var consecutiveNoSpeech = 0 + + mutating func madeProgress() { consecutiveNoSpeech = 0 } + + mutating func retryDelay(after error: NSError) -> TimeInterval? { + // Apple's documented no-speech error. Code 203 is a generic failure, + // not a documented timeout; authorization/network failures stay visible. + // https://developer.apple.com/documentation/speech/sfspeechrecognitiontask/error + guard error.domain == "kAFAssistantErrorDomain", error.code == 1110, consecutiveNoSpeech < 3 else { return nil } + consecutiveNoSpeech += 1 + return 0.25 * Double(1 << (consecutiveNoSpeech - 1)) + } +} + +// The engine keeps recording while a recognizer finalizes an utterance. Only +// that short interval needs copies; otherwise buffers go straight to the request. +// No engine operations or UI callbacks run while this lock is held. +final class SpeechAudioCapture { + private let lock = NSLock() + private var appendToRequest: ((AVAudioPCMBuffer) -> Void)? + private var requestGeneration: UInt64? + private var pending: [AVAudioPCMBuffer] = [] + private var pendingDuration: TimeInterval = 0 + private var active = false + private var overflowed = false + + func prepareRequest(_ generation: UInt64) { + lock.lock() + active = true + requestGeneration = generation + appendToRequest = nil + lock.unlock() + } + + func beginRequest(generation: UInt64? = nil, _ append: @escaping (AVAudioPCMBuffer) -> Void) { + lock.lock() + defer { lock.unlock() } + // A task may finish immediately, before recognitionTask() even returns. + // Do not bind its request after that callback has already detached it. + if let generation = generation, requestGeneration != generation { return } + active = true + appendToRequest = append + for buffer in pending { append(buffer) } + pending.removeAll(keepingCapacity: true) + pendingDuration = 0 + } + + func endRequest(ifCurrent generation: UInt64? = nil) { + lock.lock() + defer { lock.unlock() } + if let generation = generation, requestGeneration != generation { return } + requestGeneration = nil + appendToRequest = nil + } + + // False reports one overflow/allocation error. Further buffers are discarded + // until the main queue handles that terminal error and stops capture. + func append(_ buffer: AVAudioPCMBuffer) -> Bool { + lock.lock() + defer { lock.unlock() } + guard active, !overflowed else { return true } + if let append = appendToRequest { append(buffer); return true } + let duration = Double(buffer.frameLength) / buffer.format.sampleRate + guard duration.isFinite, pendingDuration + duration <= 4, + let copy = AVAudioPCMBuffer(pcmFormat: buffer.format, frameCapacity: buffer.frameLength) + else { + overflowed = true + pending.removeAll() + pendingDuration = 0 + return false + } + copy.frameLength = buffer.frameLength + let source = UnsafeMutableAudioBufferListPointer(UnsafeMutablePointer(mutating: buffer.audioBufferList)) + let destination = UnsafeMutableAudioBufferListPointer(copy.mutableAudioBufferList) + guard source.count == destination.count, + source.indices.allSatisfy({ source[$0].mDataByteSize <= destination[$0].mDataByteSize }) + else { + overflowed = true + pending.removeAll() + pendingDuration = 0 + return false + } + for index in source.indices { + if let from = source[index].mData, let to = destination[index].mData { + memcpy(to, from, Int(source[index].mDataByteSize)) + } + } + pending.append(copy) + pendingDuration += duration + return true + } + + var hasPendingAudio: Bool { + lock.lock() + defer { lock.unlock() } + return !pending.isEmpty + } + + func stop(discardPending: Bool = true) { + lock.lock() + active = false + requestGeneration = nil + appendToRequest = nil + if discardPending { + pending.removeAll() + pendingDuration = 0 + } + lock.unlock() + } +} + +final class NativeSpeech { + let id: UInt64 + let locale: Locale? + let preferOnDevice: Bool + // Do not initialize audio hardware before the privacy checks and permissions. + var engine: AVAudioEngine? + let capture = SpeechAudioCapture() + var recognizer: SFSpeechRecognizer? + var request: SFSpeechAudioBufferRecognitionRequest? + var task: SFSpeechRecognitionTask? + var stopping = false + var finished = false + var tapInstalled = false + var started = false + var lastPartial = "" + var generation: UInt64 = 0 + var retryPolicy = SpeechRetryPolicy() + var endingUtterance = false + var rollover: DispatchWorkItem? + var pendingRestart: DispatchWorkItem? + var observers: [NSObjectProtocol] = [] + #if os(iOS) + var previousAudioConfiguration: (AVAudioSession.Category, AVAudioSession.Mode, AVAudioSession.CategoryOptions)? + var activatedAudioSession = false + #endif + + init(id: UInt64, locale: String, preferOnDevice: Bool) { + self.id = id + self.locale = locale.isEmpty ? nil : Locale(identifier: locale) + self.preferOnDevice = preferOnDevice + } + + func send(_ kind: Int32, _ text: String = "", _ level: Float = 0) { + guard !finished else { return } + text.withCString { nativeEvent(id, kind, $0, level) } + } + + func authorize() { + let usageKeys = ["NSMicrophoneUsageDescription", "NSSpeechRecognitionUsageDescription"] + #if os(macOS) + // An embedded __info_plist satisfies Bundle.main, but TCC can still + // attribute an unbundled process to its launching terminal or editor. + // Those apps lack our speech description, and TCC terminates the process. + let bundle = Bundle.main.bundleURL + guard bundle.pathExtension == "app", + let data = try? Data(contentsOf: bundle.appendingPathComponent("Contents/Info.plist")), + let plist = (try? PropertyListSerialization.propertyList(from: data, format: nil)) as? [String: Any], + usageKeys.allSatisfy({ (plist[$0] as? String)?.isEmpty == false }) + else { + fail("Launch the application from its .app bundle to use speech input.", errPermission) + return + } + #endif + // Missing privacy keys cause a process termination in Apple's APIs. + // Report an actionable error before invoking either permission prompt. + for key in usageKeys { + guard let reason = Bundle.main.object(forInfoDictionaryKey: key) as? String, !reason.isEmpty else { + fail("This app is missing its \(key) privacy description.", errPermission) + return + } + } + #if ROBIUS_SPEECH_TESTS + // Native lifecycle regressions must never request real microphone or + // speech permission, including when testing an invalid launch context. + preconditionFailure("Unexpected permission request in native speech tests") + #else + SFSpeechRecognizer.requestAuthorization { [weak self] status in + DispatchQueue.main.async { + guard let self = self, !self.finished, !self.stopping else { return } + guard status == .authorized else { + self.fail("Speech recognition permission was denied. Allow speech recognition for this app in system privacy settings.", errPermission) + return + } + AVCaptureDevice.requestAccess(for: .audio) { [weak self] granted in + DispatchQueue.main.async { + guard let self = self, !self.finished, !self.stopping else { return } + guard granted else { + self.fail("Microphone permission was denied. Allow microphone access for this app in system privacy settings.", errPermission) + return + } + self.start() + } + } + } + } + #endif + } + + func start() { + // Use the native default-language selection rather than a region-based + // Locale.current such as en_NL. Explicit locales retain Apple's built-in + // fallback to the keyboard's dictation language if unsupported. + let selectedRecognizer = locale.map { SFSpeechRecognizer(locale: $0) } ?? SFSpeechRecognizer() + guard let recognizer = selectedRecognizer, recognizer.isAvailable else { + fail("Speech recognition is unavailable for the current language. Check the system speech settings and network connection.", errLanguage) + return + } + self.recognizer = recognizer + recognizer.defaultTaskHint = .dictation + // A busy UI must not delay detaching a completed task's audio sink. + // Only the synchronized sink is touched here; session state stays on main. + let callbacks = OperationQueue() + callbacks.name = "org.robius.speech.recognition" + callbacks.maxConcurrentOperationCount = 1 + callbacks.qualityOfService = .userInitiated + recognizer.queue = callbacks + #if os(iOS) + do { + let audio = AVAudioSession.sharedInstance() + previousAudioConfiguration = (audio.category, audio.mode, audio.categoryOptions) + try audio.setCategory(.playAndRecord, mode: .measurement, options: speechAudioOptions) + try audio.setActive(true) + activatedAudioSession = true + } catch { + fail("Could not activate the microphone: \(error.localizedDescription)", errAudio) + return + } + observers.append(observeAudioNotification(AVAudioSession.interruptionNotification, object: nil) { [weak self] in + guard let self = self, !self.stopping, !self.finished else { return } + self.fail("Speech recording was interrupted by another audio session.", errAudio) + }) + #endif + let engine = AVAudioEngine() + self.engine = engine + observers.append(observeAudioNotification(.AVAudioEngineConfigurationChange, object: engine) { [weak self] in + guard let self = self, !self.stopping, !self.finished else { return } + self.fail("The microphone changed or disconnected. Start dictation again to use the new microphone.", errAudio) + }) + beginUtterance() + } + + func beginUtterance(finishing: Bool = false) { + guard !finished, (!stopping || finishing), let recognizer = recognizer, let engine = engine else { return } + guard recognizer.isAvailable else { + // isAvailable also drops on a brief network loss for server-backed + // locales, so this is retryable rather than a missing recognizer. + fail("The system speech recognition service became unavailable.") + return + } + generation &+= 1 + let utterance = generation + endingUtterance = false + let request = SFSpeechAudioBufferRecognitionRequest() + request.shouldReportPartialResults = true + request.taskHint = .dictation + if preferOnDevice && recognizer.supportsOnDeviceRecognition { + request.requiresOnDeviceRecognition = true + } + if #available(macOS 13.0, iOS 16.0, *) { request.addsPunctuation = true } + self.request = request + lastPartial = "" + let capture = self.capture + capture.prepareRequest(utterance) + task = recognizer.recognitionTask(with: request) { [weak self] result, error in + if result?.isFinal == true || error != nil { + // Start buffering immediately, before the main queue handles the + // transcript. A late previous callback cannot detach its successor. + capture.endRequest(ifCurrent: utterance) + } + DispatchQueue.main.async { + guard let self = self, !self.finished, self.generation == utterance else { return } + if let result = result { + self.lastPartial = result.bestTranscription.formattedString + if !self.lastPartial.isEmpty { self.retryPolicy.madeProgress() } + self.send(result.isFinal ? 2 : 1, self.lastPartial) + if result.isFinal { + self.lastPartial = "" + if self.stopping { self.finishStoppedUtterance() } + else { self.restartUtterance(after: 0.15) } + return + } + } + if let error = error { + if self.stopping { + // Preserve the most recent partial if a system recognizer + // ends without delivering a final result after endAudio. + self.finishStoppedUtterance() + } else if self.endingUtterance { + self.commitPartial() + self.restartUtterance(after: 0.15) + } else if let delay = self.retryPolicy.retryDelay(after: error as NSError) { + self.commitPartial() + self.restartUtterance(after: delay) + } else { + self.fail("Speech recognition failed: \(error.localizedDescription)") + } + } + } + } + capture.beginRequest(generation: utterance) { request.append($0) } + if finishing { endRequest(); scheduleStopTimeout(); return } + // The existing tap now targets the new request, including PCM saved + // during finalization/backoff. Keeping the engine running avoids gaps. + if tapInstalled { scheduleRollover(utterance); return } + let input = engine.inputNode + let format = input.outputFormat(forBus: 0) + guard format.sampleRate > 0 && format.channelCount > 0 else { + fail("No working microphone is available.", errAudio) + return + } + var meterFrames: AVAudioFrameCount = 0 + input.installTap(onBus: 0, bufferSize: 1024, format: format) { [weak self] buffer, _ in + if !capture.append(buffer) { + DispatchQueue.main.async { [weak self] in + self?.fail("Speech recognition took too long to resume. Your draft has been kept; please start dictation again.") + } + } + meterFrames += buffer.frameLength + guard meterFrames >= AVAudioFrameCount(format.sampleRate / 30) else { return } + meterFrames = 0 + guard let samples = buffer.floatChannelData, buffer.frameLength > 0 else { return } + var energy: Float = 0 + let channels = Int(buffer.format.channelCount) + let count = Int(buffer.frameLength) + for channel in 0.., _ preferOnDevice: Bool) { + let locale = String(cString: locale) + DispatchQueue.main.async { + let session = NativeSpeech(id: id, locale: locale, preferOnDevice: preferOnDevice) + sessions[id] = session + session.authorize() + } +} + +@_cdecl("robius_speech_stop") +public func stopNativeSpeech(_ id: UInt64, _ cancel: Bool) { + DispatchQueue.main.async { sessions[id]?.stop(cancel: cancel) } +} diff --git a/crates/speech/swift/tests/SpeechTests.swift b/crates/speech/swift/tests/SpeechTests.swift new file mode 100644 index 0000000..8afbeca --- /dev/null +++ b/crates/speech/swift/tests/SpeechTests.swift @@ -0,0 +1,269 @@ +// Host-side regressions for the Swift bridge, driving NativeSpeech.swift's own +// types directly. Note that nothing here opens a microphone or touches a +// permission API, so these are safe to run anywhere macOS builds. + +import Foundation +import AVFoundation + +private var nativeEvents: [(id: UInt64, kind: Int32, text: String)] = [] + +// The production bridge links this callback from Rust. The test build traps +// before any permission API, even if the unbundled-launch guard regresses. +@_cdecl("robius_speech_event") +func testNativeSpeechEvent(_ id: UInt64, _ kind: Int32, _ text: UnsafePointer?, _ level: Float) { + precondition(Thread.isMainThread) + nativeEvents.append((id, kind, text.map(String.init(cString:)) ?? "")) +} + +@main +enum SpeechTests { + static let cases: [(String, () -> Void)] = [ + ("audio notifications defer to the main queue", audioNotificationsDeferToMainQueue), + ("an unbundled launch is rejected", unbundledLaunchIsRejected), + ("silence retries are bounded", silenceRetriesAreBounded), + ("capture buffers audio across a request handoff", captureBuffersAcrossHandoff), + ("capture stops allocating when a recognizer stalls", captureStopsAllocatingWhenStalled), + ("capture keeps every buffer under concurrent handoff", captureKeepsEveryBufferConcurrently), + ("capture binds audio to the current request generation", captureBindsAudioToItsGeneration), + ("utterance rollover commits the draft", rolloverCommitsTheDraft), + ("cancelling during rollover emits nothing", cancellingDuringRolloverIsSilent), + ("a graceful stop replays buffered speech", gracefulStopReplaysBufferedSpeech), + ] + + static func main() { + for (name, run) in cases { + // Flush before running: a precondition failure aborts the process, + // and piped stdout would otherwise lose the name of the failing case. + print(" \(name)") + fflush(stdout) + run() + } + print("Passed \(cases.count) native speech regressions; no microphone used.") + } + + // MARK: - Notification delivery + + static func audioNotificationsDeferToMainQueue() { + let name = Notification.Name("RobiusSpeech.NotificationDeliveryTest") + let source = NSObject() + let posted = DispatchSemaphore(value: 0) + var received = 0 + let observer = observeAudioNotification(name, object: source) { + precondition(Thread.isMainThread, "Audio state must remain on the main thread") + received += 1 + } + defer { NotificationCenter.default.removeObserver(observer) } + + DispatchQueue.global().async { + NotificationCenter.default.post(name: name, object: source) + posted.signal() + } + // Model the main thread waiting for an engine operation. The engine's + // notification-publishing worker must be able to return independently. + precondition(posted.wait(timeout: .now() + 2) == .success, "Audio notification blocked its posting thread waiting for the main queue") + precondition(received == 0, "Observer ran before the main queue could process it") + drainMainQueue(until: { received == 1 }) + + // Even a notification posted on main must defer engine teardown until + // the posting operation has returned, rather than reentering the engine. + NotificationCenter.default.post(name: name, object: source) + precondition(received == 1, "Audio notification ran its handler reentrantly") + drainMainQueue(until: { received == 2 }) + } + + // MARK: - Privacy and permissions + + static func unbundledLaunchIsRejected() { + // Reproduce cargo-run's misleading embedded privacy descriptions. TCC + // can ignore them and attribute access to the parent editor instead. + precondition(Bundle.main.bundleURL.pathExtension != "app") + for key in ["NSMicrophoneUsageDescription", "NSSpeechRecognitionUsageDescription"] { + precondition(Bundle.main.object(forInfoDictionaryKey: key) as? String != nil, "The test executable must embed its privacy descriptions") + } + nativeEvents.removeAll() + "".withCString { startNativeSpeech(42, $0, true) } + drainMainQueue(until: { !nativeEvents.isEmpty }) + precondition(nativeEvents.count == 1) + precondition(nativeEvents[0].id == 42 && nativeEvents[0].kind == errPermission) + precondition(nativeEvents[0].text == "Launch the application from its .app bundle to use speech input.") + } + + // MARK: - Retry policy + + static func silenceRetriesAreBounded() { + var retries = SpeechRetryPolicy() + let noSpeech = NSError(domain: "kAFAssistantErrorDomain", code: 1110) + precondition(retries.retryDelay(after: noSpeech) == 0.25) + precondition(retries.retryDelay(after: noSpeech) == 0.5) + precondition(retries.retryDelay(after: noSpeech) == 1.0) + precondition(retries.retryDelay(after: noSpeech) == nil, "Silence retries must be bounded") + retries.madeProgress() + precondition(retries.retryDelay(after: noSpeech) == 0.25, "Recognized speech resets the retry budget") + for code in [203, 1100, 1101, 1107, 1700] { + precondition(retries.retryDelay(after: NSError(domain: "kAFAssistantErrorDomain", code: code)) == nil, "A generic failure is not a documented timeout") + } + precondition(retries.retryDelay(after: NSError(domain: "UnrelatedDomain", code: 1110)) == nil) + } + + // MARK: - Audio capture + + static func captureBuffersAcrossHandoff() { + let capture = SpeechAudioCapture() + var samples: [Float] = [] + capture.beginRequest { samples.append($0.floatChannelData![0][0]) } + precondition(capture.append(audioBuffer(sample: 1))) + capture.endRequest() + let reusedBuffer = audioBuffer(sample: 2) + precondition(capture.append(reusedBuffer)) + reusedBuffer.floatChannelData![0][0] = 99 + precondition(capture.append(audioBuffer(sample: 3))) + precondition(samples == [1], "Finalizing a task must buffer new audio") + capture.beginRequest { samples.append($0.floatChannelData![0][0]) } + precondition(capture.append(audioBuffer(sample: 4))) + precondition(samples == [1, 2, 3, 4], "Copied rollover audio must precede live audio without reuse corruption") + capture.endRequest() + precondition(capture.append(audioBuffer(sample: 5))) + capture.stop(discardPending: false) + precondition(capture.hasPendingAudio, "Graceful stop must keep buffered speech for final recognition") + precondition(capture.append(audioBuffer(sample: 99))) + capture.beginRequest { samples.append($0.floatChannelData![0][0]) } + precondition(samples == [1, 2, 3, 4, 5], "Final recognition must replay saved speech but no post-stop audio") + capture.stop() + } + + static func captureStopsAllocatingWhenStalled() { + let bounded = SpeechAudioCapture() + bounded.beginRequest { _ in } + bounded.endRequest() + for _ in 0..<4 { precondition(bounded.append(audioBuffer(sample: 0, frames: 8_000))) } + precondition(!bounded.append(audioBuffer(sample: 0)), "A stalled recognizer must not allocate unbounded PCM storage") + precondition(bounded.append(audioBuffer(sample: 0)), "Buffer overflow must report one terminal failure") + precondition(!bounded.hasPendingAudio) + bounded.stop() + } + + static func captureKeepsEveryBufferConcurrently() { + let concurrent = SpeechAudioCapture() + var ordered: [Int] = [] + let receive: (AVAudioPCMBuffer) -> Void = { ordered.append(Int($0.floatChannelData![0][0])) } + concurrent.beginRequest(receive) + let producerFinished = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + for value in 0..<2_000 { precondition(concurrent.append(audioBuffer(sample: Float(value)))) } + producerFinished.signal() + } + for _ in 0..<200 { + concurrent.endRequest() + concurrent.beginRequest(receive) + } + precondition(producerFinished.wait(timeout: .now() + 2) == .success) + concurrent.beginRequest(receive) + precondition(ordered == Array(0..<2_000), "Concurrent request handoff must preserve every PCM buffer in order") + concurrent.stop() + } + + static func captureBindsAudioToItsGeneration() { + let generations = SpeechAudioCapture() + var oldRequest: [Float] = [] + var newRequest: [Float] = [] + generations.prepareRequest(10) + generations.beginRequest(generation: 10) { oldRequest.append($0.floatChannelData![0][0]) } + precondition(generations.append(audioBuffer(sample: 10))) + let detached = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + generations.endRequest(ifCurrent: 10) + precondition(generations.append(audioBuffer(sample: 11))) + detached.signal() + } + // Simulate the UI being blocked while a terminal task callback arrives. + precondition(detached.wait(timeout: .now() + 2) == .success) + precondition(oldRequest == [10], "PCM after a terminal callback must buffer without waiting for UI dispatch") + generations.prepareRequest(20) + generations.beginRequest(generation: 20) { newRequest.append($0.floatChannelData![0][0]) } + generations.endRequest(ifCurrent: 10) + precondition(generations.append(audioBuffer(sample: 12))) + precondition(newRequest == [11, 12], "A delayed old callback must not detach the active successor") + generations.endRequest(ifCurrent: 20) + generations.prepareRequest(30) + generations.endRequest(ifCurrent: 30) + generations.beginRequest(generation: 30) { _ in preconditionFailure("An already-completed request must not be rebound") } + precondition(generations.append(audioBuffer(sample: 13))) + generations.prepareRequest(40) + generations.beginRequest(generation: 40) { newRequest.append($0.floatChannelData![0][0]) } + precondition(newRequest == [11, 12, 13], "Immediate task completion must preserve subsequent PCM for its successor") + generations.stop() + } + + // MARK: - Session lifecycle + + static func rolloverCommitsTheDraft() { + nativeEvents.removeAll() + let rollover = NativeSpeech(id: 43, locale: "", preferOnDevice: true) + precondition(rollover.locale == nil, "An unspecified locale must use native default-language selection") + precondition(rollover.engine == nil, "Creating a session must not initialize microphone hardware") + rollover.started = true + rollover.generation = 7 + rollover.lastPartial = "Keep this draft" + rollover.endUtteranceBeforeLimit(6, finalizationTimeout: 0) + precondition(!rollover.endingUtterance, "An old utterance timer must not affect its successor") + rollover.endUtteranceBeforeLimit(7, finalizationTimeout: 0) + drainMainQueue(until: { !nativeEvents.isEmpty }) + precondition(nativeEvents.count == 1 && nativeEvents[0].kind == 2 && nativeEvents[0].text == "Keep this draft") + precondition(rollover.generation == 8 && rollover.pendingRestart != nil) + rollover.stop(cancel: false) + precondition(rollover.finished && rollover.pendingRestart == nil, "Stopping during backoff must cancel the pending restart") + precondition(nativeEvents.count == 2 && nativeEvents[1].kind == 4) + } + + static func cancellingDuringRolloverIsSilent() { + let before = nativeEvents.count + let cancelled = NativeSpeech(id: 44, locale: "en-US", preferOnDevice: true) + cancelled.generation = 1 + cancelled.lastPartial = "Cancelled partial" + cancelled.endUtteranceBeforeLimit(1, finalizationTimeout: 0) + cancelled.stop(cancel: true) + var drained = false + DispatchQueue.main.async { drained = true } + drainMainQueue(until: { drained }) + precondition(nativeEvents.count == before, "Cancelled rollover must not emit a transcript or restart") + } + + static func gracefulStopReplaysBufferedSpeech() { + nativeEvents.removeAll() + let stopping = NativeSpeech(id: 45, locale: "", preferOnDevice: true) + stopping.stopping = true + stopping.lastPartial = "Previous utterance" + stopping.capture.beginRequest { _ in } + stopping.capture.endRequest() + precondition(stopping.capture.append(audioBuffer(sample: 6))) + stopping.endCapture(discardPending: false) + stopping.finishStoppedUtterance() + precondition(nativeEvents.count == 1 && nativeEvents[0].text == "Previous utterance") + precondition(!stopping.finished && stopping.pendingRestart != nil, "Queued speech must finish before the stopped event") + var replayed: [Float] = [] + stopping.capture.beginRequest { replayed.append($0.floatChannelData![0][0]) } + precondition(replayed == [6]) + stopping.lastPartial = "Buffered final words" + stopping.finishStoppedUtterance() + precondition(nativeEvents.map(\.kind) == [2, 2, 4]) + precondition(nativeEvents[1].text == "Buffered final words" && stopping.finished) + } + + // MARK: - Helpers + + private static func audioBuffer(sample: Float, frames: AVAudioFrameCount = 1) -> AVAudioPCMBuffer { + let format = AVAudioFormat(standardFormatWithSampleRate: 8_000, channels: 1)! + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames)! + buffer.frameLength = frames + for frame in 0.. Bool) { + let deadline = Date(timeIntervalSinceNow: 2) + while !complete() && Date() < deadline { + _ = RunLoop.main.run(mode: .default, before: Date(timeIntervalSinceNow: 0.01)) + } + precondition(complete(), "Deferred audio notification was never delivered") + } +} diff --git a/crates/speech/swift/tests/run.sh b/crates/speech/swift/tests/run.sh new file mode 100755 index 0000000..f3d7e47 --- /dev/null +++ b/crates/speech/swift/tests/run.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Builds NativeSpeech.swift together with its regressions and runs them. +# You need macOS and the Xcode command line tools. No microphone is used. +set -euo pipefail + +tests_dir=$(cd "$(dirname "$0")" && pwd) +tmp=$(mktemp -d "${TMPDIR:-/tmp}/robius-speech-tests.XXXXXX") +trap 'rm -rf "$tmp"' EXIT + +# The privacy keys must be embedded for the unbundled-launch regression, which +# checks that the bridge still refuses to touch a permission API without a bundle. +cat > "$tmp/Info.plist" <<'PLIST' + + + + + CFBundleIdentifier + org.robius.speech.tests + NSMicrophoneUsageDescription + Privacy validation regression only; these tests do not access the microphone. + NSSpeechRecognitionUsageDescription + Privacy validation regression only; these tests do not request speech recognition. + + +PLIST + +xcrun --sdk macosx swiftc -swift-version 5 -O -parse-as-library -D ROBIUS_SPEECH_TESTS \ + "$tests_dir/../NativeSpeech.swift" "$tests_dir/SpeechTests.swift" \ + -Xlinker -sectcreate -Xlinker __TEXT -Xlinker __info_plist -Xlinker "$tmp/Info.plist" \ + -o "$tmp/speech-tests" +"$tmp/speech-tests" diff --git a/crates/speech/tests/__pycache__/android_retry_test.cpython-313.pyc b/crates/speech/tests/__pycache__/android_retry_test.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..10f16d57f3fdf11415235cf9392568f0c1ba023b GIT binary patch literal 20480 zcmc&+No*WfdafqBxwI%*3oTQ&WR+VQyJ?Y4$+Emq%M3*}En3{Bio|6cXiyLYD*eBS^2zq+PRD9XRmjr_EWi?9C?7e7-p z#a6V0)^<5zC-B>Tx$V_QaPQqhuHs+iC^v`&0?Y2Elv zYrF8>qwU7`9_)iE5WYME-yTB?>*eMzsHj;odp*RWTrX4P@gx@d5Sb zTG6i?PA;7;7=PTT+J>X*>Z@ayRoyJBqiV@osaI=;eO5J%Rl~-;X&NQhQC-WM(y~>n zZkRN1rCPGB`m$vjx%3=Rdv$t3b(X6uz_Dc6WkV{ET&Rx@2b=6p@xis87d^ZK9E_0n~H z$sp|M@!52}u~-AirJC+Is_d-ZQboJtVnX@2PCVl5i*~3~O)yz=?W(z?7V@Jrg|Xtu z{Mf{ddS1=wjdIn>oK@de-%hU|mY9K6E@&JYrQe$Z!IKdLbE4N-i4aFrj8SjBz5YZL z-UI{7=+(cIP?YT;hDqn6)LUs4A3;2C%L=M=CV_s;+1dvq-26?~)@A4G#@c{n$Esdy z7#Ig3yA7Lkvg&{t4+>?`vTBBIs^1b!)pWN70kJ$CWn+D1XqGDrv;rz)a}BC(!_HiN zC8A8>(Jd;)HHNU7%@?L7G;LyLy7)?AWO^3`_u&dX@7uiHpOouIjF#a^yzpb=U?B0EP$9-t~DH`wcblG~0hR zX85?RFRehs1^?8FxOYGv0f;=$l3oZvLqn=&xauN&0c_V9R!^!+who6dpbnmbw=%2n zLk-id)|j(sIB-3>>D(}Eav27BjFq~Z3#yo~WMX*0j)~F1UQ*er&Nz9)Wtej|pt=>k z<`@y(^@ofBSKC@M$`3wdbXhl-FbTv^sV+5a#%yP&@kB&v*(hDtj9O*Z^GeeAOnoJ2 zZ$7yZ%mWfsmkc*>8$=)=`N;T)00~B57N4>B}NQZ&Vh-a+hg`mn=i^gD)IFkECH6Uaeyvt(qYEg@2%8RUj0W`itq6guJ> zS?S3vCwH-7mZ2@gaZ>!aK?TEUtQbUH+*4t>+dZb#2e$27nnU1fZk8FMRW@X-TX<6? z6RkjGt`T6G6hojbM} zO}$|?YUPk?;}+C{qi_cyoRFZEiAA0o39W>l@USfIN6`&!)SD}8h-)EV%CwsL^NvH$ zK8wtcTyek?v&$PgPwkku=2WQ94h6lhfXj`8!SFPKxqqV^L|y8k7g&I>G%|UCCbLni zan#6cq@L$|uZTVZy82xNnqh<(zkfvpt(;|Us|5j`4Y0_@GbS+pmdO3y9`G1WNKo|S z3UaGuYV~aq1DSu5@p=eect{c~43E3Ht_R8kB10qIPK0~m0(@7(WfRH?DCf>K5O{N! zbZ2==ub<<@`abId!uFiLM4Shit^F@}*#b|!BdE2yL&%)_H0h53T3$wCiG)Q7Oj+Js zBy?nKboC{ICsI*{X%6l$OYPeLn?vT|(1b{X77aw}Kts@726LKV7oOJ$|HpN^Tn^yl zv(D4xpqg_Wkzf;!1Adnn9TfpZGG~CXO{f#;UBy>s;Y7(e;yXx($*1AwFC5K^wQ6J~ zcRir3IJ1l$$xO~R*bq5={Dhtl4$df6BHYg+;S2JcGsdZ|!>x;S;CL`!mKX+P!A<6F z+rwlAf#I=cBd3w!PzJn^@rX4PB#4!qFeo`t)yrk0JOG^}E@4edq7sD5J^>^=&POKn z%tRmnE0L&#V{Z&hLcxRUNmNEPl^c!c!g%UINMnwQ&?%93peq^~ox$ZE8ykutaaagF zk&iss9%+I_p=pEDgb8KpiV6iP1S7SYbpzR#Mk&IiH9&Dr# zDYtG8H4tR06*VhSe~L$>ekmMj9O8MvHfR)64Yg$qi3>bNT0l||V_3lO$_o@QM*Kw^ z*Zf~Tkmd)5NsZ^OTG;49YKIqy|UWEciSio*n01V!O)veIx{A4~1 zmyu;!H$wU<*p?OYsxjF|1A#07)UgTC%7uVVQHL?3rc=F5)`wxg6L8;*WjV*P-K@{CQPd-KRQD_!l~=k%E;0$x zBZ@;q205d1B(5MRncCQmxK%id6FhD3LvIUdOj#Hiyz4|IJg1~bJ?|?~l!rpIW3zBU zu9t-e{F4~QbpZuv#{QsT^0Bo1I@!;NR|i#6ns6E?z;JRK<}@G?5Kl;R#6reLN<2wm zoKgVg>PIM%u=ry(k^g{3P)U&$97S!Bfexi7Ash@8t{547Hl#kfO<5$LgH`Sk9-fgd z#$mfvLsa-#GOM(VB*$);(&hX-CSLF{UhMJ{m>4L66(VgYpn$qIZrOZ~ET&3EdblDB z?PAbHTGpD-$(f26Cs-$et~jhx4O6SsENscyEf5-&ieUpg1N(F3?Bbehp#HJQTEnpN zW_g=g^0sZ+tfpqrRd^V$xH&Nd$s!w;l7?K(0H+_gHYdP0PO{LbSZ zBPwTxev3il|E7xMYy_)KAn`c`bi z@IcLu6y_#ICNZlvKRKt-b1;RN?RA+Nxf*=DqTdWm6h?@K*RTiDz!sY~ezuSw&lid_ z@F|iEk`hEp zc}@#7;fdoSVzjTy5t5vUKK=%XZ9}1y5Eps~H5LNc=8`9I8F8}#(a1JcY8OkDXj@G- zfFJTQB6>arDKCeD(Ah#74+2!^M^GWyT-i^Yh=*m>eNgD#N)dL1N=A(mkJe|x23S_& z*#pp*S_`i^P&QVFY-_RFaE2V7k_1oF6b#p1n}aEF@A>N|eD8^YIX*DQ2cAt=SL#S` zz4(Y?Q%(9_~NVO5uYd*Bd+bbMpf->I()(bqWto`iWTUvXo zadA8;y2s&n^cP0FbSCVV>5a{{0JAJFuA=Zuj_^7*Ec|*aQe0H}S*BIPR5i%lvsXSf zTa?fsEI~5E8GL8y6cJ_dRMW>!Rwje3ENYW03ZXS&QW-+*EP=Px2Foh5G}XfgJDLvS za}WzEPCn)J?8i=njSQQ-$B2W|7?kDmbWV&VG*yoJdBT=oh8;^7GqI_AgU*G?k_ICu zzzfl5k?;R$0-RwGe+$Z_`rry!2J5H3NZ;2qdX zSK4y6AKD4M%pf&mQAr4lhI$CdB^f7$9<@wO!Nv2cY%4XiNFk<5vE5H47)KTx&f0`? z(Sppe&>R z1O+2NDK~`cB$>tIWrd1~l{FM6i(;oXxSh!qR<5~91$&-`nxpSO0 zh%d9qIfWBob)37znJFApq%u8H>P%!taztz%kSg4a*k1Fzk55@-g~jbO0~yqeOiFg;gN_$L73d@xv;@as;k9tM0^TOd z7AXQ$33%}#Ea(>DKD@RQkGOlLs~rJ;Lq=|^P$x;D;%D`e32@W6xr~hkY+nc)T5Ob2 zyTJ_#AUIcy3zl!ej|Mr&XO*IJk*}^`OSn?4VNWK@nyLQ9E`*DEt?DjkGiQdOMyQtg zY7Z2k!H8*eR2f=qP?bQ{D`=3SY@N7;Ea?W%tIq=Ttb2`2XlAn=TDjZI{Szb4c zIy(&zGR?@)`>GXRX$G+6awQO+j*uD&K~s88XUnsh=Y|85gX1N}DW)v!tT>FAc4!@S zwdz#q&>zcVNDcxn>=VwH4-3loK4!^ri}Ewa);QQq1w0=Su0=v_IDLg;hqJtmES zjPMlVrxRdV;U29;2dA0cu!js=2&<%e&W&{5qh2~Nq}w=5s50-yb5-C>(*)TQOMOsN zBB)UyQ5!G=9CsCl~n!l=ChdoAfWxl5pZ<&`jF zTF|Fc6tK@}X+2VH?OU$qNRFiX>H5 zJ&QsPt;8OT@Gyb_2gp!ki$HLe&WbpE_klTN7#zKMA@SJWQ#sh`9c$1A#6_M3Ov7YR zx#fN_#zmbDp#8rklvb9VeyEIdd!EIHHa`ZS>V}-fs;gzj5}6sPWsB)8MK{(N%3H_V z3nAqV+wJ`$8%}59?07cwd|Xt+@xziqro%8{Z?qr?1tT5RrFRysf2MH)afh8KHOhF; z#?<&*KeM(~G7K!If*G&01Q1Hw8sBgs0e?*|)Z~JdfbbbhoT@}_gA=fng2@AyYliEk zmT@4w^AX5MX_XMyC%Dv$=n?8OSzR3Y7*J1<0X^JGhKo2jt2~?p;@HVoc{m54>HQ|W z9)t~DUl8gFGA7m5s}978BfriX5~vmNf`XJNC4GFR7W?uwm5>)17CVZ_0243SnM8bJ zvBT%!Z$GG2dfS5z;u_`PM6ty1j@gKl64VO@(m6><9A$!yL zyGCVUuiTi)=IDh+X3zc`jX@pZz=fvqCA7D27p-E~o7*>E@w~tlkzQ{?wq!GDJL;_W zeh=vZVm+DBfXK63YiYhOk&f?M+bv&Vab94TFOUud8UC%++!?WrX^UP7!J<5_vh#dk z<{e$eeY3;u_Ot>Ma1qMj81!^7Fwc0iamGTUZNLHJVVu3#L1zUT+Y(7g_Cav$edZchOBYn zO$;otMf~}|P|_v5!hjUlS%W^x)jckG-Q)%q>Z+_S$(quDnkD4>L$yT`D%)Kc9uC#- zA}>bJWMX~Dmt~lgFQ}QZ`6$3I+o4f2H`zZSqxdq!TP_dJS(6F#o(SWQ_Q2zMTn|KX^y}0NZCb#G9V9(-(bdXLHFvHB%408$QM*Aqx zX8tli+r9vwmY|yST0h>aHm$&0_9~qN&iTIP?X>=jgrXd)nx$F;Z+M-%W>#~{7t(kS zKYw+0rZC4H4sJ(BCMVTbai+G2-M=E={VO_bzG5%~Dzb2vWkeAab5(uTFjoiEYy5D9 zbpz*|aOZ7yU85Qb-|f2Q@b~4Z;DatP`(0V`fBg&w7n*>|-jGAFo2Kd?yfBV^uu*tm zC(EYJJm5~JU|gZ|M^0ezw$EpZ0=Kp#_?SF-$4DZivB3mxJ{Dc~O_q93h|{rALi z^ke_7LC*_4(O!y@QMC4<1pPb4%WZ{6TA#~*tq9Go0qQ>4cj16{)hn+G2j!LjizucxO7;{3l`oHrjiI1vGy`U(&NHV)H=hK)cx!_B8z#k*8 zGRi#uONB6+PqOcJZMT1SMrb?WLhuAPpp`iZ!~e|$^lM(x_7EhAoA!vWd-Ltu-r%#L z_Jvat%EDfGHQ!!1EwBAwZJ&fFvHL#R!G6Y{cI^PcH9O}!<2bbdDe#Yk0g4{P9>E8# zZ3h1?V|HDAN!Td0J&Lx2Y{u>#IU}c59@vLGru9i0wL|njHUSPQ+TkDv2LV0q`FvNj zR{%z*Y%sq-W>bk1CZ!|5MY=?pH^yj1(`;Oy!#$W!=W>Y3m1v8jd|wt1cQh zjnWVWl;X0rV$9M1+E5^9rl_s-JHS-ddrbtURb!AxdpSPW+x72 zBK>fhUB*ouMJQU=``gy{1;aNx?6ZD^ucl_bm8Q3Nrs>uXvdGpFRM+>lHgzvnO@?=A zy=}1E>|{xR;jDK&%Vq+0DYmd$?~-MhW(o&88#mW`L_>KEvnr)WT>_qBN zwJo#Rj+J?{9d){9R~X{WKs(qPAF_Amm9NCa`9q?w`|rjGuuZBry}b~wxQ zdUP{&W4T&dZgwIwvudjbq{3v;>?#(q_h}T1HXMM`?4oT)YH04C!*Hs@+WAuyJ2x8qrm<>T3)@JMznB`-)y~rtGDfwb*P& zLe`&dc6)jR=Hnd%gLG-7-b~t-+MRt^Pv z%2xTOd1+`KTjtKtbsTv#aR$m6r)Td^F8}cI-JT;GJx4w%e>QiAu7;_3=7%#MjQ^(R z*zY@)BLiP|DUV)G+#b8vv;QYEKbrX(iT?M_+vJrO#ix-Sy@EgP(V84t(#!+!xn3_7`q<-V+G4eVqO< z{qdd;_k7}hI=9*P+-DP?-vEI>J^k+5CogY2Hn4eMaHD7NQ}^ek&-=fqZVb=dDZIWh z{QBk#3mZcV_j(?Ax8ui`Kj{6)`Q*jFcxx-w);IWD;FfCZ9{!IGrTc(KV|wH8bDMpq zKbsHI_+`(TpF5ww__uEnQTfD|le0usA@QHbl=N8QZujVKx<~)Zp+xzYZR5k2Q|gfQm#o}(_JQ|CwsOjg|qwUi;_w&_>~YHEf_)yh-5Q}OT5M9Ep2FN@RoJ7Jh`PDQj=S^ zc#D}-Qn@Xq=VWrrO0u~wrP%Bpn0+F}TCVVxKD4~XTP}4dsnc7^t;890Vk~;6b89T! o)tP*!{SOBpPZSbc!%DK}_WEY~p?^wt{pt8tyOMm|J_n}%Hz5Y<9RL6T literal 0 HcmV?d00001 diff --git a/crates/speech/tests/android_retry_test.py b/crates/speech/tests/android_retry_test.py new file mode 100644 index 0000000..71d965a --- /dev/null +++ b/crates/speech/tests/android_retry_test.py @@ -0,0 +1,734 @@ +#!/usr/bin/env python3 +"""Runs the real Android session logic against deterministic host-side fakes. + +You need a JDK and a C compiler. Note that this never talks to a device or opens +a microphone: the fakes stand in for the Android APIs, and the JNI shim just +records the events that the production NativeSpeech.java emits. +""" + +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +SOURCES = { + "android/Manifest.java": """package android; +public class Manifest { public static class permission { public static final String RECORD_AUDIO = "audio"; } } +""", + "android/content/Context.java": "package android.content; public class Context {}", + "android/content/Intent.java": """package android.content; +public class Intent { + public Intent(String action) {} + public Intent putExtra(String key, String value) { return this; } + public Intent putExtra(String key, boolean value) { return this; } + public Intent putExtra(String key, int value) { return this; } +} +""", + "android/content/pm/PackageManager.java": """package android.content.pm; +public class PackageManager { public static final int PERMISSION_GRANTED = 0; } +""", + "android/app/Activity.java": """package android.app; +public class Activity extends android.content.Context { + private final Application application; + private final FragmentManager fragments = new FragmentManager(this); + // Set by tests: 0 grants, -1 denies until the user answers the prompt. + public static int permission = 0; + public boolean resumed = true; + private boolean destroyed, changingConfigurations; + public Activity() { this(new Application()); } + public Activity(Application application) { this.application = application; } + public boolean isFinishing() { return false; } + public boolean isDestroyed() { return destroyed; } + public boolean isChangingConfigurations() { return changingConfigurations; } + public int checkSelfPermission(String name) { return permission; } + public Application getApplication() { return application; } + public FragmentManager getFragmentManager() { return fragments; } + public ClassLoader getClassLoader() { return Activity.class.getClassLoader(); } + public void runOnUiThread(Runnable runnable) { runnable.run(); } + public void pause() { + if (!resumed) return; + resumed = false; + fragments.pauseAll(); + application.paused(this); + } + public void resume() { + if (destroyed || resumed) return; + resumed = true; + application.resumed(this); + fragments.resumeAll(); + } + public void destroy(boolean configurationChange) { + changingConfigurations = configurationChange; + pause(); + destroyed = true; + fragments.destroy(); + application.destroyed(this); + } +} +""", + "android/app/Fragment.java": """package android.app; +public class Fragment { + private Activity activity; + private FragmentManager manager; + private final FragmentManager children = new FragmentManager(this); + public static Fragment pending; + public static int requests; + void attach(Activity activity, FragmentManager manager) { + this.activity = activity; this.manager = manager; + } + void detach() { this.activity = null; this.manager = null; } + public Activity getActivity() { return activity; } + public FragmentManager getFragmentManager() { return manager; } + public FragmentManager getChildFragmentManager() { return children; } + public boolean isAdded() { return manager != null && manager.contains(this); } + public void setRetainInstance(boolean retain) { + if (retain) throw new AssertionError("permission helpers must not retain discarded Activities"); + } + public void onCreate(android.os.Bundle state) {} + public void onResume() {} + public void onPause() {} + public void onDestroy() {} + public void onDetach() {} + public void onSaveInstanceState(android.os.Bundle state) {} + public void onRequestPermissionsResult(int code, String[] names, int[] results) {} + // Records the in-flight request so a test can answer it, as the OS would. + public void requestPermissions(String[] names, int code) { + if (pending != null) throw new AssertionError("a second system prompt was requested while one was pending"); + requests++; + pending = this; pendingCode = code; + activity.pause(); + } + public static int pendingCode; + /** Answer the prompt the way the user would. */ + public static void answer(boolean granted) { + Fragment fragment = pending; pending = null; + if (fragment == null) throw new AssertionError("no permission request is pending"); + Activity.permission = granted ? 0 : -1; + Activity owner = fragment.getActivity(); + // Activity.findFragmentByWho drops results for a detached request fragment. + if (owner != null) { + fragment.onRequestPermissionsResult(pendingCode, new String[] {"audio"}, + new int[] { granted ? 0 : -1 }); + owner.resume(); + } + } +} +""", + "android/app/FragmentManager.java": """package android.app; +public class FragmentManager { + private final Activity activity; + private final Fragment parent; + private boolean destroyed, executing; + private final java.util.LinkedHashMap byTag = new java.util.LinkedHashMap<>(); + private final java.util.ArrayList callbacks = new java.util.ArrayList<>(); + private final java.util.ArrayList transactions = new java.util.ArrayList<>(); + public FragmentManager(Activity activity) { this.activity = activity; this.parent = null; } + public FragmentManager(Fragment parent) { this.activity = null; this.parent = parent; } + private Activity activity() { return parent == null ? activity : parent.getActivity(); } + public boolean isDestroyed() { return destroyed; } + public boolean isStateSaved() { return false; } + public Fragment findFragmentByTag(String tag) { return byTag.get(tag); } + public FragmentTransaction beginTransaction() { return new FragmentTransaction(this); } + public boolean contains(Fragment fragment) { return byTag.containsValue(fragment); } + public void registerFragmentLifecycleCallbacks(FragmentLifecycleCallbacks callback, boolean recursive) { + callbacks.add(callback); + } + public void unregisterFragmentLifecycleCallbacks(FragmentLifecycleCallbacks callback) { callbacks.remove(callback); } + public int callbackCount() { return callbacks.size(); } + public static abstract class FragmentLifecycleCallbacks { + public void onFragmentSaveInstanceState(FragmentManager manager, Fragment fragment, android.os.Bundle state) {} + public void onFragmentDetached(FragmentManager manager, Fragment fragment) {} + public void onFragmentDestroyed(FragmentManager manager, Fragment fragment) {} + } + void commit(Runnable action, boolean immediate) { + if (destroyed) return; // commitAllowingStateLoss drops work on a destroyed host. + if (immediate) { execute(action); return; } + transactions.add(action); + new android.os.Handler(android.os.Looper.getMainLooper()).post(() -> { + if (transactions.remove(action) && !destroyed) execute(action); + }); + } + private void execute(Runnable action) { + if (executing) throw new IllegalStateException("FragmentManager is already executing transactions"); + executing = true; + try { action.run(); } finally { executing = false; } + } + public boolean executePendingTransactions() { + boolean any = !transactions.isEmpty(); + for (Runnable action : new java.util.ArrayList<>(transactions)) { + transactions.remove(action); + if (!destroyed) execute(action); + } + return any; + } + void add(String tag, Fragment fragment) { + byTag.put(tag, fragment); + fragment.attach(activity(), this); + fragment.onCreate(null); + if (activity().resumed) fragment.onResume(); + } + void remove(Fragment fragment) { + if (!contains(fragment)) return; + byTag.values().remove(fragment); + fragment.getChildFragmentManager().destroy(); + fragment.onDestroy(); + for (FragmentLifecycleCallbacks callback : new java.util.ArrayList<>(callbacks)) { + callback.onFragmentDestroyed(this, fragment); + } + fragment.onDetach(); + fragment.detach(); + for (FragmentLifecycleCallbacks callback : new java.util.ArrayList<>(callbacks)) { + callback.onFragmentDetached(this, fragment); + } + } + public void pauseAll() { + for (Fragment fragment : new java.util.ArrayList<>(byTag.values())) { + fragment.getChildFragmentManager().pauseAll(); fragment.onPause(); + } + } + public void resumeAll() { + for (Fragment fragment : new java.util.ArrayList<>(byTag.values())) { + fragment.onResume(); fragment.getChildFragmentManager().resumeAll(); + } + } + public void destroy() { + destroyed = true; + transactions.clear(); + for (Fragment fragment : new java.util.ArrayList<>(byTag.values())) remove(fragment); + } + public static final class SavedFragment { + public final String className, tag; + public final android.os.Bundle state; + SavedFragment(Fragment fragment, String tag, android.os.Bundle state) { + this.className = fragment.getClass().getName(); this.tag = tag; this.state = state; + } + } + public java.util.List saveAllState() { + executePendingTransactions(); + java.util.List result = new java.util.ArrayList<>(); + for (java.util.Map.Entry entry : byTag.entrySet()) { + Fragment fragment = entry.getValue(); + android.os.Bundle state = new android.os.Bundle(); + // The framework snapshots the class name before requesting this Bundle. + SavedFragment saved = new SavedFragment(fragment, entry.getKey(), state); + fragment.onSaveInstanceState(state); + java.util.List childState = fragment.getChildFragmentManager().saveAllState(); + if (!childState.isEmpty()) state.put("children", childState); + // Android's callback runs after performSaveInstanceState saved children. + for (FragmentLifecycleCallbacks callback : new java.util.ArrayList<>(callbacks)) { + callback.onFragmentSaveInstanceState(this, fragment, state); + } + result.add(saved); + } + return result; + } + @SuppressWarnings("unchecked") + public void restoreAllState(java.util.List saved, ClassLoader loader) throws Exception { + for (SavedFragment entry : saved) { + Fragment fragment = (Fragment) loader.loadClass(entry.className).getConstructor().newInstance(); + byTag.put(entry.tag, fragment); + fragment.attach(activity(), this); + fragment.onCreate(entry.state); + Object children = entry.state.get("children"); + if (children != null) fragment.getChildFragmentManager().restoreAllState((java.util.List) children, loader); + } + } + public int count() { return byTag.size(); } +} +""", + "android/app/FragmentTransaction.java": """package android.app; +public class FragmentTransaction { + private final FragmentManager manager; + private String tag; private Fragment added, removed; + public FragmentTransaction(FragmentManager manager) { this.manager = manager; } + public FragmentTransaction add(Fragment fragment, String tag) { + this.added = fragment; this.tag = tag; return this; + } + public FragmentTransaction remove(Fragment fragment) { this.removed = fragment; return this; } + public void commitAllowingStateLoss() { manager.commit(this::apply, false); } + public void commitNowAllowingStateLoss() { manager.commit(this::apply, true); } + private void apply() { + if (removed != null) manager.remove(removed); + if (added != null) manager.add(tag, added); + } +} +""", + "android/app/Application.java": """package android.app; +public class Application { + private final java.util.ArrayList callbacks = new java.util.ArrayList<>(); + public void registerActivityLifecycleCallbacks(ActivityLifecycleCallbacks callback) { callbacks.add(callback); } + public void unregisterActivityLifecycleCallbacks(ActivityLifecycleCallbacks callback) { callbacks.remove(callback); } + public void paused(Activity activity) { + for (ActivityLifecycleCallbacks callback : new java.util.ArrayList<>(callbacks)) callback.onActivityPaused(activity); + } + public void resumed(Activity activity) { + for (ActivityLifecycleCallbacks callback : new java.util.ArrayList<>(callbacks)) callback.onActivityResumed(activity); + } + public void destroyed(Activity activity) { + for (ActivityLifecycleCallbacks callback : new java.util.ArrayList<>(callbacks)) callback.onActivityDestroyed(activity); + } + public interface ActivityLifecycleCallbacks { + void onActivityCreated(Activity activity, android.os.Bundle state); + void onActivityStarted(Activity activity); + void onActivityResumed(Activity activity); + void onActivityPaused(Activity activity); + void onActivityStopped(Activity activity); + void onActivitySaveInstanceState(Activity activity, android.os.Bundle state); + void onActivityDestroyed(Activity activity); + } +} +""", + "android/os/Build.java": """package android.os; +public class Build { public static class VERSION { public static int SDK_INT = 33; } } +""", + "android/os/Bundle.java": """package android.os; +public class Bundle { + private java.util.ArrayList values; + private final java.util.HashMap data = new java.util.HashMap<>(); + public void clear() { data.clear(); values = null; } + public boolean isEmpty() { return data.isEmpty() && values == null; } + public void put(String key, Object value) { data.put(key, value); } + public Object get(String key) { return data.get(key); } + public void putString(String key, String value) { data.put(key, value); } + public String getString(String key) { return (String) data.get(key); } + public void putStringArrayList(String key, java.util.ArrayList value) { values = value; } + public java.util.ArrayList getStringArrayList(String key) { return values; } +} +""", + "android/os/Looper.java": """package android.os; +public class Looper { public static Looper getMainLooper() { return new Looper(); } } +""", + "android/os/Handler.java": """package android.os; +public class Handler { + private static final class Task { + Runnable runnable; long time; + Task(Runnable runnable, long time) { this.runnable = runnable; this.time = time; } + } + private static final java.util.ArrayList tasks = new java.util.ArrayList<>(); + private static long now; + public Handler(Looper looper) {} + public boolean post(Runnable runnable) { return postDelayed(runnable, 0); } + public boolean postDelayed(Runnable runnable, long delay) { + tasks.add(new Task(runnable, now + delay)); + tasks.sort(java.util.Comparator.comparingLong(task -> task.time)); + return true; + } + public void removeCallbacks(Runnable runnable) { tasks.removeIf(task -> task.runnable == runnable); } + public static int count() { return tasks.size(); } + public static long nextDelay() { return tasks.get(0).time - now; } + public static void runNext() { + Task task = tasks.remove(0); + now = task.time; + task.runnable.run(); + } + public static void reset() { tasks.clear(); now = 0; } + public static void drain() { + int limit = 100; + while (!tasks.isEmpty() && tasks.get(0).time == now) { + if (--limit == 0) throw new AssertionError("unbounded immediate work"); + runNext(); + } + } +} +""", + "harness/HostStateFragment.java": """package harness; +public class HostStateFragment extends android.app.Fragment { + public String value = "host draft must survive"; + @Override public void onCreate(android.os.Bundle state) { + if (state != null) value = state.getString("draft"); + } + @Override public void onSaveInstanceState(android.os.Bundle state) { state.putString("draft", value); } +} +""", + "harness/Launcher.java": """package harness; +public class Launcher { + public static void main(String[] args) throws Exception { + java.net.URL url = new java.io.File(args[0]).toURI().toURL(); + try (java.net.URLClassLoader child = new java.net.URLClassLoader(new java.net.URL[] {url}, Launcher.class.getClassLoader())) { + Class test = child.loadClass("dev.robius.speech.NativeSpeechRetryTest"); + test.getMethod("main", String[].class).invoke(null, (Object) new String[] {args[1]}); + } + } +} +""", + "android/speech/RecognitionListener.java": """package android.speech; +public interface RecognitionListener { + void onReadyForSpeech(android.os.Bundle params); + void onBeginningOfSpeech(); + void onRmsChanged(float rms); + void onBufferReceived(byte[] buffer); + void onEndOfSpeech(); + void onError(int error); + void onResults(android.os.Bundle results); + void onPartialResults(android.os.Bundle results); + void onEvent(int type, android.os.Bundle params); +} +""", + "android/speech/RecognizerIntent.java": """package android.speech; +public class RecognizerIntent { + public static final String ACTION_RECOGNIZE_SPEECH = "speech", EXTRA_LANGUAGE_MODEL = "model", + LANGUAGE_MODEL_FREE_FORM = "free", EXTRA_PARTIAL_RESULTS = "partial", EXTRA_MAX_RESULTS = "max", + EXTRA_LANGUAGE = "language", EXTRA_PREFER_OFFLINE = "offline"; +} +""", + "android/speech/SpeechRecognizer.java": """package android.speech; +public class SpeechRecognizer { + public static final String RESULTS_RECOGNITION = "results"; + public static final int ERROR_NETWORK_TIMEOUT = 1, ERROR_NETWORK = 2, ERROR_AUDIO = 3, + ERROR_CLIENT = 5, ERROR_SPEECH_TIMEOUT = 6, ERROR_NO_MATCH = 7, + ERROR_RECOGNIZER_BUSY = 8, ERROR_INSUFFICIENT_PERMISSIONS = 9; + public static SpeechRecognizer last; + public static int starts; + public RecognitionListener listener; + public boolean destroyed; + public static boolean isRecognitionAvailable(android.content.Context context) { return true; } + public static boolean isOnDeviceRecognitionAvailable(android.content.Context context) { return false; } + public static SpeechRecognizer createSpeechRecognizer(android.content.Context context) { + return last = new SpeechRecognizer(); + } + public static SpeechRecognizer createOnDeviceSpeechRecognizer(android.content.Context context) { + return createSpeechRecognizer(context); + } + public void setRecognitionListener(RecognitionListener listener) { this.listener = listener; } + public void startListening(android.content.Intent intent) { starts++; } + public void stopListening() {} + public void cancel() {} + public void destroy() { destroyed = true; } +} +""", + "dev/robius/speech/NativeSpeechRetryTest.java": """package dev.robius.speech; +import android.app.Activity; +import android.os.Bundle; +import android.os.Handler; +import android.speech.RecognitionListener; +import android.speech.SpeechRecognizer; + +public class NativeSpeechRetryTest { + private static final java.util.ArrayList events = new java.util.ArrayList<>(); + private static final java.util.ArrayList eventIds = new java.util.ArrayList<>(); + public static void record(long id, int kind, String text, float level) { + events.add(kind + ":" + (text == null ? "" : text)); + eventIds.add(id); + } + private static void check(boolean condition, String message) { + if (!condition) throw new AssertionError(message + " events=" + events); + } + private static Bundle words(String text) { + Bundle bundle = new Bundle(); + bundle.putStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION, + new java.util.ArrayList<>(java.util.Arrays.asList(text))); + return bundle; + } + private static void start(long id) { + Handler.reset(); events.clear(); eventIds.clear(); SpeechRecognizer.starts = 0; + NativeSpeech.start(new Activity(), id, "", false); + Handler.runNext(); + SpeechRecognizer.last.listener.onReadyForSpeech(new Bundle()); + } + private static void busyIsBounded() { + start(1); + for (int retry = 0; retry < 3; retry++) { + RecognitionListener old = SpeechRecognizer.last.listener; + old.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY); + check(Handler.count() == 1 && Handler.nextDelay() == (500L << retry), "bounded exponential backoff"); + int count = events.size(); + old.onPartialResults(words("stale")); + old.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY); + check(events.size() == count && Handler.count() == 1, "late callbacks cannot schedule retries or insert words"); + Handler.runNext(); + SpeechRecognizer.last.listener.onReadyForSpeech(new Bundle()); + } + SpeechRecognizer.last.listener.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY); + check(Handler.count() == 0 && SpeechRecognizer.last.destroyed, "exhausted retry budget releases recognizer"); + check(events.stream().filter(event -> event.startsWith("5:")).count() == 1, "one terminal error after retries"); + check(SpeechRecognizer.starts == 4, "initial attempt and only three retries"); + } + private static void clientRecreatesAndSuccessResetsBudget() { + start(2); + SpeechRecognizer old = SpeechRecognizer.last; + old.listener.onPartialResults(words("keep this")); + old.listener.onError(SpeechRecognizer.ERROR_CLIENT); + check(old.destroyed && events.contains("2:keep this"), "client reconnect commits visible text and destroys old client"); + Handler.runNext(); + check(SpeechRecognizer.last != old, "client error creates a fresh recognizer"); + int count = events.size(); + old.listener.onResults(words("obsolete")); + check(events.size() == count, "old client cannot append late final results"); + SpeechRecognizer.last.listener.onResults(words("next utterance")); + check(Handler.nextDelay() == 150, "successful utterance resumes normal cadence"); + Handler.runNext(); + SpeechRecognizer.last.listener.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY); + check(Handler.nextDelay() == 500, "completed utterance resets transient failure budget"); + NativeSpeech.stop(2, true); + Handler.runNext(); + check(Handler.count() == 0, "cancel removes delayed reconnect"); + } + private static void stopDuringBackoffNeverRestarts() { + start(3); + SpeechRecognizer.last.listener.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY); + NativeSpeech.stop(3, false); + Handler.runNext(); + check(Handler.count() == 0 && SpeechRecognizer.starts == 1, "stop during backoff does not restart capture"); + check(events.stream().filter(event -> event.equals("4:")).count() == 1, "graceful stop emits one terminal event"); + } + private static void permissionIsRequestedThenSessionProceeds() { + Handler.reset(); events.clear(); SpeechRecognizer.starts = 0; + android.app.Activity.permission = -1; // not granted yet + android.app.Activity activity = new android.app.Activity(); + NativeSpeech.start(activity, 10, "", false); + Handler.runNext(); + check(android.app.Fragment.pending != null, "a missing permission must raise the system prompt"); + check(SpeechRecognizer.starts == 0, "recognition must not begin before the prompt is answered"); + android.app.Fragment.answer(true); + check(activity.getFragmentManager().callbackCount() == 1, + "the save-state guard remains until queued removal actually detaches its host"); + Handler.drain(); + check(activity.getFragmentManager().count() == 0, "the fragment removes itself once answered"); + SpeechRecognizer.last.listener.onReadyForSpeech(new Bundle()); + check(SpeechRecognizer.starts == 1 && events.contains("0:"), "granting resumes the session"); + NativeSpeech.stop(10, true); + Handler.drain(); + android.app.Activity.permission = 0; + } + private static void permissionDenialIsReportedOnce() { + Handler.reset(); events.clear(); SpeechRecognizer.starts = 0; + android.app.Activity.permission = -1; + android.app.Activity activity = new android.app.Activity(); + NativeSpeech.start(activity, 11, "", false); + Handler.runNext(); + android.app.Fragment.answer(false); + Handler.drain(); + check(SpeechRecognizer.starts == 0, "a denied prompt must not open the microphone"); + check(events.size() == 1 && events.get(0).startsWith("6:"), "denial reports PermissionDenied once, events=" + events); + check(activity.getFragmentManager().count() == 0, "the fragment removes itself after a denial"); + android.app.Activity.permission = 0; + } + private static void permissionsAreNotRetried() { + start(4); + SpeechRecognizer.last.listener.onError(SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS); + check(Handler.count() == 0 && SpeechRecognizer.last.destroyed, "permission errors stay terminal"); + } + private static Activity missingPermission() { + Handler.reset(); events.clear(); eventIds.clear(); SpeechRecognizer.starts = 0; + Activity.permission = -1; + check(android.app.Fragment.pending == null, "the preceding case must release its system prompt"); + android.app.Fragment.requests = 0; + return new Activity(); + } + private static void cancelDiscardsLateGrant() { + Activity activity = missingPermission(); + NativeSpeech.start(activity, 20, "", false); + Handler.runNext(); + NativeSpeech.stop(20, true); + Handler.drain(); + check(events.isEmpty() && SpeechRecognizer.starts == 0, "canceling a pending session is silent"); + android.app.Fragment.answer(true); + Handler.drain(); + check(events.isEmpty() && SpeechRecognizer.starts == 0, "a late grant cannot restart a canceled session"); + check(activity.getFragmentManager().count() == 0 && activity.getFragmentManager().callbackCount() == 0, + "answering an abandoned request releases its parent and save listener"); + } + private static void nextSessionAdoptsPendingPrompt() { + Activity activity = missingPermission(); + NativeSpeech.start(activity, 21, "", false); + Handler.runNext(); + android.app.Fragment pending = android.app.Fragment.pending; + // Both operations are queued before the old request receives its answer. + NativeSpeech.stop(21, true); + NativeSpeech.start(activity, 22, "", false); + Handler.drain(); + check(events.isEmpty(), "a successor must not receive a false permission denial"); + check(android.app.Fragment.pending == pending && android.app.Fragment.requests == 1, + "the successor adopts the existing system prompt without requesting a second one"); + android.app.Fragment.answer(true); + Handler.drain(); + check(SpeechRecognizer.starts == 1, "grant starts only the successor"); + SpeechRecognizer.last.listener.onReadyForSpeech(new Bundle()); + check(events.size() == 1 && events.get(0).equals("0:") && eventIds.get(0) == 22L, + "the canceled session cannot receive the successor's result"); + check(activity.getFragmentManager().count() == 0 && activity.getFragmentManager().callbackCount() == 0, + "adopted prompt cleanup releases all helper resources"); + NativeSpeech.stop(22, true); + Handler.drain(); + } + private static void cancelBeforeResumeDoesNotAsk() { + Activity activity = missingPermission(); + activity.resumed = false; + NativeSpeech.start(activity, 23, "", false); + Handler.runNext(); + check(android.app.Fragment.pending == null, "a paused owner does not launch the prompt yet"); + NativeSpeech.stop(23, true); + Handler.runNext(); + // Resume before the queued fragment removal executes. + activity.resume(); + Handler.drain(); + check(android.app.Fragment.requests == 0 && SpeechRecognizer.starts == 0 && events.isEmpty(), + "cancel must suppress a deferred prompt even before its transaction is removed"); + check(activity.getFragmentManager().count() == 0 && activity.getFragmentManager().callbackCount() == 0, + "cancel before prompt launch releases the parent and listener"); + } + private static void restartWhileOldParentRemovalIsQueued() { + Activity activity = missingPermission(); + activity.resumed = false; + NativeSpeech.start(activity, 24, "", false); + Handler.runNext(); + NativeSpeech.stop(24, true); + NativeSpeech.start(activity, 25, "", false); + Handler.runNext(); // cancel queues a removal behind the already queued start + Handler.runNext(); + activity.resume(); + Handler.drain(); + check(events.isEmpty() && android.app.Fragment.requests == 1, + "an old removal cannot cancel the replacement request or cause a false denial"); + android.app.Fragment.answer(true); + Handler.drain(); + check(SpeechRecognizer.starts == 1, "the replacement still starts after old removal drains"); + SpeechRecognizer.last.listener.onReadyForSpeech(new Bundle()); + check(eventIds.size() == 1 && eventIds.get(0) == 25L, "only the replacement receives Started"); + NativeSpeech.stop(25, true); + Handler.drain(); + } + private static void configurationTeardownStopsPendingSession() { + Activity activity = missingPermission(); + NativeSpeech.start(activity, 26, "", false); + Handler.runNext(); + java.util.List saved = activity.getFragmentManager().saveAllState(); + activity.destroy(true); + Handler.drain(); + check(events.size() == 1 && events.get(0).equals("4:") && eventIds.get(0) == 26L, + "configuration teardown ends the pending session once with Stopped, not permission denial"); + check(activity.getFragmentManager().callbackCount() == 0, "discarded Activity releases its save listener"); + android.app.Fragment.answer(true); + Handler.drain(); + check(events.size() == 1 && SpeechRecognizer.starts == 0, "a grant cannot restart the discarded Activity"); + Activity replacement = new Activity(activity.getApplication()); + try { replacement.getFragmentManager().restoreAllState(saved, replacement.getClassLoader()); } + catch (Exception error) { throw new AssertionError("configuration state must be restorable", error); } + NativeSpeech.start(replacement, 27, "", false); + Handler.drain(); + check(SpeechRecognizer.starts == 1, "the replacement Activity can start a new session after grant"); + SpeechRecognizer.last.listener.onReadyForSpeech(new Bundle()); + check(eventIds.get(eventIds.size() - 1) == 27L, "the new Activity owns the restarted session"); + NativeSpeech.stop(27, true); + Handler.drain(); + } + private static void processRestoreContainsOnlyHostVisibleClasses() { + Activity activity = missingPermission(); + check(NativeSpeech.class.getClassLoader() != activity.getClassLoader(), + "production speech must be loaded in a separate child classloader"); + try { + activity.getClassLoader().loadClass("dev.robius.speech.SpeechPermissionFragment"); + throw new AssertionError("host loader must be unable to load the embedded helper"); + } catch (ClassNotFoundException expected) {} + activity.getFragmentManager().beginTransaction().add(new harness.HostStateFragment(), "unrelated") + .commitNowAllowingStateLoss(); + NativeSpeech.start(activity, 28, "", false); + Handler.runNext(); + java.util.List saved = activity.getFragmentManager().saveAllState(); + check(saved.size() == 2, "saving preserves the unrelated app fragment and framework permission parent"); + // Repeated saves must remain safe while the same OS request is pending. + saved = activity.getFragmentManager().saveAllState(); + Activity restored = new Activity(); + try { restored.getFragmentManager().restoreAllState(saved, restored.getClassLoader()); } + catch (Exception error) { throw new AssertionError("process restoration cannot load child DEX classes", error); } + harness.HostStateFragment unrelated = (harness.HostStateFragment) restored.getFragmentManager().findFragmentByTag("unrelated"); + check(unrelated != null && "host draft must survive".equals(unrelated.value), + "clearing permission state cannot alter an unrelated app fragment's state"); + android.app.Fragment host = restored.getFragmentManager().findFragmentByTag("dev.robius.speech.PermissionHost"); + check(host != null && host.getClass() == android.app.Fragment.class && host.getChildFragmentManager().count() == 0, + "the restored helper host contains no custom child fragment"); + // A fresh process would lose these callbacks; explicitly release the old process's fake state. + NativeSpeech.stop(28, true); + Handler.drain(); + android.app.Fragment.answer(false); + Handler.drain(); + check(events.isEmpty(), "an abandoned old-process request has no recipient"); + NativeSpeech.start(restored, 29, "", false); + Handler.drain(); + check(android.app.Fragment.pending != null && restored.getFragmentManager().count() == 2, + "a new request replaces the empty restored host without disturbing app fragments"); + android.app.Fragment.answer(true); + Handler.drain(); + check(SpeechRecognizer.starts == 1, "speech works after restoring a process with a pending permission request"); + NativeSpeech.stop(29, true); + Handler.drain(); + } + private interface Case { void run(); } + public static void main(String[] args) { + System.load(args[0]); + Object[][] cases = { + {"a busy recognizer retries with bounded backoff", (Case) NativeSpeechRetryTest::busyIsBounded}, + {"a client error recreates the recognizer", (Case) NativeSpeechRetryTest::clientRecreatesAndSuccessResetsBudget}, + {"stopping during backoff never restarts", (Case) NativeSpeechRetryTest::stopDuringBackoffNeverRestarts}, + {"permission errors are not retried", (Case) NativeSpeechRetryTest::permissionsAreNotRetried}, + {"a missing permission is requested, then the session proceeds", (Case) NativeSpeechRetryTest::permissionIsRequestedThenSessionProceeds}, + {"a denied permission is reported once", (Case) NativeSpeechRetryTest::permissionDenialIsReportedOnce}, + {"canceling a pending prompt discards a late grant", (Case) NativeSpeechRetryTest::cancelDiscardsLateGrant}, + {"a new session adopts an abandoned system prompt", (Case) NativeSpeechRetryTest::nextSessionAdoptsPendingPrompt}, + {"cancel before resume never launches the deferred prompt", (Case) NativeSpeechRetryTest::cancelBeforeResumeDoesNotAsk}, + {"queued old removal cannot remove a new permission request", (Case) NativeSpeechRetryTest::restartWhileOldParentRemovalIsQueued}, + {"configuration teardown stops the old pending session", (Case) NativeSpeechRetryTest::configurationTeardownStopsPendingSession}, + {"saved permission state restores through the host classloader", (Case) NativeSpeechRetryTest::processRestoreContainsOnlyHostVisibleClasses}, + }; + for (Object[] entry : cases) { + System.out.println(" " + entry[0]); + ((Case) entry[1]).run(); + } + System.out.println("Passed " + cases.length + " Android speech regressions; no microphone used."); + } +} +""", +} + +JNI = """#include +JNIEXPORT void JNICALL Java_dev_robius_speech_NativeSpeech_event( + JNIEnv *env, jclass owner, jlong id, jint kind, jstring text, jfloat level) { + (void)owner; + jclass test = (*env)->FindClass(env, "dev/robius/speech/NativeSpeechRetryTest"); + jmethodID record = (*env)->GetStaticMethodID(env, test, "record", "(JILjava/lang/String;F)V"); + (*env)->CallStaticVoidMethod(env, test, record, id, kind, text, level); +} +""" + + +def main(): + java_home = os.environ.get("JAVA_HOME") + if not java_home and sys.platform == "darwin": + java_home = subprocess.check_output(["/usr/libexec/java_home"], text=True).strip() + if not java_home: + java_home = str(Path(shutil.which("javac") or "javac").resolve().parent.parent) + java_home = Path(java_home) + crate = Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory(prefix="native-speech-retry-") as temp: + root = Path(temp) + for name, source in SOURCES.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source) + for name in ("NativeSpeech.java", "SpeechPermissionFragment.java"): + shutil.copyfile(crate / "java/dev/robius/speech" / name, root / "dev/robius/speech" / name) + # Keep platform/host classes on the parent loader and embedded speech + # classes on a child loader, matching the Android InMemoryDexClassLoader. + host_classes = root / "host-classes" + speech_classes = root / "speech-classes" + subprocess.run([str(java_home / "bin/javac"), "-d", str(host_classes), + *map(str, (root / "android").rglob("*.java")), + *map(str, (root / "harness").rglob("*.java"))], check=True) + subprocess.run([str(java_home / "bin/javac"), "-classpath", str(host_classes), "-d", str(speech_classes), + *map(str, (root / "dev").rglob("*.java"))], check=True) + c_file = root / "events.c" + c_file.write_text(JNI) + platform = "darwin" if sys.platform == "darwin" else "linux" + library = root / ("events.dylib" if platform == "darwin" else "events.so") + subprocess.run([os.environ.get("CC", "cc"), "-shared", "-fPIC", + "-I" + str(java_home / "include"), "-I" + str(java_home / "include" / platform), + str(c_file), "-o", str(library)], check=True) + subprocess.run([str(java_home / "bin/java"), "-cp", str(host_classes), + "harness.Launcher", str(speech_classes), str(library)], check=True) + + +if __name__ == "__main__": + main() From fb33766a5c63f84dc89494218cfdcae25702d444 Mon Sep 17 00:00:00 2001 From: Kevin Boos Date: Wed, 9 Sep 2026 17:31:15 -0700 Subject: [PATCH 2/4] fix readme and remove unnecessary pycache file --- README.md | 2 +- crates/speech/.gitignore | 1 + .../android_retry_test.cpython-313.pyc | Bin 20480 -> 0 bytes 3 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 crates/speech/tests/__pycache__/android_retry_test.cpython-313.pyc diff --git a/README.md b/README.md index d7f61db..9cad7b1 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Symbol legend: ✅ fully supported · ⚠️ partial, or has issues · 🚧 unde | [`robius-location`](crates/location/) | ✅ `CLLocationManager` (CoreLocation) | ✅ `CLLocationManager` (CoreLocation) | ✅ `LocationManager` | ✅ `Geolocator` (`Windows.Devices.Geolocation`, WinRT) | ✅ XDG Location portal, with a `GeoClue` fallback | | [`robius-open`](crates/open/) | ✅ `NSWorkspace.openURL` | ✅ `UIApplication.openURL` | ✅ `Intent` (`ACTION_VIEW`) | ✅ `Launcher.LaunchUriAsync` (WinRT) | ✅ `xdg-open` | | [`robius-share`](crates/share/) | ✅ `NSSharingServicePicker` | ✅ `UIActivityViewController` | ✅ `ACTION_SEND` / `ACTION_SEND_MULTIPLE` via `Intent.createChooser` | ✅ WinRT Share UI (`DataTransferManager`) | ✅ XDG portal "Open With" chooser (`OpenURI` / `OpenFile`), or its `SaveFiles` dialog for multi-item payloads; `xdg-open` fallback | -| [`robius-speech`](crates/speech/) | ⚠️ `SFSpeechRecognizer` + `AVAudioEngine` | ⚠️ `SFSpeechRecognizer` + `AVAudioEngine` | ⚠️ `SpeechRecognizer` | ⚠️ SAPI dictation | ❌ not supported | +| [`robius-speech`](crates/speech/) | ✅ `SFSpeechRecognizer` + `AVAudioEngine` | ✅ `SFSpeechRecognizer` + `AVAudioEngine` | ✅ `SpeechRecognizer` (needs an installed recognition service) | ✅ SAPI dictation (needs an installed speech language) | ❌ no OS-native speech-to-text service exists | | [`robius-web-auth-session`](crates/web_auth_session/) | ❌ not supported | ✅ `ASWebAuthenticationSession` | 🚧 planned (custom chrome tabs) | ❌ not supported | ❌ not supported | diff --git a/crates/speech/.gitignore b/crates/speech/.gitignore index 4fffb2f..a254237 100644 --- a/crates/speech/.gitignore +++ b/crates/speech/.gitignore @@ -1,2 +1,3 @@ /target /Cargo.lock +__pycache__/ diff --git a/crates/speech/tests/__pycache__/android_retry_test.cpython-313.pyc b/crates/speech/tests/__pycache__/android_retry_test.cpython-313.pyc deleted file mode 100644 index 10f16d57f3fdf11415235cf9392568f0c1ba023b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20480 zcmc&+No*WfdafqBxwI%*3oTQ&WR+VQyJ?Y4$+Emq%M3*}En3{Bio|6cXiyLYD*eBS^2zq+PRD9XRmjr_EWi?9C?7e7-p z#a6V0)^<5zC-B>Tx$V_QaPQqhuHs+iC^v`&0?Y2Elv zYrF8>qwU7`9_)iE5WYME-yTB?>*eMzsHj;odp*RWTrX4P@gx@d5Sb zTG6i?PA;7;7=PTT+J>X*>Z@ayRoyJBqiV@osaI=;eO5J%Rl~-;X&NQhQC-WM(y~>n zZkRN1rCPGB`m$vjx%3=Rdv$t3b(X6uz_Dc6WkV{ET&Rx@2b=6p@xis87d^ZK9E_0n~H z$sp|M@!52}u~-AirJC+Is_d-ZQboJtVnX@2PCVl5i*~3~O)yz=?W(z?7V@Jrg|Xtu z{Mf{ddS1=wjdIn>oK@de-%hU|mY9K6E@&JYrQe$Z!IKdLbE4N-i4aFrj8SjBz5YZL z-UI{7=+(cIP?YT;hDqn6)LUs4A3;2C%L=M=CV_s;+1dvq-26?~)@A4G#@c{n$Esdy z7#Ig3yA7Lkvg&{t4+>?`vTBBIs^1b!)pWN70kJ$CWn+D1XqGDrv;rz)a}BC(!_HiN zC8A8>(Jd;)HHNU7%@?L7G;LyLy7)?AWO^3`_u&dX@7uiHpOouIjF#a^yzpb=U?B0EP$9-t~DH`wcblG~0hR zX85?RFRehs1^?8FxOYGv0f;=$l3oZvLqn=&xauN&0c_V9R!^!+who6dpbnmbw=%2n zLk-id)|j(sIB-3>>D(}Eav27BjFq~Z3#yo~WMX*0j)~F1UQ*er&Nz9)Wtej|pt=>k z<`@y(^@ofBSKC@M$`3wdbXhl-FbTv^sV+5a#%yP&@kB&v*(hDtj9O*Z^GeeAOnoJ2 zZ$7yZ%mWfsmkc*>8$=)=`N;T)00~B57N4>B}NQZ&Vh-a+hg`mn=i^gD)IFkECH6Uaeyvt(qYEg@2%8RUj0W`itq6guJ> zS?S3vCwH-7mZ2@gaZ>!aK?TEUtQbUH+*4t>+dZb#2e$27nnU1fZk8FMRW@X-TX<6? z6RkjGt`T6G6hojbM} zO}$|?YUPk?;}+C{qi_cyoRFZEiAA0o39W>l@USfIN6`&!)SD}8h-)EV%CwsL^NvH$ zK8wtcTyek?v&$PgPwkku=2WQ94h6lhfXj`8!SFPKxqqV^L|y8k7g&I>G%|UCCbLni zan#6cq@L$|uZTVZy82xNnqh<(zkfvpt(;|Us|5j`4Y0_@GbS+pmdO3y9`G1WNKo|S z3UaGuYV~aq1DSu5@p=eect{c~43E3Ht_R8kB10qIPK0~m0(@7(WfRH?DCf>K5O{N! zbZ2==ub<<@`abId!uFiLM4Shit^F@}*#b|!BdE2yL&%)_H0h53T3$wCiG)Q7Oj+Js zBy?nKboC{ICsI*{X%6l$OYPeLn?vT|(1b{X77aw}Kts@726LKV7oOJ$|HpN^Tn^yl zv(D4xpqg_Wkzf;!1Adnn9TfpZGG~CXO{f#;UBy>s;Y7(e;yXx($*1AwFC5K^wQ6J~ zcRir3IJ1l$$xO~R*bq5={Dhtl4$df6BHYg+;S2JcGsdZ|!>x;S;CL`!mKX+P!A<6F z+rwlAf#I=cBd3w!PzJn^@rX4PB#4!qFeo`t)yrk0JOG^}E@4edq7sD5J^>^=&POKn z%tRmnE0L&#V{Z&hLcxRUNmNEPl^c!c!g%UINMnwQ&?%93peq^~ox$ZE8ykutaaagF zk&iss9%+I_p=pEDgb8KpiV6iP1S7SYbpzR#Mk&IiH9&Dr# zDYtG8H4tR06*VhSe~L$>ekmMj9O8MvHfR)64Yg$qi3>bNT0l||V_3lO$_o@QM*Kw^ z*Zf~Tkmd)5NsZ^OTG;49YKIqy|UWEciSio*n01V!O)veIx{A4~1 zmyu;!H$wU<*p?OYsxjF|1A#07)UgTC%7uVVQHL?3rc=F5)`wxg6L8;*WjV*P-K@{CQPd-KRQD_!l~=k%E;0$x zBZ@;q205d1B(5MRncCQmxK%id6FhD3LvIUdOj#Hiyz4|IJg1~bJ?|?~l!rpIW3zBU zu9t-e{F4~QbpZuv#{QsT^0Bo1I@!;NR|i#6ns6E?z;JRK<}@G?5Kl;R#6reLN<2wm zoKgVg>PIM%u=ry(k^g{3P)U&$97S!Bfexi7Ash@8t{547Hl#kfO<5$LgH`Sk9-fgd z#$mfvLsa-#GOM(VB*$);(&hX-CSLF{UhMJ{m>4L66(VgYpn$qIZrOZ~ET&3EdblDB z?PAbHTGpD-$(f26Cs-$et~jhx4O6SsENscyEf5-&ieUpg1N(F3?Bbehp#HJQTEnpN zW_g=g^0sZ+tfpqrRd^V$xH&Nd$s!w;l7?K(0H+_gHYdP0PO{LbSZ zBPwTxev3il|E7xMYy_)KAn`c`bi z@IcLu6y_#ICNZlvKRKt-b1;RN?RA+Nxf*=DqTdWm6h?@K*RTiDz!sY~ezuSw&lid_ z@F|iEk`hEp zc}@#7;fdoSVzjTy5t5vUKK=%XZ9}1y5Eps~H5LNc=8`9I8F8}#(a1JcY8OkDXj@G- zfFJTQB6>arDKCeD(Ah#74+2!^M^GWyT-i^Yh=*m>eNgD#N)dL1N=A(mkJe|x23S_& z*#pp*S_`i^P&QVFY-_RFaE2V7k_1oF6b#p1n}aEF@A>N|eD8^YIX*DQ2cAt=SL#S` zz4(Y?Q%(9_~NVO5uYd*Bd+bbMpf->I()(bqWto`iWTUvXo zadA8;y2s&n^cP0FbSCVV>5a{{0JAJFuA=Zuj_^7*Ec|*aQe0H}S*BIPR5i%lvsXSf zTa?fsEI~5E8GL8y6cJ_dRMW>!Rwje3ENYW03ZXS&QW-+*EP=Px2Foh5G}XfgJDLvS za}WzEPCn)J?8i=njSQQ-$B2W|7?kDmbWV&VG*yoJdBT=oh8;^7GqI_AgU*G?k_ICu zzzfl5k?;R$0-RwGe+$Z_`rry!2J5H3NZ;2qdX zSK4y6AKD4M%pf&mQAr4lhI$CdB^f7$9<@wO!Nv2cY%4XiNFk<5vE5H47)KTx&f0`? z(Sppe&>R z1O+2NDK~`cB$>tIWrd1~l{FM6i(;oXxSh!qR<5~91$&-`nxpSO0 zh%d9qIfWBob)37znJFApq%u8H>P%!taztz%kSg4a*k1Fzk55@-g~jbO0~yqeOiFg;gN_$L73d@xv;@as;k9tM0^TOd z7AXQ$33%}#Ea(>DKD@RQkGOlLs~rJ;Lq=|^P$x;D;%D`e32@W6xr~hkY+nc)T5Ob2 zyTJ_#AUIcy3zl!ej|Mr&XO*IJk*}^`OSn?4VNWK@nyLQ9E`*DEt?DjkGiQdOMyQtg zY7Z2k!H8*eR2f=qP?bQ{D`=3SY@N7;Ea?W%tIq=Ttb2`2XlAn=TDjZI{Szb4c zIy(&zGR?@)`>GXRX$G+6awQO+j*uD&K~s88XUnsh=Y|85gX1N}DW)v!tT>FAc4!@S zwdz#q&>zcVNDcxn>=VwH4-3loK4!^ri}Ewa);QQq1w0=Su0=v_IDLg;hqJtmES zjPMlVrxRdV;U29;2dA0cu!js=2&<%e&W&{5qh2~Nq}w=5s50-yb5-C>(*)TQOMOsN zBB)UyQ5!G=9CsCl~n!l=ChdoAfWxl5pZ<&`jF zTF|Fc6tK@}X+2VH?OU$qNRFiX>H5 zJ&QsPt;8OT@Gyb_2gp!ki$HLe&WbpE_klTN7#zKMA@SJWQ#sh`9c$1A#6_M3Ov7YR zx#fN_#zmbDp#8rklvb9VeyEIdd!EIHHa`ZS>V}-fs;gzj5}6sPWsB)8MK{(N%3H_V z3nAqV+wJ`$8%}59?07cwd|Xt+@xziqro%8{Z?qr?1tT5RrFRysf2MH)afh8KHOhF; z#?<&*KeM(~G7K!If*G&01Q1Hw8sBgs0e?*|)Z~JdfbbbhoT@}_gA=fng2@AyYliEk zmT@4w^AX5MX_XMyC%Dv$=n?8OSzR3Y7*J1<0X^JGhKo2jt2~?p;@HVoc{m54>HQ|W z9)t~DUl8gFGA7m5s}978BfriX5~vmNf`XJNC4GFR7W?uwm5>)17CVZ_0243SnM8bJ zvBT%!Z$GG2dfS5z;u_`PM6ty1j@gKl64VO@(m6><9A$!yL zyGCVUuiTi)=IDh+X3zc`jX@pZz=fvqCA7D27p-E~o7*>E@w~tlkzQ{?wq!GDJL;_W zeh=vZVm+DBfXK63YiYhOk&f?M+bv&Vab94TFOUud8UC%++!?WrX^UP7!J<5_vh#dk z<{e$eeY3;u_Ot>Ma1qMj81!^7Fwc0iamGTUZNLHJVVu3#L1zUT+Y(7g_Cav$edZchOBYn zO$;otMf~}|P|_v5!hjUlS%W^x)jckG-Q)%q>Z+_S$(quDnkD4>L$yT`D%)Kc9uC#- zA}>bJWMX~Dmt~lgFQ}QZ`6$3I+o4f2H`zZSqxdq!TP_dJS(6F#o(SWQ_Q2zMTn|KX^y}0NZCb#G9V9(-(bdXLHFvHB%408$QM*Aqx zX8tli+r9vwmY|yST0h>aHm$&0_9~qN&iTIP?X>=jgrXd)nx$F;Z+M-%W>#~{7t(kS zKYw+0rZC4H4sJ(BCMVTbai+G2-M=E={VO_bzG5%~Dzb2vWkeAab5(uTFjoiEYy5D9 zbpz*|aOZ7yU85Qb-|f2Q@b~4Z;DatP`(0V`fBg&w7n*>|-jGAFo2Kd?yfBV^uu*tm zC(EYJJm5~JU|gZ|M^0ezw$EpZ0=Kp#_?SF-$4DZivB3mxJ{Dc~O_q93h|{rALi z^ke_7LC*_4(O!y@QMC4<1pPb4%WZ{6TA#~*tq9Go0qQ>4cj16{)hn+G2j!LjizucxO7;{3l`oHrjiI1vGy`U(&NHV)H=hK)cx!_B8z#k*8 zGRi#uONB6+PqOcJZMT1SMrb?WLhuAPpp`iZ!~e|$^lM(x_7EhAoA!vWd-Ltu-r%#L z_Jvat%EDfGHQ!!1EwBAwZJ&fFvHL#R!G6Y{cI^PcH9O}!<2bbdDe#Yk0g4{P9>E8# zZ3h1?V|HDAN!Td0J&Lx2Y{u>#IU}c59@vLGru9i0wL|njHUSPQ+TkDv2LV0q`FvNj zR{%z*Y%sq-W>bk1CZ!|5MY=?pH^yj1(`;Oy!#$W!=W>Y3m1v8jd|wt1cQh zjnWVWl;X0rV$9M1+E5^9rl_s-JHS-ddrbtURb!AxdpSPW+x72 zBK>fhUB*ouMJQU=``gy{1;aNx?6ZD^ucl_bm8Q3Nrs>uXvdGpFRM+>lHgzvnO@?=A zy=}1E>|{xR;jDK&%Vq+0DYmd$?~-MhW(o&88#mW`L_>KEvnr)WT>_qBN zwJo#Rj+J?{9d){9R~X{WKs(qPAF_Amm9NCa`9q?w`|rjGuuZBry}b~wxQ zdUP{&W4T&dZgwIwvudjbq{3v;>?#(q_h}T1HXMM`?4oT)YH04C!*Hs@+WAuyJ2x8qrm<>T3)@JMznB`-)y~rtGDfwb*P& zLe`&dc6)jR=Hnd%gLG-7-b~t-+MRt^Pv z%2xTOd1+`KTjtKtbsTv#aR$m6r)Td^F8}cI-JT;GJx4w%e>QiAu7;_3=7%#MjQ^(R z*zY@)BLiP|DUV)G+#b8vv;QYEKbrX(iT?M_+vJrO#ix-Sy@EgP(V84t(#!+!xn3_7`q<-V+G4eVqO< z{qdd;_k7}hI=9*P+-DP?-vEI>J^k+5CogY2Hn4eMaHD7NQ}^ek&-=fqZVb=dDZIWh z{QBk#3mZcV_j(?Ax8ui`Kj{6)`Q*jFcxx-w);IWD;FfCZ9{!IGrTc(KV|wH8bDMpq zKbsHI_+`(TpF5ww__uEnQTfD|le0usA@QHbl=N8QZujVKx<~)Zp+xzYZR5k2Q|gfQm#o}(_JQ|CwsOjg|qwUi;_w&_>~YHEf_)yh-5Q}OT5M9Ep2FN@RoJ7Jh`PDQj=S^ zc#D}-Qn@Xq=VWrrO0u~wrP%Bpn0+F}TCVVxKD4~XTP}4dsnc7^t;890Vk~;6b89T! o)tP*!{SOBpPZSbc!%DK}_WEY~p?^wt{pt8tyOMm|J_n}%Hz5Y<9RL6T From c2d3de0999cd97178caf01e647b4531e5eb1ed1a Mon Sep 17 00:00:00 2001 From: Kevin Boos Date: Thu, 10 Sep 2026 00:23:24 -0700 Subject: [PATCH 3/4] robius-speech: add Dictation, the text model behind dictating into a field Transcripts are only half the job: a text field wants "replace these bytes with this text", partials have to revise the current utterance in place, dictated words need a space between them and whatever the user typed, and users keep typing and clicking mid-sentence. Every app that uses this crate would end up writing that same bookkeeping. Dictation owns it, with no dependency on any UI toolkit. Feed it each transcript and apply the Replacement it hands back; tell it when the user is about to change the field (interrupt) and what the field holds afterwards (settle), and it re-anchors at the caret adding only the words that aren't on screen yet, so nothing is lost or repeated. Each Replacement also says whether it continues an earlier one, so a run of revisions can share one undo step. --- crates/speech/README.md | 33 +++ crates/speech/src/dictation.rs | 510 +++++++++++++++++++++++++++++++++ crates/speech/src/lib.rs | 6 + 3 files changed, 549 insertions(+) create mode 100644 crates/speech/src/dictation.rs diff --git a/crates/speech/README.md b/crates/speech/README.md index 2b72652..9117725 100644 --- a/crates/speech/README.md +++ b/crates/speech/README.md @@ -68,6 +68,39 @@ Similarly, `Unavailable` means that the audio input or speech recognition servic just doesn't exist and retrying it will never work. +## Putting the words into a text field + +Transcripts are only half the job; a text field wants "replace these bytes with this text". +`Dictation` does that bookkeeping for you, and it knows nothing about any UI toolkit: + +```rust +use robius_speech::Dictation; + +// Speech replaces the current selection and carries on from there. +let mut dictation = Dictation::new(&field.text(), field.selection()); + +// For each `Transcript` event, forwarded to your UI thread: +if let Some(edit) = dictation.transcript(&text, is_final) { + field.replace_range(edit.range, &edit.text); + dictation.applied(); +} +``` + +Partials revise the current utterance in place and finals commit it, with a space kept +between the dictated words and whatever the user typed (but not before punctuation, +or after an opening bracket, or between CJK characters). + +Users also keep typing and clicking while they dictate, and that shouldn't lose or repeat +a word. When you see the user about to change the field (a keystroke, a click that moves +the caret), call `interrupt()` before their edit lands, and `settle()` with the field's new +text and selection once it has. Dictation then picks up again from wherever the caret is, +adding only the words that aren't on screen yet. If the platform IME is mid-composition, +treat that as an interruption too, and don't `settle()` until it has committed. + +Each `Replacement` also says whether it `continues` an earlier one, so you can group a +whole run of revisions into a single undo step. + + ## Platform integration **Apple.** Your application must declare `NSMicrophoneUsageDescription` and diff --git a/crates/speech/src/dictation.rs b/crates/speech/src/dictation.rs new file mode 100644 index 0000000..7c2df3a --- /dev/null +++ b/crates/speech/src/dictation.rs @@ -0,0 +1,510 @@ +//! Puts dictated words into a text field. +//! +//! A recognizer hands you transcripts, but a text field wants "replace these +//! bytes with this text". [`Dictation`] does the bookkeeping in between: it +//! remembers where the words go, revises the current utterance in place as +//! partial transcripts arrive, keeps a space between speech and whatever the +//! user typed, and copes with the user editing the field mid-sentence without +//! losing or repeating a word. It knows nothing about any UI toolkit; you apply +//! the [`Replacement`]s it returns to your own text field. + +use std::ops::Range; + +/// One edit to make to the field: replace `range` (byte offsets) with `text`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Replacement { + pub range: Range, + pub text: String, + /// Whether this revises what an earlier replacement wrote, rather than + /// starting to write somewhere new. Handy for grouping undo: a whole run of + /// revisions of one phrase reads best as a single undo step. + pub continues: bool, +} + +/// Where dictated words go in a text field, and what is there now. +/// +/// The cycle is: feed each transcript to [`transcript`](Self::transcript), +/// apply the replacement it returns, then call [`applied`](Self::applied). +/// When the user is about to change the field themselves (a keystroke, a click +/// that moves the caret), call [`interrupt`](Self::interrupt) first so nothing +/// is written underneath their edit, and once it has landed call +/// [`settle`](Self::settle) with the field's new text and selection. Dictation +/// then carries on from wherever the caret is, adding only the words that +/// aren't on screen yet. Calling `settle` after every event is fine; it also +/// notices changes you didn't see coming. +#[derive(Clone, Debug)] +pub struct Dictation { + /// The field's text when we last anchored, and the range of it we write over. + draft: String, + start: usize, + end: usize, + /// Utterances the recognizer has finished, and the one it is still revising. + committed: String, + pending: String, + /// What has actually been written over `start..end`; `None` until the first + /// replacement lands. + shown: Option, + /// A replacement handed out but not yet confirmed by `applied`. + offered: Option, + /// Set while the user is editing. `true` means an utterance was mid-flight + /// when they started, so we wait for it to finish before re-anchoring. + interrupted: Option, +} + +impl Dictation { + /// Speech replaces `selection` (byte offsets into `text`) and carries on from there. + pub fn new(text: &str, selection: Range) -> Self { + let Range { start, end } = ordered(selection); + let start = floor_char_boundary(text, start); + let end = floor_char_boundary(text, end); + Self { + draft: text.to_owned(), + start, + end, + committed: String::new(), + pending: String::new(), + shown: None, + offered: None, + interrupted: None, + } + } + + /// A transcript from the recognizer. A partial revises the current utterance, + /// a final commits it. Returns the edit that brings the field up to date, if + /// there is one; apply it and call [`applied`](Self::applied). + pub fn transcript(&mut self, text: &str, is_final: bool) -> Option { + self.pending = text.trim().to_owned(); + if is_final { + append_words(&mut self.committed, &self.pending); + self.pending.clear(); + } + if let Some(awaiting_final) = self.interrupted.as_mut() { + if is_final { + *awaiting_final = false; + } + return None; + } + self.offer() + } + + /// The replacement handed out last has been applied to the field. + pub fn applied(&mut self) { + if let Some(offered) = self.offered.take() { + self.shown = Some(offered); + } + } + + /// The user is changing the field or moving the caret. Nothing more is + /// written until [`settle`](Self::settle) sees their change has landed. + pub fn interrupt(&mut self) { + if self.interrupted.is_none() { + self.interrupted = Some(!self.pending.is_empty()); + self.offered = None; + } + } + + pub fn is_interrupted(&self) -> bool { + self.interrupted.is_some() + } + + /// The field's text and selection now that the current event has been + /// handled. Anything we didn't write ourselves counts as an interruption. + /// Once an interrupted utterance has finished (or none was in flight), + /// dictation re-anchors at the caret and returns whatever was said since + /// that isn't on screen yet, if anything. + pub fn settle(&mut self, text: &str, selection: Range) -> Option { + let selection = ordered(selection); + if self.interrupted.is_none() && !self.matches(text, &selection) { + self.interrupt(); + } + if self.interrupted != Some(false) { + return None; + } + let unseen = words_beyond(self.shown.as_deref().unwrap_or(""), &self.committed); + *self = Self { + committed: unseen, + pending: std::mem::take(&mut self.pending), + ..Self::new(text, selection) + }; + self.offer() + } + + fn offer(&mut self) -> Option { + let insertion = self.insertion(); + let same = match &self.shown { + Some(shown) => *shown == insertion, + None => insertion.is_empty(), + }; + if same { + return None; + } + let range = match &self.shown { + Some(shown) => self.start..self.start + shown.len(), + None => self.start..self.end, + }; + self.offered = Some(insertion.clone()); + Some(Replacement { range, text: insertion, continues: self.shown.is_some() }) + } + + /// Whether the field holds exactly what we last left in it. + fn matches(&self, text: &str, selection: &Range) -> bool { + let Some(shown) = &self.shown else { + return text == self.draft && *selection == (self.start..self.end); + }; + let caret = self.start + shown.len(); + *selection == (caret..caret) + && text.len() == self.draft.len() - (self.end - self.start) + shown.len() + && text[..self.start] == self.draft[..self.start] + && text[self.start..caret] == **shown + && text[caret..] == self.draft[self.end..] + } + + /// Everything said since the anchor, spaced off the draft around it. + fn insertion(&self) -> String { + let mut spoken = self.committed.clone(); + append_words(&mut spoken, &self.pending); + if spoken.is_empty() { + return spoken; + } + if needs_space(&self.draft[..self.start], &spoken) { + spoken.insert(0, ' '); + } + if needs_space(&spoken, &self.draft[self.end..]) { + spoken.push(' '); + } + spoken + } +} + +fn ordered(range: Range) -> Range { + range.start.min(range.end)..range.start.max(range.end) +} + +fn floor_char_boundary(text: &str, index: usize) -> usize { + let mut index = index.min(text.len()); + while !text.is_char_boundary(index) { + index -= 1; + } + index +} + +/// The part of `spoken` that goes beyond what the user can already see in +/// `shown`. Words are compared loosely, because a recognizer routinely re-cases +/// or re-punctuates what it already sent. Returns nothing when the two share no +/// leading words at all, so nothing is ever inserted twice. +fn words_beyond(shown: &str, spoken: &str) -> String { + if shown.trim().is_empty() { + return spoken.trim().to_owned(); + } + let loose = |word: &str| word.trim_matches(|c: char| !c.is_alphanumeric()).to_lowercase(); + let shown_words: Vec<_> = shown.split_whitespace().map(loose).collect(); + let mut matched = 0; + let mut rest = spoken.trim(); + for word in spoken.split_whitespace() { + if matched >= shown_words.len() || loose(word) != shown_words[matched] { + break; + } + matched += 1; + rest = rest[word.len()..].trim_start(); + } + // Every word the user can see must be accounted for. If the revision inserted + // a word inside them, the tail is not safely separable, so insert nothing. + if matched < shown_words.len() { String::new() } else { rest.to_owned() } +} + +fn append_words(destination: &mut String, text: &str) { + if needs_space(destination, text) { + destination.push(' '); + } + destination.push_str(text); +} + +/// Whether dictated text needs a space to keep it off whatever precedes it. +/// +/// Speech joins onto a draft the user may have typed, so the default is to +/// separate them. The exceptions are all cases where jamming them together is +/// what was meant: an opening bracket the user just typed, punctuation that the +/// recognizer itself supplies, and scripts that don't space their words. +/// An apostrophe is only a joiner on the right, for endings like `'s`; a draft +/// that happens to end in one still gets its space. +fn needs_space(left: &str, right: &str) -> bool { + let (Some(left), Some(right)) = (left.chars().last(), right.chars().next()) else { return false }; + let cjk = |c| matches!(c, '\u{3000}'..='\u{30ff}' | '\u{3400}'..='\u{9fff}' | '\u{ac00}'..='\u{d7af}' | '\u{f900}'..='\u{faff}'); + !left.is_whitespace() && !right.is_whitespace() + && !matches!(left, '(' | '[' | '{') + && !matches!(right, '.' | ',' | '!' | '?' | ':' | ';' | ')' | ']' | '}' | '\'' | '\u{2019}') + && !cjk(left) && !cjk(right) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A stand-in text field: applies each replacement and reports back. + struct Field { + text: String, + caret: usize, + } + + impl Field { + fn new(text: &str, caret: usize) -> Self { + Self { text: text.into(), caret } + } + + fn apply(&mut self, dictation: &mut Dictation, replacement: Option) { + if let Some(replacement) = replacement { + self.text.replace_range(replacement.range.clone(), &replacement.text); + self.caret = replacement.range.start + replacement.text.len(); + dictation.applied(); + } + } + + fn say(&mut self, dictation: &mut Dictation, text: &str, is_final: bool) { + let replacement = dictation.transcript(text, is_final); + self.apply(dictation, replacement); + } + + /// The user types at the caret, as a toolkit event would deliver it. + fn type_text(&mut self, dictation: &mut Dictation, text: &str) { + dictation.interrupt(); + self.text.insert_str(self.caret, text); + self.caret += text.len(); + self.settle(dictation); + } + + fn settle(&mut self, dictation: &mut Dictation) { + let replacement = dictation.settle(&self.text, self.caret..self.caret); + self.apply(dictation, replacement); + } + } + + #[test] + fn partials_replace_the_utterance_and_finals_append() { + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "I scream", false); + assert_eq!(field.text, "I scream"); + field.say(&mut dictation, "Ice cream.", true); + assert_eq!(field.text, "Ice cream."); + field.say(&mut dictation, "Please", false); + assert_eq!(field.text, "Ice cream. Please"); + field.say(&mut dictation, "Please!", true); + assert_eq!(field.text, "Ice cream. Please!"); + assert_eq!(field.caret, field.text.len()); + } + + #[test] + fn the_first_replacement_starts_a_run_and_later_ones_continue_it() { + let mut dictation = Dictation::new("Hi old friend", 3..6); + let first = dictation.transcript("new", false).unwrap(); + assert_eq!(first, Replacement { range: 3..6, text: "new".into(), continues: false }); + dictation.applied(); + let second = dictation.transcript("dear", false).unwrap(); + assert_eq!(second, Replacement { range: 3..6, text: "dear".into(), continues: true }); + // A revision that changes nothing is not an edit, nor is the final that + // merely confirms it. + dictation.applied(); + assert_eq!(dictation.transcript("dear", false), None); + assert_eq!(dictation.transcript("dear", true), None); + } + + #[test] + fn a_replacement_that_never_landed_is_simply_offered_again() { + let mut dictation = Dictation::new("", 0..0); + assert_eq!(dictation.transcript("one", false).unwrap().range, 0..0); + // Not applied, so the field still holds nothing; the next one starts over. + let again = dictation.transcript("one two", false).unwrap(); + assert_eq!(again, Replacement { range: 0..0, text: "one two".into(), continues: false }); + dictation.applied(); + assert_eq!(dictation.transcript("one two three", true).unwrap().range, 0..7); + } + + #[test] + fn speech_is_spaced_off_the_draft_but_not_off_punctuation_or_brackets() { + let after = |draft: &str, spoken: &str| { + let mut dictation = Dictation::new(draft, draft.len()..draft.len()); + let mut field = Field::new(draft, draft.len()); + field.say(&mut dictation, spoken, true); + field.text + }; + assert_eq!(after("hello", "world"), "hello world"); + assert_eq!(after("hello,", "world"), "hello, world"); + assert_eq!(after("hello'", "world"), "hello' world"); + assert_eq!(after("hello ", "world"), "hello world"); + assert_eq!(after("hello\n", "world"), "hello\nworld"); + assert_eq!(after("hello(", "world"), "hello(world"); + assert_eq!(after("你好", "世界"), "你好世界"); + assert_eq!(after("", "say enter"), "say enter"); + + // Recognizer punctuation joins onto the words before it. + let mut dictation = Dictation::new("(", 1..1); + let mut field = Field::new("(", 1); + field.say(&mut dictation, "say enter", true); + field.say(&mut dictation, ", then stop)", true); + assert_eq!(field.text, "(say enter, then stop)"); + + // And a caret parked mid-draft gets a space on both sides. + let mut dictation = Dictation::new("abcdef", 3..3); + let mut field = Field::new("abcdef", 3); + field.say(&mut dictation, "MID", true); + assert_eq!(field.text, "abc MID def"); + + // Silence replaces nothing and adds no whitespace. + let mut dictation = Dictation::new("keep this", 0..9); + assert_eq!(dictation.transcript(" ", false), None); + assert_eq!(dictation.transcript("", true), None); + } + + #[test] + fn selection_offsets_are_ordered_and_kept_on_char_boundaries() { + let draft = "Hi 🦀, old text today"; + let start = draft.find("old").unwrap(); + let end = draft.find(" today").unwrap(); + let mut dictation = Dictation::new(draft, end..start); + assert_eq!(dictation.transcript("new text", false).unwrap().range, start..end); + // Inside the crab: floored to its start. Past the end: clamped. + let inside = draft.find('🦀').unwrap() + 1; + assert_eq!(Dictation::new(draft, inside..inside).start, inside - 1); + assert_eq!(Dictation::new(draft, 999..999).start, draft.len()); + } + + #[test] + fn typing_mid_utterance_keeps_the_words_spoken_afterwards() { + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "hello world", false); + field.type_text(&mut dictation, "X"); + assert!(dictation.is_interrupted(), "still waiting for the utterance to finish"); + // Revisions in the meantime are not written under the user's edit. + field.say(&mut dictation, "hello world and", false); + assert_eq!(field.text, "hello worldX"); + field.say(&mut dictation, "hello world and more", true); + field.settle(&mut dictation); + assert_eq!(field.text, "hello worldX and more"); + assert!(!dictation.is_interrupted()); + } + + #[test] + fn typing_between_utterances_re_anchors_at_once() { + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "first", true); + field.type_text(&mut dictation, "X"); + assert!(!dictation.is_interrupted()); + field.say(&mut dictation, "second", true); + assert_eq!(field.text, "firstX second"); + + // Typing before anything was recognized at all. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.type_text(&mut dictation, "typed first"); + field.say(&mut dictation, "spoken after", true); + assert_eq!(field.text, "typed first spoken after"); + } + + #[test] + fn a_revision_after_an_edit_never_undoes_it_or_repeats_itself() { + // Backspace, then the same utterance again: nothing to add. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "typo here", false); + dictation.interrupt(); + field.text.pop(); + field.caret -= 1; + field.settle(&mut dictation); + field.say(&mut dictation, "typo here", false); + field.say(&mut dictation, "typo here.", true); + field.settle(&mut dictation); + assert_eq!(field.text, "typo her"); + + // Wiping the dictated text and typing over it. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "wipe me", false); + dictation.interrupt(); + field.text = "fresh".into(); + field.caret = 5; + field.settle(&mut dictation); + field.say(&mut dictation, "wipe me", true); + field.settle(&mut dictation); + field.say(&mut dictation, "then more", true); + assert_eq!(field.text, "fresh then more"); + + // A wholesale revision that shares no words with what was shown is + // dropped rather than repeated; speech resumes with the next utterance. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "keep these words", false); + dictation.interrupt(); + field.caret = 4; + field.settle(&mut dictation); + field.say(&mut dictation, "stale revision", true); + field.settle(&mut dictation); + assert_eq!(field.text, "keep these words"); + field.say(&mut dictation, "and more", true); + assert_eq!(field.text, "keep and more these words"); + } + + #[test] + fn a_final_that_arrives_during_the_edit_waits_for_it() { + // The final is fed before the edit lands, exactly as when both happen in + // one event; the words it adds still go after the edit. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "keep these words", false); + dictation.interrupt(); + assert_eq!(dictation.transcript("keep these words and more", true), None); + field.text = "keep words".into(); + field.caret = 5; + field.settle(&mut dictation); + assert_eq!(field.text, "keep and more words"); + + // Two finals in the same gap both make it in. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "hello", false); + dictation.interrupt(); + assert_eq!(dictation.transcript("hello world", true), None); + assert_eq!(dictation.transcript("again", true), None); + field.text = "hello!".into(); + field.caret = 6; + field.settle(&mut dictation); + assert_eq!(field.text, "hello! world again"); + } + + #[test] + fn settle_notices_changes_that_arrived_without_warning() { + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "hello", false); + // Nothing changed: not an interruption. + field.settle(&mut dictation); + assert!(!dictation.is_interrupted()); + // The caret moved: an interruption, resolved at the utterance boundary. + field.caret = 0; + field.settle(&mut dictation); + assert!(dictation.is_interrupted()); + field.say(&mut dictation, "hello there", true); + field.settle(&mut dictation); + // The caret follows the words, including the space that keeps them apart. + assert_eq!(field.text, "there hello"); + assert_eq!(field.caret, 6); + } + + #[test] + fn words_beyond_only_returns_what_was_not_already_shown() { + assert_eq!(words_beyond("hello world", "hello world and more"), "and more"); + // Re-casing and re-punctuating is not new speech. + assert_eq!(words_beyond("hello world", "Hello, world! And more"), "And more"); + assert_eq!(words_beyond("hello world", "Hello world."), ""); + assert_eq!(words_beyond("", "all of it"), "all of it"); + assert_eq!(words_beyond(" ", "all of it"), "all of it"); + // No shared leading words: lose the tail rather than repeat the draft. + assert_eq!(words_beyond("hello world", "goodbye everyone"), ""); + // A word inserted inside what is shown is not separable either. + assert_eq!(words_beyond("hello world", "hello big world and more"), ""); + assert_eq!(words_beyond("the cat sat", "the cat quickly sat down"), ""); + // A shorter final is a revision, not new speech. + assert_eq!(words_beyond("hello world and more", "hello world"), ""); + } +} diff --git a/crates/speech/src/lib.rs b/crates/speech/src/lib.rs index 733fbfd..c2a3323 100644 --- a/crates/speech/src/lib.rs +++ b/crates/speech/src/lib.rs @@ -8,6 +8,9 @@ //! spoken works as commands or synthesizes key events from them, //! so saying something like "enter" will just type the word "enter". //! +//! [`Dictation`] turns those transcripts into edits of a text field, so an app +//! only has to apply the replacements it hands back. +//! //! Apple builds need the Xcode command line tools (`xcode-select --install`), and //! support macOS 11+ and iOS 13+. For macOS builds, set the `MACOSX_DEPLOYMENT_TARGET` //! env var to 11.0 or newer for the whole Cargo invocation, including the final app. @@ -17,6 +20,9 @@ use std::collections::HashMap; use std::sync::{Arc, Mutex, OnceLock}; use std::sync::atomic::{AtomicU64, Ordering}; +pub mod dictation; +pub use dictation::{Dictation, Replacement}; + #[cfg(any(target_os = "macos", target_os = "ios"))] mod apple; #[cfg(all(target_os = "android", native_speech_android))] From 166991b1b18188ee2d8441f7118721cee2a0c6eb Mon Sep 17 00:00:00 2001 From: Kevin Boos Date: Thu, 10 Sep 2026 11:20:28 -0700 Subject: [PATCH 4/4] robius-speech: call off an interruption that changed nothing A caret key that doesn't move the caret (End at the end, ArrowRight at the end) or a click right on the caret is warned about as an edit before anyone can know it will change nothing. settle() then kept the interruption, swallowed every further revision of the utterance, and once its final arrived offered only the words beyond what was shown, so a recognizer that rewrote "I scream" as "Ice cream please" lost both the correction and the new word. settle() now checks the field first: if it holds exactly what dictation left in it, the interruption is called off and the utterance carries on, revisions included. Also covers a partial fed during an edit with a test. --- crates/speech/src/dictation.rs | 60 ++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/crates/speech/src/dictation.rs b/crates/speech/src/dictation.rs index 7c2df3a..0918fdb 100644 --- a/crates/speech/src/dictation.rs +++ b/crates/speech/src/dictation.rs @@ -108,15 +108,19 @@ impl Dictation { } /// The field's text and selection now that the current event has been - /// handled. Anything we didn't write ourselves counts as an interruption. - /// Once an interrupted utterance has finished (or none was in flight), - /// dictation re-anchors at the caret and returns whatever was said since - /// that isn't on screen yet, if anything. + /// handled. Anything we didn't write ourselves counts as an interruption, + /// and an interruption that turns out to have changed nothing (End with the + /// caret already at the end, a click on the caret) is called off. Once an + /// interrupted utterance has finished (or none was in flight), dictation + /// re-anchors at the caret and returns whatever was said since that isn't + /// on screen yet, if anything. pub fn settle(&mut self, text: &str, selection: Range) -> Option { let selection = ordered(selection); - if self.interrupted.is_none() && !self.matches(text, &selection) { - self.interrupt(); + if self.matches(text, &selection) { + self.interrupted = None; + return self.offer(); } + self.interrupt(); if self.interrupted != Some(false) { return None; } @@ -491,6 +495,50 @@ mod tests { assert_eq!(field.caret, 6); } + #[test] + fn an_interruption_that_changed_nothing_is_called_off() { + // End with the caret already at the end, or a click on the caret: the + // field is untouched, so the utterance carries on, revisions included. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "I scream", false); + dictation.interrupt(); + assert_eq!(dictation.transcript("I scream for", false), None, "held while the edit is pending"); + field.settle(&mut dictation); + assert!(!dictation.is_interrupted()); + assert_eq!(field.text, "I scream for", "the held revision lands as soon as nothing changed"); + field.say(&mut dictation, "Ice cream please", true); + assert_eq!(field.text, "Ice cream please"); + + // The same when the final itself arrived while the interruption was pending. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "hello world", false); + dictation.interrupt(); + assert_eq!(dictation.transcript("Hello, world.", true), None); + field.settle(&mut dictation); + assert_eq!(field.text, "Hello, world."); + } + + #[test] + fn a_partial_that_arrives_during_an_edit_is_written_after_it() { + // Between utterances the user types, and the next utterance's first + // partial is fed before their edit lands. Nothing was in flight, so the + // edit re-anchors at once and the partial goes after it. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "first", true); + dictation.interrupt(); + assert_eq!(dictation.transcript("second", false), None); + field.text.push('X'); + field.caret += 1; + field.settle(&mut dictation); + assert_eq!(field.text, "firstX second"); + assert!(!dictation.is_interrupted()); + field.say(&mut dictation, "second one", true); + assert_eq!(field.text, "firstX second one"); + } + #[test] fn words_beyond_only_returns_what_was_not_already_shown() { assert_eq!(words_beyond("hello world", "hello world and more"), "and more");