From 56b059340f417892ab20850b31e80cfd4f9407f8 Mon Sep 17 00:00:00 2001 From: Kevin Boos Date: Fri, 11 Sep 2026 15:42:00 -0700 Subject: [PATCH 1/4] robius-speech: don't discard words spoken before a pause Apple's on-device recognizer ends a segment after two seconds or so of silence, but then restarts the best transcription without setting "is final". But our swift bridge code didn't really account for that, which means it got revised into one "utterance". we now never drop any words that were already "sent" (digested?) in the segment tracker. We "commit" each chunk of words when it's over, and then make sure that we still allow shorter replacements (corrections?) to take effect and replace previously committed chunks. i tested this with a ton of different phrases and it's solid. For example, "three hundred and forty seven" does get replaced with "347" using the proper native revision. --- crates/speech/swift/NativeSpeech.swift | 121 ++++++++++++++++++-- crates/speech/swift/tests/SpeechTests.swift | 64 ++++++++++- 2 files changed, 174 insertions(+), 11 deletions(-) diff --git a/crates/speech/swift/NativeSpeech.swift b/crates/speech/swift/NativeSpeech.swift index 377d210..c84f4fa 100644 --- a/crates/speech/swift/NativeSpeech.swift +++ b/crates/speech/swift/NativeSpeech.swift @@ -54,6 +54,108 @@ struct SpeechRetryPolicy { } } +/// Turns one recognition task's results into transcripts that never drop words already sent. +/// +/// After a pause, on-device recognition closes a segment with one result that has metadata, +/// then restarts `bestTranscription` from the new words only, without ever setting `isFinal`. +/// Its final result after `endAudio` can hold only the last segment, or nothing at all. +struct SegmentTracker { + /// Words sent as a partial but not yet committed. + private(set) var open = "" + /// The last text committed, and the whole transcription at that point. + private var committed = "" + private var closedTranscription = "" + /// Whether this task restarts its transcription after each segment, rather than growing it. + private var restarts = false + + /// Returns the transcripts to send (kind 1 partial, kind 2 final) for one result. + mutating func result(_ transcription: String, isFinal: Bool, closesSegment: Bool) -> [(Int32, String)] { + var text = transcription.trimmingCharacters(in: .whitespacesAndNewlines) + // A transcription that grows instead of restarting still holds what was committed. + if !closedTranscription.isEmpty && !restarts { + if let rest = looseRemainder(of: text, after: closedTranscription) { + text = rest + } else if !text.isEmpty { + restarts = true + } + } + var out: [(Int32, String)] = [] + if isFinal { + // An empty or cut-short final keeps what was shown, and a repeat of the + // segment just committed isn't committed again. + if text.isEmpty || (open.isEmpty && looselyEqual(text, committed)) || isShortenedRevision(of: open, text) { + return flush() + } + if startsOver(from: open, to: text) { out += flush() } + open = "" + return out + commit(text) + } + guard !text.isEmpty, !(closesSegment && open.isEmpty && looselyEqual(text, committed)) else { return [] } + if startsOver(from: open, to: text) { out += flush() } + if closesSegment { + open = "" + closedTranscription = transcription + return out + commit(text) + } + open = text + return out + [(1, text)] + } + + /// Commits the words shown but not yet committed, e.g. when a task ends without a final. + mutating func flush() -> [(Int32, String)] { + defer { open = "" } + return commit(open) + } + + private mutating func commit(_ text: String) -> [(Int32, String)] { + guard !text.isEmpty else { return [] } + committed = text + return [(2, text)] + } +} + +/// Words compared loosely, ignoring case and punctuation, as `dictation.rs` does. +func looseWords(_ text: String) -> [String] { + text.split(whereSeparator: { $0.isWhitespace }) + .map { $0.trimmingCharacters(in: CharacterSet.alphanumerics.inverted).lowercased() } + .filter { !$0.isEmpty } +} + +func looselyEqual(_ a: String, _ b: String) -> Bool { + looseWords(a) == looseWords(b) +} + +/// Whether `new` starts over rather than revising `old`, as after a pause without metadata: +/// it drops most words of a long `old`, rather than reformatting a few (e.g. "300 and" to "340"). +func startsOver(from old: String, to new: String) -> Bool { + let (old, new) = (looseWords(old), looseWords(new)) + let kept = old.filter(new.contains).count + return old.count >= 6 && !new.isEmpty && new.count * 2 < old.count && kept * 4 < old.count +} + +/// Whether `new` is `old` with words cut off the end. +func isShortenedRevision(of old: String, _ new: String) -> Bool { + let (old, new) = (looseWords(old), looseWords(new)) + return !new.isEmpty && new.count < old.count && Array(old.prefix(new.count)) == new +} + +/// The rest of `text` after its first words, if they loosely match all of `prefix`'s words. +func looseRemainder(of text: String, after prefix: String) -> String? { + var expected = looseWords(prefix)[...] + var rest = Substring(text) + while let word = expected.first { + rest = rest.drop(while: { $0.isWhitespace }) + guard !rest.isEmpty else { return nil } + let token = rest.prefix(while: { !$0.isWhitespace }) + rest = rest.dropFirst(token.count) + let loose = looseWords(String(token)) + if loose.isEmpty { continue } + guard loose == [word] else { return nil } + expected = expected.dropFirst() + } + return rest.trimmingCharacters(in: .whitespacesAndNewlines) +} + // The engine keeps recording while a recognizer finalizes an utterance. Only // that short interval needs copies; otherwise buffers go straight to the request. // No engine operations or UI callbacks run while this lock is held. @@ -165,7 +267,7 @@ final class NativeSpeech { var finished = false var tapInstalled = false var started = false - var lastPartial = "" + var segments = SegmentTracker() var generation: UInt64 = 0 var retryPolicy = SpeechRetryPolicy() var endingUtterance = false @@ -301,7 +403,7 @@ final class NativeSpeech { } if #available(macOS 13.0, iOS 16.0, *) { request.addsPunctuation = true } self.request = request - lastPartial = "" + segments = SegmentTracker() let capture = self.capture capture.prepareRequest(utterance) task = recognizer.recognitionTask(with: request) { [weak self] result, error in @@ -313,11 +415,16 @@ final class NativeSpeech { DispatchQueue.main.async { guard let self = self, !self.finished, self.generation == utterance else { return } if let result = result { - self.lastPartial = result.bestTranscription.formattedString - if !self.lastPartial.isEmpty { self.retryPolicy.madeProgress() } - self.send(result.isFinal ? 2 : 1, self.lastPartial) + let text = result.bestTranscription.formattedString + if !text.isEmpty { self.retryPolicy.madeProgress() } + var closesSegment = false + if #available(iOS 14.0, *) { + closesSegment = !result.isFinal && result.speechRecognitionMetadata != nil + } + for (kind, text) in self.segments.result(text, isFinal: result.isFinal, closesSegment: closesSegment) { + self.send(kind, text) + } if result.isFinal { - self.lastPartial = "" if self.stopping { self.finishStoppedUtterance() } else { self.restartUtterance(after: 0.15) } return @@ -502,7 +609,7 @@ final class NativeSpeech { } func commitPartial() { - if !lastPartial.isEmpty { send(2, lastPartial); lastPartial = "" } + for (kind, text) in segments.flush() { send(kind, text) } } func finish() { diff --git a/crates/speech/swift/tests/SpeechTests.swift b/crates/speech/swift/tests/SpeechTests.swift index 8afbeca..5c184cd 100644 --- a/crates/speech/swift/tests/SpeechTests.swift +++ b/crates/speech/swift/tests/SpeechTests.swift @@ -28,6 +28,11 @@ enum SpeechTests { ("utterance rollover commits the draft", rolloverCommitsTheDraft), ("cancelling during rollover emits nothing", cancellingDuringRolloverIsSilent), ("a graceful stop replays buffered speech", gracefulStopReplaysBufferedSpeech), + ("a segment closed before a restart is kept", segmentsSurviveRestarts), + ("a growing transcription commits each segment once", growingTranscriptionIsNotRepeated), + ("a restart without metadata keeps the earlier words", restartWithoutMetadataKeepsWords), + ("an empty, cut-short or repeated final keeps what was shown", finalsNeverRemoveShownWords), + ("a revision of the current utterance stays a revision", revisionsStayRevisions), ] static func main() { @@ -203,7 +208,7 @@ enum SpeechTests { precondition(rollover.engine == nil, "Creating a session must not initialize microphone hardware") rollover.started = true rollover.generation = 7 - rollover.lastPartial = "Keep this draft" + _ = rollover.segments.result("Keep this draft", isFinal: false, closesSegment: false) rollover.endUtteranceBeforeLimit(6, finalizationTimeout: 0) precondition(!rollover.endingUtterance, "An old utterance timer must not affect its successor") rollover.endUtteranceBeforeLimit(7, finalizationTimeout: 0) @@ -219,7 +224,7 @@ enum SpeechTests { let before = nativeEvents.count let cancelled = NativeSpeech(id: 44, locale: "en-US", preferOnDevice: true) cancelled.generation = 1 - cancelled.lastPartial = "Cancelled partial" + _ = cancelled.segments.result("Cancelled partial", isFinal: false, closesSegment: false) cancelled.endUtteranceBeforeLimit(1, finalizationTimeout: 0) cancelled.stop(cancel: true) var drained = false @@ -232,7 +237,7 @@ enum SpeechTests { nativeEvents.removeAll() let stopping = NativeSpeech(id: 45, locale: "", preferOnDevice: true) stopping.stopping = true - stopping.lastPartial = "Previous utterance" + _ = stopping.segments.result("Previous utterance", isFinal: false, closesSegment: false) stopping.capture.beginRequest { _ in } stopping.capture.endRequest() precondition(stopping.capture.append(audioBuffer(sample: 6))) @@ -243,14 +248,65 @@ enum SpeechTests { var replayed: [Float] = [] stopping.capture.beginRequest { replayed.append($0.floatChannelData![0][0]) } precondition(replayed == [6]) - stopping.lastPartial = "Buffered final words" + _ = stopping.segments.result("Buffered final words", isFinal: false, closesSegment: false) stopping.finishStoppedUtterance() precondition(nativeEvents.map(\.kind) == [2, 2, 4]) precondition(nativeEvents[1].text == "Buffered final words" && stopping.finished) } + // MARK: - Segments + + static func segmentsSurviveRestarts() { + // Apple's on-device trace: the closing result repeats the last partial, then the text restarts. + var tracker = SegmentTracker() + let sent = feed(&tracker, [("Can", false, false), ("Can you", false, false), ("Can you", false, true), + ("Hear", false, false), ("Hear this", false, false), ("Hear this", false, true), ("", true, false)]) + precondition(sent == ["1:Can", "1:Can you", "2:Can you", "1:Hear", "1:Hear this", "2:Hear this"], "\(sent)") + } + + static func growingTranscriptionIsNotRepeated() { + var tracker = SegmentTracker() + let sent = feed(&tracker, [("Can you", false, false), ("Can you", false, true), ("Can you hear", false, false), + ("Can you hear this", false, true), ("Can you hear this.", true, false)]) + precondition(sent == ["1:Can you", "2:Can you", "1:hear", "2:hear this"], "\(sent)") + } + + static func restartWithoutMetadataKeepsWords() { + var tracker = SegmentTracker() + let sent = feed(&tracker, [("one two three four five six", false, false), ("seven", false, false), ("seven eight", true, false)]) + precondition(sent == ["1:one two three four five six", "2:one two three four five six", "1:seven", "2:seven eight"], "\(sent)") + } + + static func finalsNeverRemoveShownWords() { + var tracker = SegmentTracker() + var sent = feed(&tracker, [("keep these words", false, false), ("", true, false)]) + precondition(sent == ["1:keep these words", "2:keep these words"], "\(sent)") + tracker = SegmentTracker() + sent = feed(&tracker, [("keep these words", false, false), ("Keep these.", true, false)]) + precondition(sent == ["1:keep these words", "2:keep these words"], "\(sent)") + tracker = SegmentTracker() + sent = feed(&tracker, [("said once", false, true), ("said once", false, true), ("Said once.", true, false)]) + precondition(sent == ["2:said once"], "\(sent)") + } + + static func revisionsStayRevisions() { + var tracker = SegmentTracker() + let sent = feed(&tracker, [("I scream", false, false), ("Ice cream", false, false), ("Ice cream please.", true, false)]) + precondition(sent == ["1:I scream", "1:Ice cream", "2:Ice cream please."], "\(sent)") + // Apple reformats numbers as they're spoken, e.g. "Three hundred and forty seven apples". + var numbers = SegmentTracker() + let reformatted = feed(&numbers, [("Three", false, false), ("300", false, false), ("300 and", false, false), + ("340", false, false), ("347 apples", false, false), ("347 apples.", true, false)]) + precondition(reformatted.last == "2:347 apples." && !reformatted.contains { $0.hasPrefix("2:300") }, "\(reformatted)") + } + // MARK: - Helpers + /// Feeds (text, isFinal, closesSegment) results and returns what would be sent, as "kind:text". + private static func feed(_ tracker: inout SegmentTracker, _ results: [(String, Bool, Bool)]) -> [String] { + results.flatMap { tracker.result($0.0, isFinal: $0.1, closesSegment: $0.2) }.map { "\($0.0):\($0.1)" } + } + private static func audioBuffer(sample: Float, frames: AVAudioFrameCount = 1) -> AVAudioPCMBuffer { let format = AVAudioFormat(standardFormatWithSampleRate: 8_000, channels: 1)! let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames)! From b581e83ade88f3ea4f2a520604b902e61451be26 Mon Sep 17 00:00:00 2001 From: Kevin Boos Date: Fri, 11 Sep 2026 15:47:38 -0700 Subject: [PATCH 2/4] robius-speech: never let a transcript remove words that were already shown If dictation got an empty transcript after we already drew pending words, then we might end up replacing all of the pending words with that empty transcript, which looks like the drafted recognition was randomly erased. Now, across all the backends, an empty transcript doesn't change anything, only a final commit can change what was pending. Of course you can still replace the current "utterance" with revisions that come from the native service. --- crates/speech/src/dictation.rs | 67 ++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/crates/speech/src/dictation.rs b/crates/speech/src/dictation.rs index 5f41ce6..c01d294 100644 --- a/crates/speech/src/dictation.rs +++ b/crates/speech/src/dictation.rs @@ -73,10 +73,21 @@ impl Dictation { /// a final commits it. Returns the edit that brings the field up to date, if /// there is one; apply it and call [`applied`](Self::applied). pub fn transcript(&mut self, text: &str, is_final: bool) -> Option { - self.pending = text.trim().to_owned(); + // Never lose words the recognizer already gave us: an empty transcript changes nothing, + // one that starts over commits what it replaces, and one that stops short keeps the rest. + let text = text.trim(); + if !text.is_empty() { + let previous = std::mem::take(&mut self.pending); + self.pending = text.to_owned(); + if starts_over(&previous, text) { + append_words(&mut self.committed, &previous); + } else { + append_words(&mut self.pending, &words_beyond(text, &previous)); + } + } if is_final { - append_words(&mut self.committed, &self.pending); - self.pending.clear(); + let pending = std::mem::take(&mut self.pending); + append_words(&mut self.committed, &pending); } if let Some(awaiting_final) = self.interrupted.as_mut() { if is_final { @@ -193,6 +204,23 @@ fn floor_char_boundary(text: &str, index: usize) -> usize { index } +/// Whether `new` starts over rather than revising `old`, like a recognizer that restarted after +/// a pause: it drops most words of a long `old`, rather than reformatting a few ("300 and" to "340"). +fn starts_over(old: &str, new: &str) -> bool { + let (old, new) = (loose_words(old), loose_words(new)); + let kept = old.iter().filter(|word| new.contains(word)).count(); + old.len() >= 6 && !new.is_empty() && new.len() * 2 < old.len() && kept * 4 < old.len() +} + +fn loose_words(text: &str) -> Vec { + text.split_whitespace().map(loose).filter(|word| !word.is_empty()).collect() +} + +/// A word compared loosely, because a recognizer routinely re-cases or re-punctuates. +fn loose(word: &str) -> String { + word.trim_matches(|c: char| !c.is_alphanumeric()).to_lowercase() +} + /// The part of `spoken` that goes beyond what the user can already see in /// `shown`. Words are compared loosely, because a recognizer routinely re-cases /// or re-punctuates what it already sent. Returns nothing when the two share no @@ -201,7 +229,6 @@ fn words_beyond(shown: &str, spoken: &str) -> String { if shown.trim().is_empty() { return spoken.trim().to_owned(); } - let loose = |word: &str| word.trim_matches(|c: char| !c.is_alphanumeric()).to_lowercase(); let shown_words: Vec<_> = shown.split_whitespace().map(loose).collect(); let mut matched = 0; let mut rest = spoken.trim(); @@ -556,6 +583,38 @@ mod tests { assert_eq!(field.text, "éX bc"); } + #[test] + fn words_already_shown_are_never_removed() { + // An empty transcript, partial or final, keeps what is shown. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "keep me", false); + field.say(&mut dictation, "", false); + field.say(&mut dictation, "", true); + assert_eq!(field.text, "keep me"); + // A recognizer that starts over after a pause keeps the phrase before it. + field.say(&mut dictation, "a long phrase that took thirty seconds", false); + field.say(&mut dictation, "Short phrase.", false); + field.say(&mut dictation, "Short phrase.", true); + assert_eq!(field.text, "keep me a long phrase that took thirty seconds Short phrase."); + + // A final that stops short keeps the words it left out. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "hello world and more", false); + field.say(&mut dictation, "Hello world.", true); + assert_eq!(field.text, "Hello world. and more"); + + // Reformatting a number as it's spoken is a revision, not a restart. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + for partial in ["Three", "300", "300 and", "340", "347 apples"] { + field.say(&mut dictation, partial, false); + } + field.say(&mut dictation, "347 apples.", true); + assert_eq!(field.text, "347 apples."); + } + #[test] fn words_beyond_only_returns_what_was_not_already_shown() { assert_eq!(words_beyond("hello world", "hello world and more"), "and more"); From 494ccc48abc6b1e7532b6e70e1877a712604b5a4 Mon Sep 17 00:00:00 2001 From: Kevin Boos Date: Fri, 11 Sep 2026 15:50:52 -0700 Subject: [PATCH 3/4] robius-speech: Android: commit any words already shown before an error this makes sure we don't lose any word chunks that were already shown in the text input, even if an error happens or something else occurs that causes a final commit to never occur. --- .../java/dev/robius/speech/NativeSpeech.java | 23 ++++++++++++++++++- crates/speech/tests/android_retry_test.py | 22 ++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/crates/speech/java/dev/robius/speech/NativeSpeech.java b/crates/speech/java/dev/robius/speech/NativeSpeech.java index 906fd35..876492c 100644 --- a/crates/speech/java/dev/robius/speech/NativeSpeech.java +++ b/crates/speech/java/dev/robius/speech/NativeSpeech.java @@ -15,6 +15,8 @@ import android.speech.SpeechRecognizer; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; +import java.util.Locale; /** * One dictation session, driving the system SpeechRecognizer. @@ -213,6 +215,22 @@ private String transcript(Bundle results) { return values == null || values.isEmpty() || values.get(0) == null ? "" : values.get(0); } + // Whether `text` is `previous` with words cut off the end, comparing words loosely. + private static boolean isShortenedRevision(String previous, String text) { + List old = looseWords(previous), cut = looseWords(text); + return !cut.isEmpty() && cut.size() < old.size() && old.subList(0, cut.size()).equals(cut); + } + + // Words ignoring case and surrounding punctuation, as the Rust side compares them. + private static List looseWords(String text) { + List words = new ArrayList<>(); + for (String word : text.trim().split("\\s+")) { + String loose = word.replaceAll("^[^\\p{L}\\p{N}]+|[^\\p{L}\\p{N}]+$", "").toLowerCase(Locale.ROOT); + if (!loose.isEmpty()) words.add(loose); + } + return words; + } + private void commitPending() { if (!pending.isEmpty()) { event(id, 2, pending, 0); @@ -261,6 +279,8 @@ private void fail(String message) { private void fail(String message, int kind) { if (disposed) return; + // Keep the words already shown before reporting the error. + commitPending(); dispose(); event(id, kind, message, 0); } @@ -312,7 +332,8 @@ private void dispose() { awaitingResult = false; transientFailures = 0; String text = results == null ? "" : transcript(results); - if (!text.isEmpty()) pending = text; + // A final that only cuts words off the end of the last partial keeps them. + if (!text.isEmpty() && !isShortenedRevision(pending, text)) pending = text; commitPending(); nextUtterance(); } diff --git a/crates/speech/tests/android_retry_test.py b/crates/speech/tests/android_retry_test.py index 71d965a..aa43a97 100644 --- a/crates/speech/tests/android_retry_test.py +++ b/crates/speech/tests/android_retry_test.py @@ -656,6 +656,26 @@ NativeSpeech.stop(29, true); Handler.drain(); } + private static void anErrorKeepsTheWordsShown() { + start(30); + SpeechRecognizer.last.listener.onPartialResults(words("keep these words")); + SpeechRecognizer.last.listener.onError(SpeechRecognizer.ERROR_NETWORK); + check(events.indexOf("2:keep these words") >= 0 && events.get(events.size() - 1).startsWith("5:"), + "an error commits the words shown before reporting itself"); + Handler.drain(); + } + private static void aCutShortFinalKeepsTheWordsShown() { + start(31); + SpeechRecognizer.last.listener.onPartialResults(words("hello world and more")); + SpeechRecognizer.last.listener.onResults(words("Hello world.")); + check(events.contains("2:hello world and more"), "a final that cuts words off the end keeps them"); + Handler.runNext(); + SpeechRecognizer.last.listener.onPartialResults(words("I scream for")); + SpeechRecognizer.last.listener.onResults(words("Ice cream for you.")); + check(events.contains("2:Ice cream for you."), "a final that revises words still replaces the partial"); + NativeSpeech.stop(31, true); + Handler.drain(); + } private interface Case { void run(); } public static void main(String[] args) { System.load(args[0]); @@ -672,6 +692,8 @@ {"queued old removal cannot remove a new permission request", (Case) NativeSpeechRetryTest::restartWhileOldParentRemovalIsQueued}, {"configuration teardown stops the old pending session", (Case) NativeSpeechRetryTest::configurationTeardownStopsPendingSession}, {"saved permission state restores through the host classloader", (Case) NativeSpeechRetryTest::processRestoreContainsOnlyHostVisibleClasses}, + {"an error keeps the words already shown", (Case) NativeSpeechRetryTest::anErrorKeepsTheWordsShown}, + {"a cut-short final keeps the words shown", (Case) NativeSpeechRetryTest::aCutShortFinalKeepsTheWordsShown}, }; for (Object[] entry : cases) { System.out.println(" " + entry[0]); From 2101b98479fabacf5310e4e89fcad862ae81309a Mon Sep 17 00:00:00 2001 From: Kevin Boos Date: Fri, 11 Sep 2026 16:08:23 -0700 Subject: [PATCH 4/4] robius-speech: fix tests and swift compilation on macOS 14 --- .github/workflows/speech.yml | 3 +++ crates/speech/swift/NativeSpeech.swift | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/speech.yml b/.github/workflows/speech.yml index fed4311..cc91c40 100644 --- a/.github/workflows/speech.yml +++ b/.github/workflows/speech.yml @@ -1,7 +1,10 @@ name: Native speech on: + # Only on main: a branch pushed to this repo would otherwise also run every + # job a second time through its pull request. push: + branches: [main] paths: - crates/speech/** - Cargo.toml diff --git a/crates/speech/swift/NativeSpeech.swift b/crates/speech/swift/NativeSpeech.swift index c84f4fa..0d22237 100644 --- a/crates/speech/swift/NativeSpeech.swift +++ b/crates/speech/swift/NativeSpeech.swift @@ -98,7 +98,8 @@ struct SegmentTracker { return out + commit(text) } open = text - return out + [(1, text)] + out.append((1, text)) + return out } /// Commits the words shown but not yet committed, e.g. when a task ends without a final.