More robius-speech improvements (mostly to avoid losing word chunks)

.github/workflows/speech.yml

@@ -1,7 +1,10 @@
 name: Native speech
 
 on:
+  # Only on main: a branch pushed to this repo would otherwise also run every
+  # job a second time through its pull request.
   push:
+    branches: [main]
     paths:
       - crates/speech/**
       - Cargo.toml

crates/speech/java/dev/robius/speech/NativeSpeech.java

@@ -15,6 +15,8 @@
 import android.speech.SpeechRecognizer;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
 
 /**
  * One dictation session, driving the system SpeechRecognizer.
@@ -213,6 +215,22 @@ private String transcript(Bundle results) {
         return values == null || values.isEmpty() || values.get(0) == null ? "" : values.get(0);
     }
 
+    // Whether `text` is `previous` with words cut off the end, comparing words loosely.
+    private static boolean isShortenedRevision(String previous, String text) {
+        List<String> old = looseWords(previous), cut = looseWords(text);
+        return !cut.isEmpty() && cut.size() < old.size() && old.subList(0, cut.size()).equals(cut);
+    }
+
+    // Words ignoring case and surrounding punctuation, as the Rust side compares them.
+    private static List<String> looseWords(String text) {
+        List<String> words = new ArrayList<>();
+        for (String word : text.trim().split("\\s+")) {
+            String loose = word.replaceAll("^[^\\p{L}\\p{N}]+|[^\\p{L}\\p{N}]+$", "").toLowerCase(Locale.ROOT);
+            if (!loose.isEmpty()) words.add(loose);
+        }
+        return words;
+    }
+
     private void commitPending() {
         if (!pending.isEmpty()) {
             event(id, 2, pending, 0);
@@ -261,6 +279,8 @@ private void fail(String message) {
 
     private void fail(String message, int kind) {
         if (disposed) return;
+        // Keep the words already shown before reporting the error.
+        commitPending();
         dispose();
         event(id, kind, message, 0);
     }
@@ -312,7 +332,8 @@ private void dispose() {
         awaitingResult = false;
         transientFailures = 0;
         String text = results == null ? "" : transcript(results);
-        if (!text.isEmpty()) pending = text;
+        // A final that only cuts words off the end of the last partial keeps them.
+        if (!text.isEmpty() && !isShortenedRevision(pending, text)) pending = text;
         commitPending();
         nextUtterance();
     }

crates/speech/src/dictation.rs

@@ -73,10 +73,21 @@ impl Dictation {
     /// a final commits it. Returns the edit that brings the field up to date, if
     /// there is one; apply it and call [`applied`](Self::applied).
     pub fn transcript(&mut self, text: &str, is_final: bool) -> Option<Replacement> {
-        self.pending = text.trim().to_owned();
+        // Never lose words the recognizer already gave us: an empty transcript changes nothing,
+        // one that starts over commits what it replaces, and one that stops short keeps the rest.
+        let text = text.trim();
+        if !text.is_empty() {
+            let previous = std::mem::take(&mut self.pending);
+            self.pending = text.to_owned();
+            if starts_over(&previous, text) {
+                append_words(&mut self.committed, &previous);
+            } else {
+                append_words(&mut self.pending, &words_beyond(text, &previous));
+            }
+        }
         if is_final {
-            append_words(&mut self.committed, &self.pending);
-            self.pending.clear();
+            let pending = std::mem::take(&mut self.pending);
+            append_words(&mut self.committed, &pending);
         }
         if let Some(awaiting_final) = self.interrupted.as_mut() {
             if is_final {
@@ -193,6 +204,23 @@ fn floor_char_boundary(text: &str, index: usize) -> usize {
     index
 }
 
+/// Whether `new` starts over rather than revising `old`, like a recognizer that restarted after
+/// a pause: it drops most words of a long `old`, rather than reformatting a few ("300 and" to "340").
+fn starts_over(old: &str, new: &str) -> bool {
+    let (old, new) = (loose_words(old), loose_words(new));
+    let kept = old.iter().filter(|word| new.contains(word)).count();
+    old.len() >= 6 && !new.is_empty() && new.len() * 2 < old.len() && kept * 4 < old.len()
+}
+
+fn loose_words(text: &str) -> Vec<String> {
+    text.split_whitespace().map(loose).filter(|word| !word.is_empty()).collect()
+}
+
+/// A word compared loosely, because a recognizer routinely re-cases or re-punctuates.
+fn loose(word: &str) -> String {
+    word.trim_matches(|c: char| !c.is_alphanumeric()).to_lowercase()
+}
+
 /// The part of `spoken` that goes beyond what the user can already see in
 /// `shown`. Words are compared loosely, because a recognizer routinely re-cases
 /// or re-punctuates what it already sent. Returns nothing when the two share no
@@ -201,7 +229,6 @@ fn words_beyond(shown: &str, spoken: &str) -> String {
     if shown.trim().is_empty() {
         return spoken.trim().to_owned();
     }
-    let loose = |word: &str| word.trim_matches(|c: char| !c.is_alphanumeric()).to_lowercase();
     let shown_words: Vec<_> = shown.split_whitespace().map(loose).collect();
     let mut matched = 0;
     let mut rest = spoken.trim();
@@ -556,6 +583,38 @@ mod tests {
         assert_eq!(field.text, "éX bc");
     }
 
+    #[test]
+    fn words_already_shown_are_never_removed() {
+        // An empty transcript, partial or final, keeps what is shown.
+        let mut dictation = Dictation::new("", 0..0);
+        let mut field = Field::new("", 0);
+        field.say(&mut dictation, "keep me", false);
+        field.say(&mut dictation, "", false);
+        field.say(&mut dictation, "", true);
+        assert_eq!(field.text, "keep me");
+        // A recognizer that starts over after a pause keeps the phrase before it.
+        field.say(&mut dictation, "a long phrase that took thirty seconds", false);
+        field.say(&mut dictation, "Short phrase.", false);
+        field.say(&mut dictation, "Short phrase.", true);
+        assert_eq!(field.text, "keep me a long phrase that took thirty seconds Short phrase.");
+
+        // A final that stops short keeps the words it left out.
+        let mut dictation = Dictation::new("", 0..0);
+        let mut field = Field::new("", 0);
+        field.say(&mut dictation, "hello world and more", false);
+        field.say(&mut dictation, "Hello world.", true);
+        assert_eq!(field.text, "Hello world. and more");
+
+        // Reformatting a number as it's spoken is a revision, not a restart.
+        let mut dictation = Dictation::new("", 0..0);
+        let mut field = Field::new("", 0);
+        for partial in ["Three", "300", "300 and", "340", "347 apples"] {
+            field.say(&mut dictation, partial, false);
+        }
+        field.say(&mut dictation, "347 apples.", true);
+        assert_eq!(field.text, "347 apples.");
+    }
+
     #[test]
     fn words_beyond_only_returns_what_was_not_already_shown() {
         assert_eq!(words_beyond("hello world", "hello world and more"), "and more");

crates/speech/swift/NativeSpeech.swift

@@ -54,6 +54,109 @@ struct SpeechRetryPolicy {
     }
 }
 
+/// Turns one recognition task's results into transcripts that never drop words already sent.
+///
+/// After a pause, on-device recognition closes a segment with one result that has metadata,
+/// then restarts `bestTranscription` from the new words only, without ever setting `isFinal`.
+/// Its final result after `endAudio` can hold only the last segment, or nothing at all.
+struct SegmentTracker {
+    /// Words sent as a partial but not yet committed.
+    private(set) var open = ""
+    /// The last text committed, and the whole transcription at that point.
+    private var committed = ""
+    private var closedTranscription = ""
+    /// Whether this task restarts its transcription after each segment, rather than growing it.
+    private var restarts = false
+
+    /// Returns the transcripts to send (kind 1 partial, kind 2 final) for one result.
+    mutating func result(_ transcription: String, isFinal: Bool, closesSegment: Bool) -> [(Int32, String)] {
+        var text = transcription.trimmingCharacters(in: .whitespacesAndNewlines)
+        // A transcription that grows instead of restarting still holds what was committed.
+        if !closedTranscription.isEmpty && !restarts {
+            if let rest = looseRemainder(of: text, after: closedTranscription) {
+                text = rest
+            } else if !text.isEmpty {
+                restarts = true
+            }
+        }
+        var out: [(Int32, String)] = []
+        if isFinal {
+            // An empty or cut-short final keeps what was shown, and a repeat of the
+            // segment just committed isn't committed again.
+            if text.isEmpty || (open.isEmpty && looselyEqual(text, committed)) || isShortenedRevision(of: open, text) {
+                return flush()
+            }
+            if startsOver(from: open, to: text) { out += flush() }
+            open = ""
+            return out + commit(text)
+        }
+        guard !text.isEmpty, !(closesSegment && open.isEmpty && looselyEqual(text, committed)) else { return [] }
+        if startsOver(from: open, to: text) { out += flush() }
+        if closesSegment {
+            open = ""
+            closedTranscription = transcription
+            return out + commit(text)
+        }
+        open = text
+        out.append((1, text))
+        return out
+    }
+
+    /// Commits the words shown but not yet committed, e.g. when a task ends without a final.
+    mutating func flush() -> [(Int32, String)] {
+        defer { open = "" }
+        return commit(open)
+    }
+
+    private mutating func commit(_ text: String) -> [(Int32, String)] {
+        guard !text.isEmpty else { return [] }
+        committed = text
+        return [(2, text)]
+    }
+}
+
+/// Words compared loosely, ignoring case and punctuation, as `dictation.rs` does.
+func looseWords(_ text: String) -> [String] {
+    text.split(whereSeparator: { $0.isWhitespace })
+        .map { $0.trimmingCharacters(in: CharacterSet.alphanumerics.inverted).lowercased() }
+        .filter { !$0.isEmpty }
+}
+
+func looselyEqual(_ a: String, _ b: String) -> Bool {
+    looseWords(a) == looseWords(b)
+}
+
+/// Whether `new` starts over rather than revising `old`, as after a pause without metadata:
+/// it drops most words of a long `old`, rather than reformatting a few (e.g. "300 and" to "340").
+func startsOver(from old: String, to new: String) -> Bool {
+    let (old, new) = (looseWords(old), looseWords(new))
+    let kept = old.filter(new.contains).count
+    return old.count >= 6 && !new.isEmpty && new.count * 2 < old.count && kept * 4 < old.count
+}
+
+/// Whether `new` is `old` with words cut off the end.
+func isShortenedRevision(of old: String, _ new: String) -> Bool {
+    let (old, new) = (looseWords(old), looseWords(new))
+    return !new.isEmpty && new.count < old.count && Array(old.prefix(new.count)) == new
+}
+
+/// The rest of `text` after its first words, if they loosely match all of `prefix`'s words.
+func looseRemainder(of text: String, after prefix: String) -> String? {
+    var expected = looseWords(prefix)[...]
+    var rest = Substring(text)
+    while let word = expected.first {
+        rest = rest.drop(while: { $0.isWhitespace })
+        guard !rest.isEmpty else { return nil }
+        let token = rest.prefix(while: { !$0.isWhitespace })
+        rest = rest.dropFirst(token.count)
+        let loose = looseWords(String(token))
+        if loose.isEmpty { continue }
+        guard loose == [word] else { return nil }
+        expected = expected.dropFirst()
+    }
+    return rest.trimmingCharacters(in: .whitespacesAndNewlines)
+}
+
 // The engine keeps recording while a recognizer finalizes an utterance. Only
 // that short interval needs copies; otherwise buffers go straight to the request.
 // No engine operations or UI callbacks run while this lock is held.
@@ -165,7 +268,7 @@ final class NativeSpeech {
     var finished = false
     var tapInstalled = false
     var started = false
-    var lastPartial = ""
+    var segments = SegmentTracker()
     var generation: UInt64 = 0
     var retryPolicy = SpeechRetryPolicy()
     var endingUtterance = false
@@ -301,7 +404,7 @@ final class NativeSpeech {
         }
         if #available(macOS 13.0, iOS 16.0, *) { request.addsPunctuation = true }
         self.request = request
-        lastPartial = ""
+        segments = SegmentTracker()
         let capture = self.capture
         capture.prepareRequest(utterance)
         task = recognizer.recognitionTask(with: request) { [weak self] result, error in
@@ -313,11 +416,16 @@ final class NativeSpeech {
             DispatchQueue.main.async {
                 guard let self = self, !self.finished, self.generation == utterance else { return }
                 if let result = result {
-                    self.lastPartial = result.bestTranscription.formattedString
-                    if !self.lastPartial.isEmpty { self.retryPolicy.madeProgress() }
-                    self.send(result.isFinal ? 2 : 1, self.lastPartial)
+                    let text = result.bestTranscription.formattedString
+                    if !text.isEmpty { self.retryPolicy.madeProgress() }
+                    var closesSegment = false
+                    if #available(iOS 14.0, *) {
+                        closesSegment = !result.isFinal && result.speechRecognitionMetadata != nil
+                    }
+                    for (kind, text) in self.segments.result(text, isFinal: result.isFinal, closesSegment: closesSegment) {
+                        self.send(kind, text)
+                    }
                     if result.isFinal {
-                        self.lastPartial = ""
                         if self.stopping { self.finishStoppedUtterance() }
                         else { self.restartUtterance(after: 0.15) }
                         return
@@ -502,7 +610,7 @@ final class NativeSpeech {
     }
 
     func commitPartial() {
-        if !lastPartial.isEmpty { send(2, lastPartial); lastPartial = "" }
+        for (kind, text) in segments.flush() { send(kind, text) }
     }
 
     func finish() {