diff --git a/crates/cc-app/examples/fix_eval.rs b/crates/cc-app/examples/fix_eval.rs new file mode 100644 index 0000000..c4471f0 --- /dev/null +++ b/crates/cc-app/examples/fix_eval.rs @@ -0,0 +1,282 @@ +//! Measures what auto-fix does with typos in running text: it puts one keyboard slip into a +//! word of each sentence, runs the word-boundary decision the app makes (with and without +//! the language model choosing between candidates), and counts right fixes, wrong fixes +//! and words left alone. +//! +//! cargo run --release -p cc-app --example fix_eval -- http://127.0.0.1:8090 +//! +//! The server is a llama-server with the completion model loaded. Plain prose works as +//! input; Harper's test texts (harper-core/tests/text/*.md) are what the thresholds were +//! tuned on. + +#[allow(dead_code)] +#[path = "../src/fixgate.rs"] +mod fixgate; + +use cc_correct::{AutocorrectList, Checker, PersonalDictionary, Severity}; +use cc_engine::CompletionClient; +use std::collections::BTreeMap; +use std::path::Path; + +const LEVEL: Severity = Severity::S2; +const CONTEXT_CHARS: usize = 300; +const NEXT_TOKENS: u32 = 40; +const SEED: u64 = 0x5eed_cafe; + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } +} + +const ROWS: [(&str, f32); 3] = [("qwertyuiop", 0.0), ("asdfghjkl", 0.25), ("zxcvbnm", 0.75)]; + +/// Keys next to `c` on a QWERTY keyboard, as cc-correct's typing model counts them. +fn neighbours(c: char) -> Vec { + let pos = |c: char| { + ROWS.iter().enumerate().find_map(|(y, (row, offset))| row.find(c).map(|x| (x as f32 + offset, y as f32))) + }; + let Some((x, y)) = pos(c) else { return Vec::new() }; + ROWS.iter() + .flat_map(|(row, _)| row.chars()) + .filter(|&k| k != c && pos(k).is_some_and(|(kx, ky)| (kx - x).powi(2) + (ky - y).powi(2) <= 2.0)) + .collect() +} + +/// One slip of the given kind in `word`, or None when it does not apply. +fn slip(word: &str, kind: &str, rng: &mut Rng) -> Option { + let mut w: Vec = word.chars().collect(); + let n = w.len(); + match kind { + "neighbour" => { + let i = rng.below(n); + let near = neighbours(w[i]); + if near.is_empty() { + return None; + } + w[i] = near[rng.below(near.len())]; + } + "swap" => { + let i = rng.below(n - 1); + if w[i] == w[i + 1] { + return None; + } + w.swap(i, i + 1); + } + "drop" => { + w.remove(rng.below(n)); + } + "extra" => { + let i = rng.below(n); + let near = neighbours(w[i]); + if near.is_empty() { + return None; + } + w.insert(i + rng.below(2), near[rng.below(near.len())]); + } + "double" => { + let i = rng.below(n); + w.insert(i, w[i]); + } + _ => unreachable!(), + } + Some(w.into_iter().collect()) +} + +/// Sentences of plain prose, each with the text before it in its paragraph. +fn sentences(text: &str) -> Vec<(String, String)> { + let mut out = Vec::new(); + for para in text.split("\n\n") { + let para: String = para + .lines() + .filter(|l| !l.trim_start().starts_with(['#', '|', '>', '-', '*', '`'])) + .collect::>() + .join(" "); + let para = para.replace(['_', '*', '[', ']', '(', ')'], ""); + let mut before = String::new(); + for s in para.split_inclusive(['.', '!', '?']) { + let s = s.trim(); + let words = s.split_whitespace().count(); + if (6..=40).contains(&words) && s.chars().all(|c| c.is_ascii()) { + out.push((before.clone(), s.to_string())); + } + before.push_str(s); + before.push(' '); + } + } + out +} + +#[derive(Default)] +struct Tally { + right: usize, + wrong: usize, + left: usize, +} + +fn main() -> anyhow::Result<()> { + let args: Vec = std::env::args().skip(1).collect(); + let (server, files) = args.split_first().expect("usage: fix_eval "); + let checker = Checker::new(); + let dict = PersonalDictionary::default(); + let auto = AutocorrectList::load(Path::new("nonexistent-autocorrect.txt")); + let client = CompletionClient::new(server); + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build()?; + let mut rng = Rng(SEED); + let kinds = ["neighbour", "swap", "drop", "extra", "double"]; + + let mut tables: BTreeMap> = BTreeMap::new(); + let mut wrong_lines = Vec::new(); + let mut samples = 0; + for file in files { + let text = std::fs::read_to_string(file)?; + for (para_before, sentence) in sentences(&text) { + let words: Vec<&str> = sentence.split(' ').collect(); + // One target per sentence: a lowercase dictionary word of four or more letters, + // not the first word. + let targets: Vec = (1..words.len()) + .filter(|&i| { + let w = words[i]; + w.len() >= 4 && w.chars().all(|c| c.is_ascii_lowercase()) && checker.is_word(w, &dict) + }) + .collect(); + if targets.is_empty() { + continue; + } + let i = targets[rng.below(targets.len())]; + let truth = words[i]; + let kind = kinds[rng.below(kinds.len())]; + let Some(typo) = slip(truth, kind, &mut rng) else { continue }; + if typo.len() < 3 || checker.is_word(&typo, &dict) { + continue; + } + samples += 1; + let mut body = format!("{para_before}{}", words[..i].join(" ")); + if i > 0 { + body.push(' '); + } + body.push_str(&typo); + let tail_start = body.len().saturating_sub(CONTEXT_CHARS); + let tail_start = (tail_start..body.len()).find(|&k| body.is_char_boundary(k)).unwrap_or(0); + let body = &body[tail_start..]; + let start = body.rfind(['.', '!', '?', '\n']).map_or(0, |k| k + 1); + let sent = body[start..].trim_start(); + let n_chars = sent.chars().count(); + let before = body[..body.len() - typo.len()].trim_end_matches(' '); + + let listed = auto.lookup(&typo); + let found = checker.check(sent, &dict); + let at_end = |c: &&cc_correct::Candidate| c.end == n_chars && sent.ends_with(&c.original); + let eligible = found + .iter() + .filter(at_end) + .find(|c| fixgate::eligible(&c.severity, &c.rule, &c.original, &c.replacement, &LEVEL)) + .filter(|c| fixgate::reject(&c.original, &c.replacement, true, false, |t| checker.is_word(t, &dict)).is_none()); + let guess = found.iter().filter(at_end).find(|c| c.rule == "spell" && c.severity == Severity::S2); + let needs_model = listed.is_none() && (eligible.is_some_and(|c| c.rule == "spell") || (eligible.is_none() && guess.is_some())); + let top = if needs_model && !before.trim().is_empty() { + rt.block_on(client.next_tokens(before, NEXT_TOKENS)).unwrap_or_default() + } else { + Vec::new() + }; + let is_word = |w: &str| checker.is_word(w, &dict); + let options = checker.typo_options(&typo, &dict); + + // 0.1.3: the rules, with the model's veto on guessed S0 fixes. + let base_fix: Option = if let Some(fix) = listed.clone() { + Some(fix) + } else if let Some(c) = eligible { + let guessed = c.rule == "spell" && c.severity == Severity::S0 && !cc_correct::is_doubled_letter_typo(&c.original, &c.replacement); + if guessed && (top.is_empty() || fixgate::model_prefers(&c.original, &c.replacement, &top, is_word).is_some()) { + None + } else { + Some(c.replacement.clone()) + } + } else { + None + }; + // New: the model picks between the typo's options where it clearly expects one. + let decide = |picked: Option, doubled_too: bool| -> Option { + if let Some(fix) = listed.clone() { + Some(fix) + } else if let Some(c) = eligible { + let guessed = c.rule == "spell" && c.severity == Severity::S0 && !cc_correct::is_doubled_letter_typo(&c.original, &c.replacement); + if !guessed { + // A doubled-key fix is never held back, but the model may pick another spelling. + match picked { + Some(p) if doubled_too && c.rule == "spell" => Some(p), + _ => Some(c.replacement.clone()), + } + } else if top.is_empty() { + None + } else if picked.is_some() { + picked + } else if fixgate::model_prefers(&c.original, &c.replacement, &top, is_word).is_some() { + None + } else { + Some(c.replacement.clone()) + } + } else if guess.is_some() { + picked + } else { + None + } + }; + let pick_fix = decide(fixgate::model_pick(&typo, &options, &top), false); + let mut fixes = vec![ + ("0.1.3 (rules + veto)".to_string(), base_fix.clone()), + ("model picks".to_string(), pick_fix.clone()), + ("model picks, doubled keys too".to_string(), decide(fixgate::model_pick(&typo, &options, &top), true)), + ]; + for weight in [1.0, 2.0, 3.0] { + for typed in [true, false] { + let name = format!("weight {weight}, typed word {}", if typed { "rivals" } else { "ignored" }); + fixes.push((name, decide(fixgate::model_pick_tuned(&typo, &options, &top, weight, typed), false))); + } + } + for (name, fix) in &fixes { + let table = tables.entry(name.clone()).or_default(); + for key in [kind.to_string(), "all".to_string()] { + let t = table.entry(key).or_default(); + match fix { + None => t.left += 1, + Some(f) if f.eq_ignore_ascii_case(truth) => t.right += 1, + Some(_) => t.wrong += 1, + } + } + } + if let Some(f) = &pick_fix { + if !f.eq_ignore_ascii_case(truth) { + let ctx: String = before.chars().rev().take(40).collect::>().into_iter().rev().collect(); + wrong_lines.push(format!( + " {kind:9} …{ctx} [{typo}] -> {f:?} (meant {truth:?}; 0.1.3: {base_fix:?})" + )); + } + } + } + } + println!("{samples} typos"); + for (name, table) in &tables { + let all = &table["all"]; + println!("{name:36} right {:4} wrong {:4} left {:4}", all.right, all.wrong, all.left); + } + for name in ["0.1.3 (rules + veto)", "model picks"] { + println!("{name}:"); + for (kind, t) in &tables[name] { + println!(" {kind:9} right {:4} wrong {:4} left {:4}", t.right, t.wrong, t.left); + } + } + println!("wrong fixes with model picks:"); + for line in wrong_lines { + println!("{line}"); + } + Ok(()) +} diff --git a/crates/cc-app/src/fixgate.rs b/crates/cc-app/src/fixgate.rs index 315e418..447fe9f 100644 --- a/crates/cc-app/src/fixgate.rs +++ b/crates/cc-app/src/fixgate.rs @@ -149,6 +149,77 @@ pub fn model_prefers(typo: &str, fix: &str, top: &[(String, f32)], is_word: impl (lp - fix_lp >= 4f32.ln()).then_some(word) } +/// How likely the word [`model_pick`] returns must be as the next token, at least. +const PICK_FLOOR: f32 = -4.6; // ln 0.01 +/// Nats of model likelihood that one unit of typing-error cost is worth in [`model_pick`]. +const COST_WEIGHT: f32 = 2.0; + +/// The word the language model clearly expects where `typo` was typed, among the words the +/// typo may stand for (`options`: word and typing-error cost, from `Checker::typo_options`), +/// given the model's likeliest next tokens (`top`) after the text before the typo. Each +/// option scores how likely the model finds it next, as a whole word token (` month`), less +/// what its typo costs, so a likely word the keys do not explain loses. The best must be +/// in the list, and score four times higher than anything else that could be meant there: +/// another option, a token that starts the typo or an option (` doc` may begin what was +/// typed), or the word as typed, which is as likely as the last token in the list for all +/// the model says. `this morth` gives `month`. After `for the`, `calory` gives nothing, +/// since the model expects neither `calorie` nor `color` there. +pub fn model_pick(typo: &str, options: &[(String, f32)], top: &[(String, f32)]) -> Option { + model_pick_tuned(typo, options, top, COST_WEIGHT, true) +} + +/// [`model_pick`] with its cost weight, and with or without the word as typed as a rival +/// (`typed_rival`), for tuning (examples/fix_eval.rs). +pub fn model_pick_tuned( + typo: &str, + options: &[(String, f32)], + top: &[(String, f32)], + weight: f32, + typed_rival: bool, +) -> Option { + let typo = typo.to_lowercase(); + let floor = top.iter().map(|(_, lp)| *lp).reduce(f32::min)?; + // Whole-word tokens; one without a leading space would continue the word before the typo. + let words: Vec<(String, f32)> = top + .iter() + .filter_map(|(token, lp)| { + let word = token.to_lowercase().strip_prefix(' ')?.to_string(); + (!word.is_empty() && word.chars().all(char::is_alphabetic)).then_some((word, *lp)) + }) + .collect(); + let listed = |w: &str| words.iter().find(|(t, _)| t == w).map(|(_, lp)| *lp); + let options: Vec<(String, f32)> = options.iter().map(|(o, cost)| (o.to_lowercase(), *cost)).collect(); + // (option, its token's log probability if listed, score); an option not in the list is + // at most as likely as the floor. + let scored: Vec<(&str, Option, f32)> = options + .iter() + .map(|(o, cost)| { + let lp = listed(o); + (o.as_str(), lp, lp.unwrap_or(floor) - weight * cost) + }) + .collect(); + let (best_at, &(word, lp, score)) = scored + .iter() + .enumerate() + .filter(|(_, (_, lp, _))| lp.is_some()) + .max_by(|a, b| a.1 .2.total_cmp(&b.1 .2))?; + let mut rival = if typed_rival { floor } else { f32::NEG_INFINITY }; + for (at, (_, _, other)) in scored.iter().enumerate() { + if at != best_at { + rival = rival.max(*other); + } + } + for (token, token_lp) in &words { + let is_option = options.iter().any(|(o, _)| o == token); + let starts = typo.starts_with(token.as_str()) || options.iter().any(|(o, _)| o.starts_with(token.as_str())); + if !is_option && token.chars().count() >= 2 && starts { + rival = rival.max(*token_lp); + } + } + let lp = lp?; + (lp >= PICK_FLOOR && score - rival >= 4f32.ln()).then(|| word.to_string()) +} + /// The end of `s` with letters written as `a` or `A` and digits as `0`, for the log: it /// shows a joined or doubled word without showing what was typed. pub fn shape(s: &str) -> String { @@ -449,6 +520,31 @@ mod tests { ); } + #[test] + fn model_pick_takes_the_option_the_model_clearly_expects() { + let top = |pairs: &[(&str, f32)]| pairs.iter().map(|(t, lp)| (t.to_string(), *lp)).collect::>(); + let options = |words: &[(&str, f32)]| words.iter().map(|(w, c)| (w.to_string(), *c)).collect::>(); + let morth = options(&[("north", 0.3), ("moth", 0.6), ("month", 0.9), ("worth", 1.4)]); + let after_this = top(&[(" month", -1.2), (" week", -1.9), (" year", -2.0), (" north", -5.0), (" time", -6.0)]); + assert_eq!(model_pick("morth", &morth, &after_this).as_deref(), Some("month")); + // Neither option is expected: nothing below the list can be ruled out. + let after_the = top(&[(" meeting", -1.4), (" next", -2.7), (" first", -2.8), (" time", -3.9)]); + assert_eq!(model_pick("calory", &options(&[("calorie", 1.0)]), &after_the), None); + // Close to another option once the typing cost counts. + let close = top(&[(" month", -1.2), (" moth", -1.9), (" time", -6.0)]); + assert_eq!(model_pick("morth", &morth, &close), None); + // Likely, but the keys do not explain it well enough. + let costly = top(&[(" worth", -1.5), (" time", -4.5)]); + assert_eq!(model_pick("morth", &morth, &costly), None); + // A token the typo starts with may be the start of the word as typed. + let typed = top(&[(" document", -2.0), (" doc", -2.5), (" time", -6.0)]); + assert_eq!(model_pick("documnet", &options(&[("document", 0.4)]), &typed), None); + // Too unlikely on its own, and a token that continues the word before is not a word. + assert_eq!(model_pick("morth", &morth, &top(&[(" month", -5.0), (" a", -9.0)])), None); + assert_eq!(model_pick("morth", &morth, &top(&[("month", -0.5), (" a", -6.0)])), None); + assert_eq!(model_pick("morth", &morth, &[]), None); + } + #[test] fn shape_hides_letters_and_digits_only() { assert_eq!(shape("Previous the 2nd, ok."), "Aaaaaaaa aaa 0aa, aa."); diff --git a/crates/cc-app/src/main.rs b/crates/cc-app/src/main.rs index eb37267..245281a 100644 --- a/crates/cc-app/src/main.rs +++ b/crates/cc-app/src/main.rs @@ -7,7 +7,7 @@ //! happen only on the orchestrator thread (which calls `cc_context::init_thread`). use std::path::PathBuf; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use std::collections::{HashMap, HashSet}; @@ -53,6 +53,9 @@ const FIX_RECHECK_AFTER: Duration = Duration::from_millis(200); const MODEL_CHECK_TIMEOUT: Duration = Duration::from_millis(300); /// Next-token candidates asked for in that check. const MODEL_CHECK_TOKENS: u32 = 40; +/// Rule of a spelling fix the language model chose (`App::settle`), so that its undos are +/// counted, and it is demoted, apart from the typing model's fixes. +const CONTEXT_RULE: &str = "spell_context"; const DEFAULT_MODEL: &str = "gemma-3-1b-pt"; /// Terminals: UIA reports the whole screen buffer, not an edit field, so the shadow buffer /// is the only context and editability is assumed. @@ -65,14 +68,25 @@ enum Msg { Done { gen: u64, result: anyhow::Result }, } -struct Req { - gen: u64, - prompt: String, - cancel: CancellationToken, +enum Req { + /// Stream a completion of `prompt` back as `Msg::Token`s and a `Msg::Done`. + Complete { gen: u64, prompt: String, cancel: CancellationToken }, + /// The likeliest next tokens after `prompt`, for a fix to check against; the engine + /// fills in `answer` unless `cancel` fires first. + NextTokens { prompt: String, answer: Answer, cancel: CancellationToken }, } -/// Engine runtime: receives prompts, streams tokens back as `Msg`. -fn spawn_engine(client: CompletionClient, out: Sender) -> tokio::sync::mpsc::UnboundedSender { +/// The model's likeliest next tokens after a prompt, once the engine has them, or why not. +type Answer = Arc, String>>>>; + +/// Engine runtime: receives prompts, streams tokens back as `Msg`. Next-token questions go +/// through `ask`, which keeps no idle connections: one llama-server has closed fails the +/// next request on it (os error 10053). +fn spawn_engine( + client: CompletionClient, + ask: CompletionClient, + out: Sender, +) -> tokio::sync::mpsc::UnboundedSender { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); std::thread::Builder::new() .name("cc-engine".into()) @@ -83,18 +97,37 @@ fn spawn_engine(client: CompletionClient, out: Sender) -> tokio::sync::mpsc .expect("tokio runtime"); rt.block_on(async move { while let Some(req) = rx.recv().await { - let client = client.clone(); - let out = out.clone(); - tokio::spawn(async move { - let gen = req.gen; - let out_tok = out.clone(); - let result = client - .complete_with_probs(&req.prompt, N_PREDICT, req.cancel, move |tok, logprob| { - let _ = out_tok.send(Msg::Token { gen, text: tok.to_string(), logprob }); - }) - .await; - let _ = out.send(Msg::Done { gen, result }); - }); + match req { + Req::Complete { gen, prompt, cancel } => { + let client = client.clone(); + let out = out.clone(); + tokio::spawn(async move { + let out_tok = out.clone(); + let result = client + .complete_with_probs(&prompt, N_PREDICT, cancel, move |tok, logprob| { + let _ = out_tok.send(Msg::Token { gen, text: tok.to_string(), logprob }); + }) + .await; + let _ = out.send(Msg::Done { gen, result }); + }); + } + Req::NextTokens { prompt, answer, cancel } => { + let ask = ask.clone(); + tokio::spawn(async move { + // A withdrawn question is dropped here, closing its connection. + let t0 = Instant::now(); + let top = tokio::select! { + _ = cancel.cancelled() => { + log::debug!("model question withdrawn after {} ms", t0.elapsed().as_millis()); + return; + } + top = ask.next_tokens(&prompt, MODEL_CHECK_TOKENS) => top.map_err(|e| format!("{e:#}")), + }; + log::debug!("model question answered in {} ms ({} chars)", t0.elapsed().as_millis(), prompt.len()); + *answer.lock().unwrap_or_else(|p| p.into_inner()) = Some(top); + }); + } + } } }); }) @@ -197,8 +230,10 @@ struct App { demoted: HashSet, /// `(typed, fix)` pairs, lowercase, the user undid (`Store::undone_pairs`); never typed again. undone: HashSet<(String, String)>, - /// For the language model's second opinion on a fix; completions go through `engine`. - client: CompletionClient, + /// The prompt for the text before the word being typed, the model's answer to it, and + /// the token that withdraws the question: asked when the word started, so a fix at its + /// end rarely waits. + ahead: Option<(String, Answer, CancellationToken)>, /// (event id, applied at, shadow length right after) of the last auto fix awaiting an outcome. pending_fix_event: Option<(i64, Instant, usize)>, pending_completion_event: Option, @@ -568,7 +603,7 @@ impl App { self.gen += 1; self.req_started = Some(Instant::now()); self.first_token_logged = false; - let _ = self.engine.send(Req { gen: self.gen, prompt: ctx, cancel: self.cancel.clone() }); + let _ = self.engine.send(Req::Complete { gen: self.gen, prompt: ctx, cancel: self.cancel.clone() }); } /// The model prompt for `ctx`: recently accepted sentences in this app give the base @@ -582,34 +617,96 @@ impl App { } } - /// Why the language model stops the fix of `typo` to `fix`, if it does: it clearly - /// expects another dictionary word close to `typo` after `before` (the text up to the - /// typo; `fixgate::model_prefers`), or it gave no answer in time. None lets the fix go - /// in; so does having no engine, or no text before the typo to ask about. - fn model_objection(&self, before: &str, typo: &str, fix: &str) -> Option { - self.sidecar.as_ref()?; - let ctx = before.trim_end_matches(' '); - if ctx.trim().is_empty() { - return None; + /// The model's answer to `prompt`: the one already asked for, or a new question. A + /// new question withdraws the last one, so that on a busy engine questions about words + /// already typed do not queue up in front of it. + fn ask_model(&mut self, prompt: String) -> Answer { + if let Some((asked, answer, cancel)) = &self.ahead { + if *asked == prompt { + return answer.clone(); + } + cancel.cancel(); + } + let answer: Answer = Arc::new(Mutex::new(None)); + let cancel = CancellationToken::new(); + let _ = self.engine.send(Req::NextTokens { prompt: prompt.clone(), answer: answer.clone(), cancel: cancel.clone() }); + self.ahead = Some((prompt, answer.clone(), cancel)); + answer + } + + /// A word was just started: ask the model now what may follow the text before it, so a + /// fix at the word's end finds the answer there. Only where a completion could go. + fn ask_ahead(&mut self) { + if self.sidecar.is_none() || self.auto_level.is_none() || !self.field_allowed() { + return; } + let text = self.shadow.text(); + let Some(last) = text.chars().last() else { return }; + let before = text[..text.len() - last.len_utf8()].trim_end_matches(' '); + if before.trim().is_empty() || privacy::sensitive(before).is_some() { + return; + } + let prompt = self.prompt_for(before.to_string()); + self.ask_model(prompt); + } + + /// The model's answer about `ctx` if it is already in: one asked for when the word + /// started. Never asks and never waits. + fn model_top_now(&self, ctx: &str) -> Option> { let prompt = self.prompt_for(ctx.to_string()); - let client = self.client.clone(); + let (asked, answer, _) = self.ahead.as_ref()?; + if *asked != prompt { + return None; + } + let top = answer.lock().unwrap_or_else(|p| p.into_inner()).clone()?; + top.ok() + } + + /// The model's likeliest next tokens after `ctx`, the text up to a typo. Err when the + /// engine gives no answer within `MODEL_CHECK_TIMEOUT`. + fn model_top(&mut self, ctx: &str) -> Result, String> { + let answer = self.ask_model(self.prompt_for(ctx.to_string())); let t0 = Instant::now(); - let asked = self.rt.block_on(async move { - tokio::time::timeout(MODEL_CHECK_TIMEOUT, client.next_tokens(&prompt, MODEL_CHECK_TOKENS)).await - }); - let ms = t0.elapsed().as_millis(); - let top = match asked { - Ok(Ok(top)) => top, - Ok(Err(e)) => { - log::debug!("model check failed after {ms} ms: {e:#}"); - return Some("the model gave no answer".into()); + loop { + if let Some(top) = answer.lock().unwrap_or_else(|p| p.into_inner()).clone() { + log::debug!("model check answered {} ms after the word", t0.elapsed().as_millis()); + return top.map_err(|e| { + log::debug!("model check failed: {e}"); + "the model gave no answer".to_string() + }); } - Err(_) => return Some(format!("the model gave no answer within {ms} ms")), - }; - log::debug!("model check answered in {ms} ms"); - fixgate::model_prefers(typo, fix, &top, |w| self.checker.is_word(w, &self.dict)) - .map(|word| format!("the model expects {word:?}")) + if t0.elapsed() >= MODEL_CHECK_TIMEOUT { + return Err(format!("the model gave no answer within {} ms", MODEL_CHECK_TIMEOUT.as_millis())); + } + std::thread::sleep(Duration::from_millis(2)); + } + } + + /// The word to type for the spelling fix `c` once the language model has read `before`, + /// the text up to the typo, or why nothing is typed. The model may pick another of the + /// typo's spellings than the typing model did (`fixgate::model_pick`); a `guess`, which + /// the rules could not settle, needs that pick. Without one, a rules fix goes in unless + /// the model clearly expects another word close to the typo (`fixgate::model_prefers`). + /// With no engine, or no text before the typo to ask about, a rules fix goes in as it is. + fn settle(&mut self, before: &str, c: &cc_correct::Candidate, guess: bool) -> Result { + let ctx = before.trim_end_matches(' '); + if self.sidecar.is_none() || ctx.trim().is_empty() { + return if guess { Err("there is no model to ask".into()) } else { Ok(c.replacement.clone()) }; + } + let top = self.model_top(ctx)?; + if !self.demoted.contains(CONTEXT_RULE) { + let options = self.checker.typo_options(&c.original, &self.dict); + if let Some(pick) = fixgate::model_pick(&c.original, &options, &top) { + return Ok(pick); + } + } + if guess { + return Err("the model expects none of its spellings".into()); + } + match fixgate::model_prefers(&c.original, &c.replacement, &top, |w| self.checker.is_word(w, &self.dict)) { + Some(word) => Err(format!("the model expects {word:?}")), + None => Ok(c.replacement.clone()), + } } fn show_ghost(&mut self) { @@ -721,6 +818,9 @@ impl App { // punctuation after it was already looked up when that punctuation was typed. let core = word.trim_start_matches(|c: char| !c.is_alphanumeric()); let listed = if core.ends_with(char::is_alphanumeric) { self.auto.lookup(core) } else { None }; + // A misspelling the rules leave open (S2) is a guess: the language model may choose + // one of its spellings (`settle`). + let mut guess = false; let candidate = match listed { Some(fix) => Some(cc_correct::Candidate { start: n_chars - core.chars().count(), @@ -731,22 +831,33 @@ impl App { severity: Severity::S0, confidence: 1.0, }), - None => self - .checker - .check(&sentence, &self.dict) - .into_iter() - .find(|c| { - c.end == n_chars + None => { + let mut found = self.checker.check(&sentence, &self.dict); + let at_end = |c: &cc_correct::Candidate| c.end == n_chars && sentence.ends_with(&c.original); + let ruled = found.iter().position(|c| { + at_end(c) && fixgate::eligible(&c.severity, &c.rule, &c.original, &c.replacement, &level) - && sentence.ends_with(&c.original) && !self.demoted.contains(&c.rule) - }), + }); + let open = || { + found.iter().position(|c| { + at_end(c) + && c.rule == "spell" + && c.severity == Severity::S2 + && self.sidecar.is_some() + && !self.demoted.contains(CONTEXT_RULE) + && !self.checker.typo_options(&c.original, &self.dict).is_empty() + }) + }; + let at = ruled.or_else(|| open().inspect(|_| guess = true)); + at.map(|i| found.swap_remove(i)) + } }; - let Some(c) = candidate else { + let Some(mut c) = candidate else { self.prev_boundary = Some((word, self.shadow.text().chars().count())); return; }; - if self.undone.contains(&(c.original.to_lowercase(), c.replacement.to_lowercase())) { + if !guess && self.undone.contains(&(c.original.to_lowercase(), c.replacement.to_lowercase())) { log::info!("fix {:?} -> {:?} [{}] not applied: undone before", c.original, c.replacement, c.rule); self.prev_boundary = Some((word, self.shadow.text().chars().count())); return; @@ -772,29 +883,62 @@ impl App { self.prev_boundary = None; return; } - let check_ms = t0.elapsed().as_millis() as u32; - log::info!( - "fix {:?} -> {:?} [{} {:?} {:.2}] checked in {} ms", - c.original, c.replacement, c.rule, c.severity, c.confidence, check_ms - ); self.prev_boundary = None; self.cancel_ghost(); - // A typing-model pick other than a doubled letter gets the language model's view of - // the word, with keys captured meanwhile so the field stays put. - if c.rule == "spell" && c.severity == Severity::S0 && !cc_correct::is_doubled_letter_typo(&c.original, &c.replacement) { + // A spelling the typing model guessed, other than doubled keys, gets the language + // model's view of the word first, with keys captured meanwhile so the field stays put. + let guessed = c.rule == "spell" && c.severity == Severity::S0 && !cc_correct::is_doubled_letter_typo(&c.original, &c.replacement); + if guess || guessed { if !self.swallow.begin_capture(self.applied) { log::info!("fix not applied: typed ahead of it"); return; } let text = self.shadow.text(); let before = text[..text.len() - boundary.len_utf8() - c.original.len()].to_string(); - if let Some(why) = self.model_objection(&before, &c.original, &c.replacement) { - log::info!("fix {:?} -> {:?} not applied: {why}", c.original, c.replacement); + match self.settle(&before, &c, guess) { + Ok(fix) => { + if guess || fix != c.replacement { + c.rule = CONTEXT_RULE.into(); + } + c.replacement = fix; + } + Err(why) => { + log::info!("fix {:?} -> {:?} [{}] not applied: {why}", c.original, c.replacement, c.rule); + self.end_capture(); + self.prev_boundary = Some((word, self.shadow.text().chars().count())); + return; + } + } + if self.undone.contains(&(c.original.to_lowercase(), c.replacement.to_lowercase())) { + log::info!("fix {:?} -> {:?} [{}] not applied: undone before", c.original, c.replacement, c.rule); self.end_capture(); self.prev_boundary = Some((word, self.shadow.text().chars().count())); return; } } + // Doubled keys are fixed without waiting for the model, but when its view of the + // word is already in, it may pick another spelling (`madd` after `It` is `made`). + if !guess && !guessed && c.rule == "spell" && !self.demoted.contains(CONTEXT_RULE) { + let text = self.shadow.text(); + let before = text[..text.len() - boundary.len_utf8() - c.original.len()].trim_end_matches(' ').to_string(); + let pick = self.model_top_now(&before).and_then(|top| { + fixgate::model_pick(&c.original, &self.checker.typo_options(&c.original, &self.dict), &top) + }); + if let Some(pick) = pick.filter(|p| *p != c.replacement) { + if self.undone.contains(&(c.original.to_lowercase(), pick.to_lowercase())) { + log::info!("fix {:?} -> {:?} [{CONTEXT_RULE}] not applied: undone before", c.original, pick); + self.prev_boundary = Some((word, self.shadow.text().chars().count())); + return; + } + c.rule = CONTEXT_RULE.into(); + c.replacement = pick; + } + } + let check_ms = t0.elapsed().as_millis() as u32; + log::info!( + "fix {:?} -> {:?} [{} {:?} {:.2}] checked in {} ms", + c.original, c.replacement, c.rule, c.severity, c.confidence, check_ms + ); let caret = cc_context::caret_rect(); let mut typed = c.replacement.clone(); typed.push(boundary); @@ -1152,6 +1296,12 @@ impl App { if self.last_fix.is_some() { return; } + } else if c.is_alphanumeric() { + let text = self.shadow.text(); + let prev = text[..text.len() - c.len_utf8()].chars().next_back(); + if prev.is_some_and(|p| p.is_whitespace() || fixgate::closes_word(p)) { + self.ask_ahead(); + } } if self.ghost_visible && self.ghost.starts_with(c) { self.ghost.remove(0); @@ -1648,10 +1798,7 @@ fn start() -> anyhow::Result<()> { let rt = tokio::runtime::Builder::new_current_thread().enable_all().build()?; let (tx, rx) = unbounded::(); - let engine = spawn_engine(CompletionClient::new(&sidecar_url()), tx.clone()); - // Its own client: a pooled connection is driven by the runtime that opened it, and this - // one runs only while the orchestrator waits on a model check. So it keeps none. - let client = CompletionClient::unpooled(&sidecar_url()); + let engine = spawn_engine(CompletionClient::new(&sidecar_url()), CompletionClient::unpooled(&sidecar_url()), tx.clone()); let overlay = Overlay::new().context("overlay")?; let swallow = Arc::new(SwallowState::default()); @@ -1732,7 +1879,7 @@ fn start() -> anyhow::Result<()> { paused_until: None, demoted, undone, - client, + ahead: None, pending_fix_event: None, pending_completion_event: None, completion_latency_ms: 0, diff --git a/crates/cc-correct/src/lib.rs b/crates/cc-correct/src/lib.rs index b01894a..363d5e2 100644 --- a/crates/cc-correct/src/lib.rs +++ b/crates/cc-correct/src/lib.rs @@ -407,6 +407,32 @@ impl Checker { self.cache.lock().unwrap_or_else(|p| p.into_inner()) } + /// Dictionary words `word` may be a typo of, one slip away (one edit, which may change the + /// first letter, or doubled keys), with their typing-error cost (lower is likelier), for + /// the language model to choose from once it has seen the sentence. Empty for a word not + /// to be guessed at whatever the sentence says: a known word (in any dialect or the + /// personal dictionary), one under four letters, one with a capital (a name or an + /// acronym), code, or an abbreviation (no vowel). Two slips are left out: they reach too + /// many words (`colud` is `could`, not `would`). + pub fn typo_options(&self, word: &str, dictionary: &PersonalDictionary) -> Vec<(String, f32)> { + let chars: Vec = word.chars().collect(); + if chars.len() < MIN_S0_LEN + || !chars.iter().all(|&c| c.is_alphabetic() || c == '\'') + || chars.iter().any(|c| c.is_uppercase()) + || self.is_word(word, dictionary) + || is_code_like(word) + { + return Vec::new(); + } + let lower: Vec = chars.iter().map(|&c| lc(c)).collect(); + self.scored_pool(word, &[]) + .into_iter() + .filter(|s| (s.distance == 1 || is_doubled_letter_typo(word, &s.text)) && !looks_coined(&lower, &s.text)) + .map(|s| (s.text, s.score)) + .take(TYPO_OPTIONS) + .collect() + } + fn rank_spelling(&self, original: &str, harper: &[String]) -> (String, Severity, f32) { let orig: Vec = original.chars().collect(); let lower: Vec = orig.iter().map(|&c| lc(c)).collect(); @@ -417,6 +443,20 @@ impl Checker { return (harper[0].clone(), Severity::S1, 0.9); } + let scored = self.scored_pool(original, harper); + let Some(best) = scored.first() else { + // Nothing within distance 2: fall back to Harper's first (distant) suggestion. + return (harper[0].clone(), Severity::S2, 0.5); + }; + self.decide_spelling(original, &orig, &lower, first_upper, best, scored.get(1)) + } + + /// Every dictionary word within `S0_MAX_DISTANCE` of `original`, cheapest typo first. + /// Harper's own suggestions (`harper`) head the pool, so their order breaks ties. + fn scored_pool(&self, original: &str, harper: &[String]) -> Vec { + let orig: Vec = original.chars().collect(); + let lower: Vec = orig.iter().map(|&c| lc(c)).collect(); + let first_upper = orig.first().is_some_and(|c| c.is_uppercase()); // Candidate pool: the suggestions Harper put on the lint (its order breaks ties), then // every dictionary word within edit distance 2 (Harper itself shows only the top 3). let mut pool: Vec = Vec::new(); @@ -439,12 +479,6 @@ impl Checker { } } - struct Scored { - text: String, - distance: usize, - score: f32, - order: usize, - } let mut scored: Vec = Vec::new(); for (order, text) in pool.iter().enumerate() { let cand: Vec = text.chars().map(lc).collect(); @@ -464,14 +498,21 @@ impl Checker { }); } scored.sort_by(|a, b| a.score.total_cmp(&b.score).then(a.order.cmp(&b.order))); + scored + } - let Some(best) = scored.first() else { - // Nothing within distance 2: fall back to Harper's first (distant) suggestion. - return (harper[0].clone(), Severity::S2, 0.5); - }; - let margin = scored - .get(1) - .map_or(f32::INFINITY, |second| second.score - best.score); + /// Severity and confidence of fixing `original` to the cheapest typo `best`, with the + /// runner-up `second`. + fn decide_spelling( + &self, + original: &str, + orig: &[char], + lower: &[char], + first_upper: bool, + best: &Scored, + second: Option<&Scored>, + ) -> (String, Severity, f32) { + let margin = second.map_or(f32::INFINITY, |second| second.score - best.score); let confidence = (1.0 - best.distance as f32 / orig.len() as f32).clamp(0.0, 1.0); let replacement = apply_case(original, &best.text); // A word the dictionary knows in another dialect (`colour`) is not a typo. A capitalised @@ -503,6 +544,16 @@ impl Checker { } } +/// A dictionary word scored as the intended spelling of a typo. +struct Scored { + text: String, + distance: usize, + /// Typing-error cost, less the common-word bonus: lower is likelier. + score: f32, + /// Position in the pool, which puts Harper's own suggestions first on a tie. + order: usize, +} + // --------------------------------------------------------------------------------------------- // Tunables // --------------------------------------------------------------------------------------------- @@ -524,6 +575,8 @@ const S0_MIN_CONFIDENCE: f32 = 0.6; const S0_MIN_MARGIN: f32 = 0.2; /// Cost credit for a candidate that is a very common English word. const COMMON_WORD_BONUS: f32 = 0.5; +/// Words `Checker::typo_options` offers at most. +const TYPO_OPTIONS: usize = 12; // --------------------------------------------------------------------------------------------- // Typing-error model @@ -1266,6 +1319,21 @@ mod tests { assert!(!keeps_first_letter(&['m', 'o', 'r', 't', 'h'], "north")); } + #[test] + fn typo_options_offer_close_words_and_nothing_for_names_or_words() { + let dict = PersonalDictionary::default(); + let morth = checker().typo_options("morth", &dict); + for word in ["north", "month"] { + assert!(morth.iter().any(|(o, _)| o == word), "{word} in {morth:?}"); + } + let colud = checker().typo_options("colud", &dict); + assert!(colud.iter().any(|(o, _)| o == "could"), "{colud:?}"); + assert!(!colud.iter().any(|(o, _)| o == "would"), "two slips away: {colud:?}"); + for typed in ["Jonhson", "colour", "month", "xlsx", "teh", "snake_case"] { + assert!(checker().typo_options(typed, &dict).is_empty(), "{typed}"); + } + } + #[test] fn is_code_like_cases() { for w in ["camelCase", "snake_case", "C:\\x\\y", "https://a.b", "a@b.c", "v2", "HTTP"] { diff --git a/tools/e2e/contenteditable.ps1 b/tools/e2e/contenteditable.ps1 index c10728e..3ff7d00 100644 --- a/tools/e2e/contenteditable.ps1 +++ b/tools/e2e/contenteditable.ps1 @@ -20,7 +20,7 @@ param( [string[]]$Fixes = @("wwant=want", "thhe=the", "aree=are", "mmore=more", "throughh=through", "thhat=that", "haave=have", "teh=the", "meetting=meeting", "Outllook=Outlook", "wiht=with", "recieve=receive", "definately=definitely", "documnet=document", - "yestreday=yesterday", "pakage=package"), + "yestreday=yesterday", "pakage=package", "morth=month"), [int]$CadenceMs = 45, [int]$AfterSpaceMs = 12, [int]$Runs = 3,