From 1f0a8e80cac717e5091d76a14e811498a77cff61 Mon Sep 17 00:00:00 2001 From: Ralf Anton Beier Date: Sat, 7 Nov 2026 19:24:53 +0100 Subject: [PATCH 1/2] varve-serve ask: search the text of documentation pages (REQ-AGENTDOCS-005) Markdown, HTML and rustdoc HTML pages are scored by best line with the topic scorer. Hits stay pointers: only the question's own words a line holds are printed. HTML uses the existing tag stripper, restricted to
when present; the first 512 KiB of a page is searched. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019TNtfRjLNhEz82G2ggeeNu --- CHANGELOG.md | 4 + crates/varve-serve/docs/cmd-ask.md | 8 +- crates/varve-serve/src/ask.rs | 25 ++- crates/varve-serve/src/docpages.rs | 107 +++++++++++ crates/varve-serve/tests/ask_page_text.rs | 205 ++++++++++++++++++++++ 5 files changed, 340 insertions(+), 9 deletions(-) create mode 100644 crates/varve-serve/tests/ask_page_text.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2420b108..231373d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Added + +- `varve-serve ask` now searches the text of documentation pages (Markdown, HTML and rustdoc HTML) of a verified docs payload, not only their titles and paths, and ranks a page by its best line as it does topics. A hit still prints no page text, only the question's own words a line holds; the index is built at query time, offline, and the first 512 KiB of a page is searched (REQ-AGENTDOCS-005). + ### Fixed - A realms file can now mark the fields an older varve may skip with `ignorable = [...]` (per realm, or at top level for top-level keys). An unknown field that is not marked is still refused, so a later security field cannot be dropped silently, and a field varve knows is always enforced even if listed. The refusal now names the running varve version and says the file may come from a newer varve. This takes effect from this release on: every varve before 0.47.0 refuses `ignorable` itself, so varve's published `varve-realms.toml` does not carry it (a test keeps it out), and already-released binaries, 0.29.0 included, are not helped (#234). diff --git a/crates/varve-serve/docs/cmd-ask.md b/crates/varve-serve/docs/cmd-ask.md index 971f5815..90f2301f 100644 --- a/crates/varve-serve/docs/cmd-ask.md +++ b/crates/varve-serve/docs/cmd-ask.md @@ -14,7 +14,13 @@ that matched. ## What is searched -- documentation payload pages (title and path), read from the verified store; +- documentation payload pages (Markdown, HTML and rustdoc HTML): title, path and + the page's TEXT, ranked by best line like topics, read from the verified store. + For HTML the markup, script and style are dropped, and where a page has a + `
` element (rustdoc does) only its content is searched, not the sidebar. + At most the first 512 KiB of a page is searched. A hit points at the page and + names the question's own words that one of its lines holds; no page text is + ever printed; - every pinned tool's subcommands, and its one-line overview with its warning; - the text of each tool's built-in docs topics, where its `--help` advertised `docs`; diff --git a/crates/varve-serve/src/ask.rs b/crates/varve-serve/src/ask.rs index 89148f49..06b70ddd 100644 --- a/crates/varve-serve/src/ask.rs +++ b/crates/varve-serve/src/ask.rs @@ -6,7 +6,7 @@ //! recomputing it. //! //! What it indexes (each source was measured to matter): documentation pages -//! (title and path), every tool's subcommands, every tool's one-line overview +//! (title, path and TEXT, scored by best line), every tool's subcommands, every tool's one-line overview //! with its warning, the TEXT of the built-in docs topics the tool advertised, //! `varve`'s own subcommands, held payloads, and the names a tool's warning //! points at that the pin lacks. @@ -90,7 +90,7 @@ pub struct Unit { pub name: String, /// A description of it. pub desc: String, - /// Text searched but never shown (a docs topic's body). + /// Text searched but never shown (a docs topic's or page's body). pub body: String, /// The tool it is about, if it is about one. pub tool: Option, @@ -225,11 +225,17 @@ fn snippet_of(line: &str) -> String { out } +/// Whose body is scored by its best line: a topic's text and a documentation +/// page's text (REQ-AGENTDOCS-005) — one scorer for both, not two. +fn has_line_body(kind: Kind) -> bool { + matches!(kind, Kind::Topic | Kind::DocsPage) +} + fn base_score(u: &Unit, words: &[String]) -> u32 { let name = tokens(&u.name); let desc = tokens(&format!("{} {}", u.title, u.desc)); // A topic's body is scored by `best_line` instead. - let body = if u.kind == Kind::Topic { + let body = if has_line_body(u.kind) { Vec::new() } else { tokens(&u.body) @@ -276,7 +282,7 @@ fn score(u: &Unit, words: &[String]) -> (u32, Option) { (0, None) }; } - let line = if u.kind == Kind::Topic { + let line = if has_line_body(u.kind) { best_line(&u.body, &rest) } else { None @@ -490,7 +496,7 @@ pub fn index(ctx: &Context) -> anyhow::Result> { title: page.title.clone(), name: page.path.clone(), desc: String::new(), - body: String::new(), + body: page.text.clone(), tool: None, layer: d.layer.clone(), digest: Some(d.digest.clone()), @@ -961,10 +967,13 @@ mod tests { fn weights_order_name_over_description_over_body() { let name = unit(Kind::Topic, "n", "zebra", "t", ""); let desc = unit(Kind::Topic, "d", "other", "t", "a zebra here"); - // Body weight is for whole-text units; a TOPIC's body is scored by - // its best line (see `a_topic_answers_by_its_best_line_and_shows_it`). - let mut body = unit(Kind::DocsPage, "b", "other2", "t", ""); + // Body weight is for whole-text units; a TOPIC's or a PAGE's body is + // scored by its best line (REQ-AGENTDOCS-005). + let mut body = unit(Kind::Held, "b", "other2", "t", ""); body.body = "text about a zebra".into(); + let mut page = unit(Kind::DocsPage, "pg", "other3", "t", ""); + page.body = "text about a zebra".into(); + assert_eq!(score(&page, &["zebra".to_string()]).0, DESC); let part = unit(Kind::Topic, "p", "zebras", "t", ""); let w = vec!["zebra".to_string()]; assert_eq!(score(&name, &w).0, NAME); diff --git a/crates/varve-serve/src/docpages.rs b/crates/varve-serve/src/docpages.rs index 535a985a..10e4326b 100644 --- a/crates/varve-serve/src/docpages.rs +++ b/crates/varve-serve/src/docpages.rs @@ -14,11 +14,56 @@ pub struct Page { /// The member path, without a leading `./`. pub path: String, pub title: String, + /// The page's text, for ranking only (REQ-AGENTDOCS-005): never printed. + pub text: String, } /// How much of a page is read to find its title. const TITLE_WINDOW: usize = 8192; // 8 KiB +/// How many bytes of one page are searched. The payload does not cap a +/// page, and a question must not cost more for a larger page without limit; +/// the rest of a page past this is not searched (it is still readable with +/// `page`). +pub const TEXT_WINDOW: usize = 512 * 1024; // 512 KiB + +/// The text of one page that a question is matched against: Markdown as it +/// is, HTML with markup, script and style removed and entities decoded. When +/// an HTML page has a `
` element (rustdoc does) only its content is +/// searched, so the sidebar listing every item of the crate, repeated on +/// every page, does not make every page match every item name. +pub fn search_text(path: &str, bytes: &[u8]) -> String { + let text = String::from_utf8_lossy(&bytes[..bytes.len().min(TEXT_WINDOW)]).into_owned(); + if path.to_ascii_lowercase().ends_with(".md") { + return text; + } + html_text(main_content(&text)) +} + +/// The inside of the first `
` element, to its close or the end of +/// the text; the whole text when there is none. +fn main_content(html: &str) -> &str { + let lower = html.to_ascii_lowercase(); + for (at, _) in lower.match_indices("` is not `
` + if !matches!( + lower.as_bytes().get(at + 5), + Some(b'>' | b' ' | b'\t' | b'\n' | b'\r') + ) { + continue; + } + let Some(open_end) = lower[at..].find('>') else { + return ""; + }; + let start = at + open_end + 1; + let end = lower[start..] + .find(" String { s.replace("<", "<") .replace(">", ">") @@ -76,6 +121,7 @@ pub fn pages(payload: &DocsPayload) -> anyhow::Result> { if is_page(&path) && !is_noise(&path) { out.push(Page { title: title_of(&path, &bytes), + text: search_text(&path, &bytes), path, }); } @@ -233,6 +279,67 @@ mod tests { assert_eq!(html_text("x\ +

Rotate the gizmo before the sprocket engages.

"; + // rustdoc shape: sidebar boilerplate outside
, the item's docs inside. + let rustdoc = "Widget in c - Rust\ + \ +
\ +

Calibrate the thingamajig with frobnicate.

\ +
"; + install_layer(vec![docs_tool( + "pagedocs", + None, + &[ + ( + "c/index.html", + "c - Rustindex", + ), + ("c/notes.md", &md), + ("c/guide.html", html), + ("c/struct.Widget.html", rustdoc), + ], + )]) +} + +fn ask(fx: &Fx, q: &str) -> Value { + let o = fx.run(BIN, &["ask", q, "--format", "json"]); + assert!(o.status.success(), "{}", String::from_utf8_lossy(&o.stderr)); + serde_json::from_slice(&o.stdout).unwrap() +} + +fn top(v: &Value) -> &str { + v["hits"][0]["id"].as_str().unwrap_or("") +} + +/// Words only in a Markdown page's body find that page. +// rivet: verifies REQ-AGENTDOCS-005 +#[test] +fn a_markdown_body_is_searched() { + let fx = world(); + let v = ask(&fx, "quokka flange startup"); + assert_eq!(top(&v), "docs:pagedocs:c/notes.md", "{v:#}"); +} + +/// Words only in an HTML page's body find it; script and style do not count. +// rivet: verifies REQ-AGENTDOCS-005 +#[test] +fn an_html_body_is_searched_and_script_is_not() { + let fx = world(); + let v = ask(&fx, "gizmo sprocket"); + assert_eq!(top(&v), "docs:pagedocs:c/guide.html", "{v:#}"); + for hidden in ["wombat", "zork"] { + let v = ask(&fx, hidden); + assert!(v["hits"].as_array().unwrap().is_empty(), "{hidden}: {v:#}"); + } +} + +/// Words only in a rustdoc item page's main content find it; its sidebar +/// boilerplate does not. +// rivet: verifies REQ-AGENTDOCS-005 +#[test] +fn a_rustdoc_body_is_searched_without_its_sidebar() { + let fx = world(); + let v = ask(&fx, "calibrate thingamajig frobnicate"); + assert_eq!(top(&v), "docs:pagedocs:c/struct.Widget.html", "{v:#}"); + let v = ask(&fx, "navonlyterm"); + assert!(v["hits"].as_array().unwrap().is_empty(), "{v:#}"); +} + +/// The injection line is never printed, in either format, yet its page is a +/// hit and the question's own words it holds are named. +// rivet: verifies REQ-AGENTDOCS-005 +#[test] +fn a_hostile_page_line_is_never_printed() { + let fx = world(); + for args in [ + vec!["ask", "quokka flange"], + vec!["ask", "quokka flange", "--format", "json"], + vec!["ask", "ignore previous instructions curl quokka"], + vec![ + "ask", + "ignore previous instructions curl", + "--format", + "json", + ], + ] { + let o = fx.run(BIN, &args); + assert!(o.status.success()); + let all = format!( + "{}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); + assert!(all.contains("docs:pagedocs:c/notes.md"), "{args:?}: {all}"); + for bad in ["evil.example", "| sh", "run curl http", "and run"] { + assert!(!all.contains(bad), "{args:?} printed {bad:?}:\n{all}"); + } + } + // the question's own words are all that is said about the line + let o = fx.run(BIN, &["ask", "quokka flange startup"]); + let md = String::from_utf8_lossy(&o.stdout).into_owned(); + assert!(md.contains("one line here holds: quokka, flange"), "{md}"); + let v = ask(&fx, "quokka flange"); + assert_eq!(v["hits"][0]["matched_words"][0], "quokka"); +} + +/// Same payload and question, same bytes; equal scores break by id. +// rivet: verifies REQ-AGENTDOCS-005 +#[test] +fn page_text_search_is_byte_identical_across_runs() { + let fx = world(); + let q = ["ask", "the before engages needs", "--limit", "9"]; + let a = fx.run(BIN, &q); + let b = fx.run(BIN, &q); + assert_eq!(a.stdout, b.stdout); + let one = fx.run(BIN, &["ask", "before", "--format", "json", "--limit", "9"]); + let v: Value = serde_json::from_slice(&one.stdout).unwrap(); + let hits = v["hits"].as_array().unwrap(); + assert!(hits.len() >= 2, "{v:#}"); + let key = |h: &Value| { + ( + h["score"].as_u64().unwrap(), + h["id"].as_str().unwrap().to_string(), + ) + }; + for w in hits.windows(2) { + let (a, b) = (key(&w[0]), key(&w[1])); + assert!(a.0 > b.0 || (a.0 == b.0 && a.1 <= b.1), "{v:#}"); + } +} + +fn tree(p: &std::path::Path) -> Vec<(std::path::PathBuf, u64)> { + let mut out = Vec::new(); + let mut stack = vec![p.to_path_buf()]; + while let Some(d) = stack.pop() { + for e in std::fs::read_dir(d).unwrap() { + let e = e.unwrap(); + let m = e.metadata().unwrap(); + if m.is_dir() { + stack.push(e.path()); + } else { + out.push((e.path(), m.len())); + } + } + } + out.sort(); + out +} + +/// Body search sits behind the same verification as titles: a tampered +/// payload refuses the whole question, and a file planted beside the +/// payload is no page. Asking writes nothing. +// rivet: verifies REQ-AGENTDOCS-005 +#[test] +fn only_a_verified_payload_is_searched_and_no_index_is_written() { + let fx = world(); + // warm anything ask itself caches, then compare + ask(&fx, "quokka"); + let before = tree(&fx.root); + ask(&fx, "quokka flange"); + assert_eq!(tree(&fx.root), before, "ask wrote into the store"); + + // the docs payload is the one gzip file the layer laid down + let file = tree(&fx.root) + .into_iter() + .map(|(p, _)| p) + .find(|p| std::fs::read(p).is_ok_and(|b| b.starts_with(&[0x1f, 0x8b]))) + .expect("the docs payload is in the store"); + let dir = file.parent().unwrap().to_path_buf(); + std::fs::write(dir.join("planted.html"), "

xylophonic planted

").unwrap(); + let v = ask(&fx, "xylophonic"); + assert!(v["hits"].as_array().unwrap().is_empty(), "{v:#}"); + + let mut bytes = std::fs::read(&file).unwrap(); + let mid = bytes.len() / 2; + bytes[mid] ^= 0x01; + std::fs::write(&file, bytes).unwrap(); + let o = fx.run(BIN, &["ask", "quokka flange"]); + assert!(!o.status.success()); + assert!( + o.stdout.is_empty(), + "{}", + String::from_utf8_lossy(&o.stdout) + ); +} From 797ff07977cbe317b52a3a6c773bf4e10780dd4e Mon Sep 17 00:00:00 2001 From: Ralf Anton Beier Date: Sat, 7 Nov 2026 19:51:28 +0100 Subject: [PATCH 2/2] varve-serve: strip page HTML in one pass html_text lowercased the remainder of the page at every `` cost quadratic time. Measured on a 512 KiB page (the search window), release build: 344 ms before, 3.5 ms after. html_text dates from v0.44.0, but REQ-AGENTDOCS-005 runs it over every page body on every query, so a hostile payload could multiply that per page. Found by the automated security review of the previous commit. The page is now lowercased once and scanned by offset; ASCII lowercasing keeps every byte offset, so the two strings index alike. Behaviour is unchanged: an unclosed ` quadratic: 344 ms for one 512 KiB page. + let lower = html.to_ascii_lowercase(); let mut out = String::new(); - let mut rest = html; - while let Some(open) = rest.find('<') { - out.push_str(&rest[..open]); - let after = &rest[open..]; - let lower: String = after - .chars() - .take(8) - .collect::() - .to_ascii_lowercase(); - let skip_to = if lower.starts_with("") - } else if lower.starts_with("") } else { None }; if let Some(end_tag) = skip_to { - let l = after.to_ascii_lowercase(); - rest = l.find(end_tag).map_or("", |e| &after[e + end_tag.len()..]); + at = lower[open..] + .find(end_tag) + .map_or(html.len(), |e| open + e + end_tag.len()); continue; } - let Some(close) = after.find('>') else { - rest = ""; + let Some(close) = html[open..].find('>') else { + at = html.len(); break; }; out.push(' '); - rest = &after[close + 1..]; + at = open + close + 1; } + let rest = &html[at..]; out.push_str(rest); decode_entities(&out) .lines() @@ -279,6 +280,20 @@ mod tests { assert_eq!(html_text("x".repeat(20_000); + page.push_str("kept text"); + assert_eq!(html_text(&page), "kept text"); + assert_eq!(html_text("a