varve-serve ask: search the text of documentation pages (REQ-AGENTDOCS-005)

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Added
+
+- `varve-serve ask` now searches the text of documentation pages (Markdown, HTML and rustdoc HTML) of a verified docs payload, not only their titles and paths, and ranks a page by its best line as it does topics. A hit still prints no page text, only the question's own words a line holds; the index is built at query time, offline, and the first 512 KiB of a page is searched (REQ-AGENTDOCS-005).
+
 ### Fixed
 
 - A realms file can now mark the fields an older varve may skip with `ignorable = [...]` (per realm, or at top level for top-level keys). An unknown field that is not marked is still refused, so a later security field cannot be dropped silently, and a field varve knows is always enforced even if listed. The refusal now names the running varve version and says the file may come from a newer varve. This takes effect from this release on: every varve before 0.47.0 refuses `ignorable` itself, so varve's published `varve-realms.toml` does not carry it (a test keeps it out), and already-released binaries, 0.29.0 included, are not helped (#234).

crates/varve-serve/docs/cmd-ask.md

@@ -14,7 +14,13 @@ that matched.
 
 ## What is searched
 
-- documentation payload pages (title and path), read from the verified store;
+- documentation payload pages (Markdown, HTML and rustdoc HTML): title, path and
+  the page's TEXT, ranked by best line like topics, read from the verified store.
+  For HTML the markup, script and style are dropped, and where a page has a
+  `<main>` element (rustdoc does) only its content is searched, not the sidebar.
+  At most the first 512 KiB of a page is searched. A hit points at the page and
+  names the question's own words that one of its lines holds; no page text is
+  ever printed;
 - every pinned tool's subcommands, and its one-line overview with its warning;
 - the text of each tool's built-in docs topics, where its `--help` advertised
   `docs`;

crates/varve-serve/src/ask.rs

@@ -6,7 +6,7 @@
 //! recomputing it.
 //!
 //! What it indexes (each source was measured to matter): documentation pages
-//! (title and path), every tool's subcommands, every tool's one-line overview
+//! (title, path and TEXT, scored by best line), every tool's subcommands, every tool's one-line overview
 //! with its warning, the TEXT of the built-in docs topics the tool advertised,
 //! `varve`'s own subcommands, held payloads, and the names a tool's warning
 //! points at that the pin lacks.
@@ -90,7 +90,7 @@ pub struct Unit {
     pub name: String,
     /// A description of it.
     pub desc: String,
-    /// Text searched but never shown (a docs topic's body).
+    /// Text searched but never shown (a docs topic's or page's body).
     pub body: String,
     /// The tool it is about, if it is about one.
     pub tool: Option<String>,
@@ -225,11 +225,17 @@ fn snippet_of(line: &str) -> String {
     out
 }
 
+/// Whose body is scored by its best line: a topic's text and a documentation
+/// page's text (REQ-AGENTDOCS-005) — one scorer for both, not two.
+fn has_line_body(kind: Kind) -> bool {
+    matches!(kind, Kind::Topic | Kind::DocsPage)
+}
+
 fn base_score(u: &Unit, words: &[String]) -> u32 {
     let name = tokens(&u.name);
     let desc = tokens(&format!("{} {}", u.title, u.desc));
     // A topic's body is scored by `best_line` instead.
-    let body = if u.kind == Kind::Topic {
+    let body = if has_line_body(u.kind) {
         Vec::new()
     } else {
         tokens(&u.body)
@@ -276,7 +282,7 @@ fn score(u: &Unit, words: &[String]) -> (u32, Option<String>) {
             (0, None)
         };
     }
-    let line = if u.kind == Kind::Topic {
+    let line = if has_line_body(u.kind) {
         best_line(&u.body, &rest)
     } else {
         None
@@ -490,7 +496,7 @@ pub fn index(ctx: &Context) -> anyhow::Result<Vec<Unit>> {
                 title: page.title.clone(),
                 name: page.path.clone(),
                 desc: String::new(),
-                body: String::new(),
+                body: page.text.clone(),
                 tool: None,
                 layer: d.layer.clone(),
                 digest: Some(d.digest.clone()),
@@ -961,10 +967,13 @@ mod tests {
     fn weights_order_name_over_description_over_body() {
         let name = unit(Kind::Topic, "n", "zebra", "t", "");
         let desc = unit(Kind::Topic, "d", "other", "t", "a zebra here");
-        // Body weight is for whole-text units; a TOPIC's body is scored by
-        // its best line (see `a_topic_answers_by_its_best_line_and_shows_it`).
-        let mut body = unit(Kind::DocsPage, "b", "other2", "t", "");
+        // Body weight is for whole-text units; a TOPIC's or a PAGE's body is
+        // scored by its best line (REQ-AGENTDOCS-005).
+        let mut body = unit(Kind::Held, "b", "other2", "t", "");
         body.body = "text about a zebra".into();
+        let mut page = unit(Kind::DocsPage, "pg", "other3", "t", "");
+        page.body = "text about a zebra".into();
+        assert_eq!(score(&page, &["zebra".to_string()]).0, DESC);
         let part = unit(Kind::Topic, "p", "zebras", "t", "");
         let w = vec!["zebra".to_string()];
         assert_eq!(score(&name, &w).0, NAME);

crates/varve-serve/src/docpages.rs

@@ -14,11 +14,56 @@ pub struct Page {
     /// The member path, without a leading `./`.
     pub path: String,
     pub title: String,
+    /// The page's text, for ranking only (REQ-AGENTDOCS-005): never printed.
+    pub text: String,
 }
 
 /// How much of a page is read to find its title.
 const TITLE_WINDOW: usize = 8192; // 8 KiB
 
+/// How many bytes of one page are searched. The payload does not cap a
+/// page, and a question must not cost more for a larger page without limit;
+/// the rest of a page past this is not searched (it is still readable with
+/// `page`).
+pub const TEXT_WINDOW: usize = 512 * 1024; // 512 KiB
+
+/// The text of one page that a question is matched against: Markdown as it
+/// is, HTML with markup, script and style removed and entities decoded. When
+/// an HTML page has a `<main>` element (rustdoc does) only its content is
+/// searched, so the sidebar listing every item of the crate, repeated on
+/// every page, does not make every page match every item name.
+pub fn search_text(path: &str, bytes: &[u8]) -> String {
+    let text = String::from_utf8_lossy(&bytes[..bytes.len().min(TEXT_WINDOW)]).into_owned();
+    if path.to_ascii_lowercase().ends_with(".md") {
+        return text;
+    }
+    html_text(main_content(&text))
+}
+
+/// The inside of the first `<main ...>` element, to its close or the end of
+/// the text; the whole text when there is none.
+fn main_content(html: &str) -> &str {
+    let lower = html.to_ascii_lowercase();
+    for (at, _) in lower.match_indices("<main") {
+        // `<mainframe>` is not `<main>`
+        if !matches!(
+            lower.as_bytes().get(at + 5),
+            Some(b'>' | b' ' | b'\t' | b'\n' | b'\r')
+        ) {
+            continue;
+        }
+        let Some(open_end) = lower[at..].find('>') else {
+            return "";
+        };
+        let start = at + open_end + 1;
+        let end = lower[start..]
+            .find("</main")
+            .map_or(html.len(), |e| start + e);
+        return &html[start..end];
+    }
+    html
+}
+
 fn decode_entities(s: &str) -> String {
     s.replace("&lt;", "<")
         .replace("&gt;", ">")
@@ -76,6 +121,7 @@ pub fn pages(payload: &DocsPayload) -> anyhow::Result<Vec<Page>> {
         if is_page(&path) && !is_noise(&path) {
             out.push(Page {
                 title: title_of(&path, &bytes),
+                text: search_text(&path, &bytes),
                 path,
             });
         }
@@ -87,35 +133,36 @@ pub fn pages(payload: &DocsPayload) -> anyhow::Result<Vec<Page>> {
 
 /// Markup removed, scripts and styles dropped, entities decoded.
 pub fn html_text(html: &str) -> String {
+    // One lowercased copy, searched by offset. `to_ascii_lowercase` keeps
+    // every byte offset, so an index into `lower` is an index into `html`.
+    // Lowercasing the remainder at each `<script`/`<style` instead made a page
+    // of repeated `<script></script>` quadratic: 344 ms for one 512 KiB page.
+    let lower = html.to_ascii_lowercase();
     let mut out = String::new();
-    let mut rest = html;
-    while let Some(open) = rest.find('<') {
-        out.push_str(&rest[..open]);
-        let after = &rest[open..];
-        let lower: String = after
-            .chars()
-            .take(8)
-            .collect::<String>()
-            .to_ascii_lowercase();
-        let skip_to = if lower.starts_with("<script") {
+    let mut at = 0;
+    while let Some(open) = html[at..].find('<').map(|o| at + o) {
+        out.push_str(&html[at..open]);
+        let skip_to = if lower[open..].starts_with("<script") {
             Some("</script>")
-        } else if lower.starts_with("<style") {
+        } else if lower[open..].starts_with("<style") {
             Some("</style>")
         } else {
             None
         };
         if let Some(end_tag) = skip_to {
-            let l = after.to_ascii_lowercase();
-            rest = l.find(end_tag).map_or("", |e| &after[e + end_tag.len()..]);
+            at = lower[open..]
+                .find(end_tag)
+                .map_or(html.len(), |e| open + e + end_tag.len());
             continue;
         }
-        let Some(close) = after.find('>') else {
-            rest = "";
+        let Some(close) = html[open..].find('>') else {
+            at = html.len();
             break;
         };
         out.push(' ');
-        rest = &after[close + 1..];
+        at = open + close + 1;
     }
+    let rest = &html[at..];
     out.push_str(rest);
     decode_entities(&out)
         .lines()
@@ -233,6 +280,81 @@ mod tests {
         assert_eq!(html_text("x<script>never closed"), "x");
     }
 
+    /// A page made of many `<script>` elements is stripped in one pass.
+    /// Lowercasing the remainder at each one made this quadratic (344 ms for
+    /// one 512 KiB page, against ~3.5 ms now). Every element is still skipped
+    /// with its contents, case-insensitively, and the text after them is kept.
+    // rivet: verifies REQ-AGENTDOCS-005
+    #[test]
+    fn many_script_elements_are_skipped_in_one_pass() {
+        let mut page = "<SCRIPT>evil()</ScRiPt>".repeat(20_000);
+        page.push_str("<style>x{}</STYLE>kept <b>text</b>");
+        assert_eq!(html_text(&page), "kept text");
+        assert_eq!(html_text("a<script>never closed"), "a");
+        assert_eq!(html_text("a<b no close"), "a");
+    }
+
+    /// What a question is matched against: markdown as written; HTML as text
+    /// from its `<main>` when it has one; a bounded prefix of a huge page.
+    // rivet: verifies REQ-AGENTDOCS-005
+    #[test]
+    fn search_text_is_markdown_or_the_main_content_of_html_within_a_window() {
+        assert_eq!(search_text("a.md", b"# T\n<b>x</b>"), "# T\n<b>x</b>");
+        assert_eq!(search_text("A.MD", b"<i>"), "<i>");
+        assert_eq!(search_text("a.html", b"<p>a<b>b</b></p>"), "a b");
+        let rd = b"<nav>side</nav><MAIN id=x><p>body &amp; soul</p></MAIN><footer>foot</footer>";
+        assert_eq!(search_text("a.html", rd), "body & soul");
+        // newline and tab also end the tag name; an unclosed main runs to the end
+        assert_eq!(search_text("a.html", b"<nav>n</nav><main\n>in"), "in");
+        assert_eq!(search_text("a.html", b"<nav>n</nav><main\t>in"), "in");
+        assert_eq!(search_text("a.html", b"<nav>n</nav><main>in</main"), "in");
+        // `<mainframe>` is not main; a later real <main> is found
+        assert_eq!(search_text("a.html", b"<mainframe>x</mainframe>"), "x");
+        assert_eq!(
+            search_text("a.html", b"<mainframe>x</mainframe><main>y</main>"),
+            "y"
+        );
+        // a main that never closes its tag has no content
+        assert_eq!(search_text("a.html", b"<p>a</p><main id=x"), "");
+        // the window: bytes past it are not searched, and cutting inside a
+        // multi-byte character does not panic
+        let mut big = "a ".repeat(TEXT_WINDOW / 2).into_bytes();
+        big.extend_from_slice(b"beyondwindow");
+        assert!(!search_text("a.md", &big).contains("beyondwindow"));
+        let mut edge = vec![b'x'; TEXT_WINDOW - 2];
+        edge.extend_from_slice(" tail".as_bytes());
+        assert_eq!(search_text("a.md", &edge).len(), TEXT_WINDOW);
+        assert_eq!(TEXT_WINDOW, 524_288, "the documented 512 KiB cap");
+        let mut cut = vec![b'x'; TEXT_WINDOW - 1];
+        cut.extend_from_slice("é".as_bytes());
+        assert!(search_text("a.md", &cut).starts_with("xxx"));
+    }
+
+    /// Pages carry their text, so the index can rank by it.
+    // rivet: verifies REQ-AGENTDOCS-005
+    #[test]
+    fn pages_carry_their_search_text() {
+        let p = payload(
+            DocsFormat::Html,
+            &[
+                ("a.html", "<title>T</title><main>hello</main>"),
+                ("b.md", "# B\nworld"),
+            ],
+        );
+        let got: Vec<(String, String)> = pages(&p)
+            .unwrap()
+            .into_iter()
+            .map(|g| (g.path, g.text))
+            .collect();
+        assert_eq!(
+            got,
+            [
+                ("a.html".into(), "hello".into()),
+                ("b.md".into(), "# B\nworld".into())
+            ]
+        );
+    }
+
     /// `page` shows a member, refuses everything else — `../` included.
     // rivet: verifies REQ-AGENTDOCS-002
     #[test]