diff --git a/Makefile b/Makefile index f9d2d284b..0f7faa410 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,8 @@ clean: lint: rstcheck -r "$(SOURCEDIR)" -html: +html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) "$(SOURCEDIR)" "$(BUILDDIR)" + python3 scripts/build_searchindex.py "$(BUILDDIR)" @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)." diff --git a/docker/requirements.in b/docker/requirements.in index 97cc9e2e4..471784b08 100644 --- a/docker/requirements.in +++ b/docker/requirements.in @@ -1,3 +1,4 @@ +beautifulsoup4>=4.12.0 Jinja2>=3.1.5 lxml>=6.1.2 rstcheck==6.2.5 diff --git a/docker/requirements.txt b/docker/requirements.txt index 8bcca7668..cb92980b9 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -12,6 +12,8 @@ annotated-types==0.8.0 # via pydantic babel==2.18.0 # via sphinx +beautifulsoup4==4.15.0 + # via -r requirements.in certifi==2026.7.22 # via requests charset-normalizer==3.5.1 @@ -29,7 +31,7 @@ jinja2==3.1.6 # via # -r requirements.in # sphinx -lxml==6.1.2 +lxml==6.1.3 # via -r requirements.in markdown-it-py==4.2.0 # via rich @@ -63,6 +65,8 @@ shellingham==1.5.4 # via typer snowballstemmer==3.1.1 # via sphinx +soupsieve==2.9.2 + # via beautifulsoup4 sphinx==9.1.0 # via # -r requirements.in @@ -94,6 +98,7 @@ typer==0.27.2 # via rstcheck typing-extensions==4.16.0 # via + # beautifulsoup4 # pydantic # pydantic-core # typing-inspection diff --git a/scripts/build_searchindex.py b/scripts/build_searchindex.py new file mode 100644 index 000000000..6bf6bf10c --- /dev/null +++ b/scripts/build_searchindex.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Build SQLite FTS5 search index from Sphinx HTML output. + +Creates searchindex.db in the build directory with four BM25-weighted fields: +title (10x), code_blocks (5x), image_alt (5x), body_text (1x). + +Usage: + python scripts/build_searchindex.py +""" + +import sqlite3 +import sys +from pathlib import Path + +from bs4 import BeautifulSoup, Comment + +_SKIP_FILES = {"search.html", "genindex.html", "searchindex.js"} + +_STRIP_TAGS = { + "script", "style", "nav", "footer", "header", + "noscript", "form", "button", +} + +_STRIP_CLASSES = { + "wy-nav-top", "wy-nav-side", "wy-breadcrumbs", + "rst-versions", "footer", "headerlink", + "toctree-wrapper", + "sphinxsidebar", "related", + "sphinx-tabs-tab", +} + +_SPHINX_INTERNALS = { + "_static", "_sources", "_images", "_downloads", + "_sphinx_design_static", "doctrees", +} + + +def _strip_noise(soup): + """Remove navigation chrome and HTML comments in-place. + + Collects tags first, then decomposes — avoids mutating the tree mid-iteration. + """ + to_remove = [] + for tag in soup.find_all(True): + if tag.name in _STRIP_TAGS: + to_remove.append(tag) + else: + classes = set(tag.get("class") or []) + if classes & _STRIP_CLASSES: + to_remove.append(tag) + for tag in to_remove: + tag.decompose() + for comment in soup.find_all(string=lambda t: isinstance(t, Comment)): + comment.extract() + + +def _extract_fields(html_path): + """Parse one HTML file and return (title, code_blocks, image_alt, body_text). + + :param html_path: Path to the HTML file + :returns: (title, code_blocks, image_alt, body_text) as plain strings + """ + soup = BeautifulSoup( + html_path.read_text(encoding="utf-8", errors="ignore"), + "html.parser", + ) + + h1 = soup.find("h1") + if h1: + for span in h1.find_all("span", class_="section-number"): + span.decompose() + title = h1.get_text(" ", strip=True) + else: + title = "" + if not title: + t = soup.find("title") + title = t.get_text(" ", strip=True) if t else html_path.stem + + # Extract code-like markup before stripping noise so these land in + # code_blocks (5x weight) not body_text (1x). + # :command: → , :program: → + code_parts = [] + for tag in soup.find_all(["pre", "code"]): + text = tag.get_text(" ", strip=True) + if text: + code_parts.append(text) + tag.decompose() + + for tag in soup.find_all("strong", class_=lambda c: c and ( + "command" in c or "program" in c)): + text = tag.get_text(" ", strip=True) + if text: + code_parts.append(text) + tag.decompose() + + code_blocks = " ".join(code_parts) + + # Extract image alt text (5x weight) + alt_parts = [] + for img in soup.find_all("img"): + alt = img.get("alt", "").strip() + if alt: + alt_parts.append(alt) + image_alt = " ".join(alt_parts) + + _strip_noise(soup) + + # Theme-specific main content selectors: RTD → classic → fallback + main = ( + soup.find("div", {"role": "main"}) + or soup.find("div", class_="document") + or soup.find("div", class_="body") + or soup.find("body") + or soup + ) + body_text = main.get_text(" ", strip=True) if main else soup.get_text(" ", strip=True) + + return title, code_blocks, image_alt, body_text + + +def build_index(build_dir): + db_path = build_dir / "searchindex.db" + if db_path.exists(): + db_path.unlink() + + con = sqlite3.connect(db_path) + con.execute("PRAGMA journal_mode=WAL") + con.execute(""" + CREATE TABLE docs ( + id INTEGER PRIMARY KEY, + docname TEXT NOT NULL, + title TEXT, + code_blocks TEXT, + image_alt TEXT, + body_text TEXT + ) + """) + con.execute(""" + CREATE VIRTUAL TABLE fts USING fts5( + title, + code_blocks, + image_alt, + body_text, + content=docs, + content_rowid=id, + tokenize='porter ascii' + ) + """) + + html_files = sorted(build_dir.rglob("*.html")) + total = len(html_files) + inserted = 0 + + print(f"Indexing {total} HTML files in {build_dir} ...") + + for i, html_path in enumerate(html_files): + if html_path.name in _SKIP_FILES: + continue + relative = html_path.relative_to(build_dir) + if relative.parts[0] in _SPHINX_INTERNALS: + continue + + try: + title, code_blocks, image_alt, body_text = _extract_fields(html_path) + except Exception as exc: + print(f" SKIP {relative}: {exc}") + continue + + con.execute( + "INSERT INTO docs (docname, title, code_blocks, image_alt, body_text) VALUES (?,?,?,?,?)", + (str(relative.with_suffix("")), title, code_blocks, image_alt, body_text), + ) + inserted += 1 + + if (i + 1) % 50 == 0: + print(f" {i + 1}/{total} processed ...") + + con.execute(""" + INSERT INTO fts(rowid, title, code_blocks, image_alt, body_text) + SELECT id, title, code_blocks, image_alt, body_text FROM docs + """) + con.commit() + con.execute("INSERT INTO fts(fts) VALUES('optimize')") + con.commit() + con.close() + + size_kb = db_path.stat().st_size // 1024 + print(f"Done. Indexed {inserted} pages → {db_path} ({size_kb} KB)") + return db_path + + +def main(): + if len(sys.argv) < 2: + print(__doc__) + sys.exit(1) + build_dir = Path(sys.argv[1]) + if not build_dir.exists(): + print(f"ERROR: {build_dir} does not exist") + sys.exit(1) + build_index(build_dir) + + +if __name__ == "__main__": + main()