Query: text conditions run through gufi_vt; an unbuilt main folder is "not indexed yet"

deploy/mac/check_app.sh

@@ -454,6 +454,20 @@ if [ "$RUN" = 1 ]; then
         } >"$T/vtdiag.txt" 2>&1 || true
         sed 's/^/        /' "$T/vtdiag.txt"
       fi
+      # The same with a words condition, whose FTS5 phrase quotes reach
+      # gufi_vt written as char(34): the OCR'd word in the main folder.
+      if [ "${ocr_found:-0}" = 1 ]; then
+        curl -fsS -m 60 -X POST -H 'content-type: application/json' \
+          -d '{"library": "all", "builder": {"source": "files", "filters": [{"field": "text", "op": "match", "value": "heliotrope"}], "groupBy": "none", "sort": "size", "dir": "desc", "limit": 20}}' \
+          "http://127.0.0.1:$PORT_FREE/api/query" >"$T/qwords.json" 2>"$T/qwords.err" || true
+        if python3 -c 'import json, sys; d = json.load(open(sys.argv[1])); found = any("ocr-sample" in str(r) for r in d.get("rows") or []); sys.exit(0 if d.get("engine") == "gufi_vt" and found and not d.get("failed") else 1)' "$T/qwords.json" 2>/dev/null; then
+          ok "and a query with a words condition runs through gufi_vt too"
+        else
+          bad "a query with a words condition did not run through gufi_vt:"
+          python3 -c 'import json, sys; d = json.load(open(sys.argv[1])); print("engine", d.get("engine"), "why", d.get("mergeReason"), "rows", len(d.get("rows") or []), "failed", d.get("failed"))' "$T/qwords.json" 2>&1 | sed 's/^/        /'
+          sed 's/^/        /' "$T/qwords.err" || true
+        fi
+      fi
     else
       bad "could not add a second library to query across"
     fi

docs/REFERENCE.md

@@ -58,7 +58,7 @@ The details behind the user guide ([`HELP.md`](HELP.md)), by part of the app. Th
 - Across all libraries, search runs each library in parallel and merges the results. Settings, General, Indexing, "Search across libraries" switches it to `gufi_vt`, as Query uses: name matches, and full-text matches where every search word is plain letters and digits, then run as one `gufi_vt` statement per kind, shown in the GUFI commands panel, while a word with punctuation, a library `gufi_vt` cannot reach, and results by meaning still run per library. Both ways return the same matches. `gufi_vt` answers its libraries one after another, so the parallel default is faster. The environment variable `STUDIO_SEARCH_ENGINE=gufi_vt` sets the default for a deployment.
 - In the parallel default each library has 20 seconds to answer (`STUDIO_SEARCH_DEADLINE_MS`). One that has not is left out and `GET /api/search` names it in `slow`; choosing that library alone waits for it.
 - A remote host whose login runs only `gufi_query` (`server/src/remoteExec.ts`): when the session loop is refused there (no bash, a restricted shell, a forced command), the site is marked direct and each command is its own ssh call over the shared connection, the program name bare and each other argument a POSIX shell word; only `gufi_query` and `gufi_sqlite3` are sent, and anything else answers 409 `<host> runs only gufi_query for this login (no shell), so this cannot run there.` The connection tests (`POST /api/user-libraries/ssh/test`, `.../pw/test`) answer `shell: false` there, the library is saved with `shell: false` and `gufiBin` `''` when `gufi_query` runs from the login's PATH, and a build check answers its ssh step with "This host runs only gufi_query: connect existing indexes read only; building an index there is not available." A connected index whose top levels have a `words` table is saved with `fullText`, so search reads its words. On macOS the run loop passes on a `KRB5CCNAME` the login shell sets, for ssh with Kerberos. Details in [`LIBRARIES.md`](LIBRARIES.md).
-- No library drops out silently (`server/src/omissions.ts`). `GET /api/search` and `POST /api/query` carry `omitted` when any library in scope was left out, or searched only in part: one entry per library, `{ id, label, reason, partial }`, `partial` false when it added nothing. The reasons: `being built; searchable once its first pass is done`; `its build failed: ...`; `index not found`; `words not searched yet: its text is being read`, or `words not searched: its index has no text yet` (a library built at a depth that reads text whose index has no `words` table yet); `not searched by meaning: its index has no vectors yet`, for a search by meaning; `did not answer within 20 s`; `not reachable over ssh: ...`; `search failed: ...`; on macOS `macOS has not allowed ACTIVATE Studio to read <folder>; allow it in System Settings, Privacy & Security, Files and Folders (or Full Disk Access)` (elsewhere `permission denied reading <folder>`), with `; results are from its last pass` when its index still answers; and `a folder could not be read (...)` or `N folders could not be read (...)` for folders inside it. Query names a library with no extracted text when the query reads the `words` table (`its index has no extracted text, so text conditions match nothing there`). A library with nothing to answer from is not asked: across libraries it is in `omitted` and in Query's `failed`; asked alone, Query answers 409 `<label> cannot be queried: <reason>`. The Search and Query pages show one line above the results, each name opening its row in Settings, Libraries, and `search_kb` ends its answer with the same list. An index connected as it is, without text, is searched by name and metadata and not named.
+- No library drops out silently (`server/src/omissions.ts`). `GET /api/search` and `POST /api/query` carry `omitted` when any library in scope was left out, or searched only in part: one entry per library, `{ id, label, reason, partial }`, `partial` false when it added nothing. The reasons: `being built; searchable once its first pass is done`; `its build failed: ...`; `not indexed yet`, for the main folder before its first build (an empty index folder does not count as an index; the main folder is indexed once its index has a `db.db` at the root); `index not found`, for a library built before whose index is gone; `words not searched yet: its text is being read`, or `words not searched: its index has no text yet` (a library built at a depth that reads text whose index has no `words` table yet); `not searched by meaning: its index has no vectors yet`, for a search by meaning; `did not answer within 20 s`; `not reachable over ssh: ...`; `search failed: ...`; on macOS `macOS has not allowed ACTIVATE Studio to read <folder>; allow it in System Settings, Privacy & Security, Files and Folders (or Full Disk Access)` (elsewhere `permission denied reading <folder>`), with `; results are from its last pass` when its index still answers; and `a folder could not be read (...)` or `N folders could not be read (...)` for folders inside it. Query names a library with no extracted text when the query reads the `words` table (`its index has no extracted text, so text conditions match nothing there`). A library with nothing to answer from is not asked: across libraries it is in `omitted` and in Query's `failed`; asked alone, Query answers 409 `<label> cannot be queried: <reason>`. The Search and Query pages show one line above the results, each name opening its row in Settings, Libraries, and `search_kb` ends its answer with the same list. An index connected as it is, without text, is searched by name and metadata and not named.
 - A search runs one walk of a library's index for every kind of match it can find there (`walkSearch` in `server/src/gufi.ts`), the one-pass pattern GUFI's authors recommend: one aggregate `gufi_query` (`-a 1`) whose per-directory step (`-E`) adds, while that directory's database is open, the name matches, the files below a folder whose path matches, the Studio's word matches (its fts5 `words` table) and its nearest passages by meaning (its vec0 `gvec`, with the query's embedding passed in `-I` as a blob literal, one copy per thread), each into one per-thread table that the merge (`-J`) gathers and the final step (`-G`) ranks. The statements run in that order, names and paths first, so a directory without a `words` or vector table answers what it can: the step that needs the missing table fails there and only the steps after it are skipped. No empty tables are written to make every directory alike (an empty fts5 table adds about 20 KB to a directory's 48 KB database, and empty vector tables 40 KB more, as measured). `--no-print-sql-on-err` keeps each such failure to one short line, and only the last 16 KB of `gufi_query`'s error output is kept. gufi_query compiles every statement again in every directory, so there are four at most: one trim that keeps each kind's best, then one insert per kind of table read.
 - On a synthetic index of 24,000 directories and 100,000 files, with words in a quarter of the directories and vectors in a tenth, a search took about 2.9 s before (the name and word walk, about 0.8 s, beside the search by meaning, which walked the tree for databases with vectors and then attached each one, about 2.9 s) and about 1.0 s in one walk. Without vectors in the index the walk takes about 0.85 s; names and words alone, as before, about 0.7 s, and the folder-path matches add about 0.15 s. Timings on a shared host, median of five.
 - Results are fused with reciprocal rank fusion (`fuseRows`): each kind is ranked on its own (words by `bm25`, vectors by distance, names by how closely they match, folder-path matches by how near the matching folder is), each file once per kind, and a file's score is the sum over the kinds that found it of 1 / (60 + its rank in that kind). The final step of each walk computes the same order for its own rows; across libraries and walks the Studio fuses them again the same way. Ties go to the more exact kind (words, then name, then folder path, then meaning), then the path, so the order is the same on every run. A hit is labelled by its most exact kind (`mode`: `fts`, `name`, `path` or `vector`) and lists all of them (`kinds`); `score` is its fused score. `bm25` is computed within each directory's table, so word scores from two directories compare roughly; vec0 distances for unit vectors compare exactly.
@@ -94,7 +94,7 @@ While open and the tab is visible, the panel reads `GET /api/tasks`, `/api/fleet
   - label: equals or contains, directory labels included.
   - Directories take name, owner, size (`totsize`) and modified conditions.
 - **SQL**: any read-only SELECT over the index tables.
-- **Across libraries**, a query runs as one `gufi_vt` statement: a virtual table per library, a library on another machine as a `remote_cmd` location, unioned with the overall order and limit outside. `POST /api/query` answers `engine: "gufi_vt"`. The Studio merges the libraries' rows itself instead (`engine: "merge"`, the same rows) when this GUFI's `gufi_sqlite3` has no `gufi_vt` with remote locations, when the query's SQL holds a double quote, or when a library's path cannot be given to `gufi_vt`. Paths with spaces can: `gufi_vt` starts `gufi_query` and the remote command with `execvp`, never a shell, and strips only a value's outer quotes, so the Studio single-quotes each path, or double-quotes one holding a single quote, as the macOS app's paths need (`ACTIVATE Studio.app`, `Application Support`). A path holding both kinds of quote, or a line break, cannot be given. A remote library's index reaches the wrapper unquoted at the end of the command line `gufi_vt` builds, so it may hold single spaces but no word of it may start with a quote.
+- **Across libraries**, a query runs as one `gufi_vt` statement: a virtual table per library, a library on another machine as a `remote_cmd` location, unioned with the overall order and limit outside. `POST /api/query` answers `engine: "gufi_vt"`. The Studio merges the libraries' rows itself instead (`engine: "merge"`, the same rows, with `mergeReason` saying why) when this GUFI's `gufi_sqlite3` has no `gufi_vt` with remote locations, when the query's SQL holds a double quote outside a string (a quoted identifier, a comment), when a library's path cannot be given to `gufi_vt`, or when the `gufi_vt` statement fails. `gufi_vt` takes each SQL stage as a double-quoted value read verbatim, so a double quote inside a string, such as the phrase quotes of a text condition's FTS5 `MATCH '"a" AND "b"'`, is written for it as `'||char(34)||'`, which gives the same string; text conditions therefore run through `gufi_vt` too. Paths with spaces can: `gufi_vt` starts `gufi_query` and the remote command with `execvp`, never a shell, and strips only a value's outer quotes, so the Studio single-quotes each path, or double-quotes one holding a single quote, as the macOS app's paths need (`ACTIVATE Studio.app`, `Application Support`). A path holding both kinds of quote, or a line break, cannot be given. A remote library's index reaches the wrapper unquoted at the end of the command line `gufi_vt` builds, so it may hold single spaces but no word of it may start with a quote.
 - **Saved queries** rerun in one click; a new deployment comes with a few examples. The code button beside one shows the SQL of each `gufi_query` stage it compiles to (`-I` and `-K` create a table for each thread and one for the result, `-E` runs in every directory, `-J` adds each thread's rows to the result, `-G` orders and limits it; raw SQL is `-E` alone) and the command that runs it, each with a copy button. It reads `POST /api/query/sql`, which takes the body `POST /api/query` takes and answers `stages` (`flag`, `sql`) and `commands`, one per library in scope as a run would record them, without running anything.
 - Result rows with a path can be ticked and labeled, as search results can.
 

server/src/gufiVt.ts

@@ -71,9 +71,46 @@ export function vtPath(p: string): string | null {
 export const vtRemoteIndex = (p: string): boolean =>
   vtPath(p) !== null && !/\s\s|[\t\v\f]|\s$/.test(p) && p.split(' ').every(w => w !== '' && !/^['"]/.test(w))
 
-/** SQL stages are double-quoted (gufi_vt's documented form), so they cannot
- *  contain a double quote. */
-const fitsVt = (plan: Plan) => Object.values(plan.stages).every(s => !s || !s.includes('"'))
+/** A SQL stage as gufi_vt can take it, or null. gufi_vt's stages are
+ *  double-quoted values (E="...") read verbatim, so the stage cannot hold a
+ *  double quote. Inside a single-quoted literal one is written as
+ *  '||char(34)||' instead, which gives the same string: an FTS5 phrase
+ *  query, MATCH '"a" AND "b"', becomes MATCH (''||char(34)||'a'||...).
+ *  A double quote anywhere else (a quoted identifier, a comment) cannot be
+ *  rewritten, and the stage stays with the per-library merge. */
+export function vtStage(sql: string): string | null {
+  let out = ''
+  for (let i = 0; i < sql.length; i++) {
+    const c = sql[i]
+    if (c === '"') return null
+    if (c === '-' && sql[i + 1] === '-') return sql.indexOf('"', i) < 0 ? out + sql.slice(i) : null
+    if (c === '/' && sql[i + 1] === '*') {
+      const end = sql.indexOf('*/', i + 2)
+      const comment = end < 0 ? sql.slice(i) : sql.slice(i, end + 2)
+      if (comment.includes('"')) return null
+      out += comment
+      i += comment.length - 1
+      continue
+    }
+    if (c !== "'") { out += c; continue }
+    // A single-quoted literal, '' being a quote inside it.
+    let j = i + 1
+    let body = ''
+    for (; j < sql.length; j++) {
+      if (sql[j] === "'") {
+        if (sql[j + 1] === "'") { body += "''"; j++; continue }
+        break
+      }
+      body += sql[j]
+    }
+    if (j >= sql.length) return null // unterminated: leave it to gufi_query's own error
+    out += body.includes('"') ? `('${body.split('"').join("'||char(34)||'")}')` : `'${body}'`
+    i = j
+  }
+  return out
+}
+
+const fitsVt = (plan: Plan) => Object.values(plan.stages).every(s => !s || vtStage(s) !== null)
 
 /** Where the wrapper reaches this server. */
 function selfUrl(): string {
@@ -128,7 +165,12 @@ export const viaShell = (r: NonNullable<VtLocation['remote']>) =>
 
 /** The script for a set of locations: one table each, then the union. */
 export function vtScript(locs: VtLocation[], order: VtOrder | null, remoteCmd = viaStudio): string {
-  const val = (k: string, v?: string) => (v ? `, ${k}="${v}"` : '')
+  const val = (k: string, v?: string) => {
+    if (!v) return ''
+    const stage = vtStage(v)
+    if (stage === null) throw new Error(`its ${k} stage has a double quote outside a string, which gufi_vt cannot take`)
+    return `, ${k}="${stage}"`
+  }
   const q = (v: string) => {
     const quoted = vtPath(v)
     if (!quoted) throw new Error(`${v} cannot be passed to gufi_vt`)

server/src/indexing.ts

@@ -17,7 +17,7 @@ import { indexSettings, meaningArgs, textArgs, threadsFor, type IndexSettings }
 import { sofficeFound } from './extractors.js'
 import { scheduleSemanticMapRebuild } from './semanticMap.js'
 import { scanLatestDetail } from './dirScan.js'
-import { deniedPaths, onlyDenied, recordDenied, setPrimaryIndexingSource } from './omissions.js'
+import { deniedPaths, onlyDenied, recordDenied, setPrimaryBuiltSource, setPrimaryIndexingSource } from './omissions.js'
 import { depthChange, depthRecordOf, readDepthRecord, writeDepthRecord, type DepthChange } from './depthState.js'
 
 /**
@@ -272,6 +272,10 @@ export function getIndexJob(id: number): IndexJob | null { return jobs.get(id)?.
 
 // Search and Query say the main folder is being built, not missing, while its first build runs.
 setPrimaryIndexingSource(() => current?.pub.kind === 'build')
+setPrimaryBuiltSource(() => {
+  const b = loadRuns().lastBuild
+  return !!b && (!b.root || b.root === KB_ROOT) && b.kind === 'build' && b.state === 'done'
+})
 
 /** Whether any index job is queued or running (an update waits for none). */
 export function indexJobsRunning(): boolean { return !!current || queue.length > 0 }

server/src/libraries.ts

@@ -286,6 +286,10 @@ async function probeNow(abs: string): Promise<LibraryCaps> {
   return caps
 }
 
+/** Whether the main folder's index has been built: GUFI is set up and its
+ *  index has a db.db at the root. */
+const primaryIndexed = () => gufiAvailable() && fs.existsSync(path.join(GUFI_INDEX, 'db.db'))
+
 function primaryLibrary(label: string): Library {
   return {
     id: PRIMARY_ID,
@@ -295,8 +299,10 @@ function primaryLibrary(label: string): Library {
     writable: true,
     primary: true,
     pinned: true,
-    // The main folder counts as indexed where GUFI is set up for it, as it always has.
-    caps: (c => ({ ...c, index: c.index || gufiAvailable() }))(capsOf(path.resolve(GUFI_INDEX), () => ({ index: gufiAvailable(), fullText: gufiAvailable(), vectors: false }))),
+    // The main folder is indexed once its index has a db.db at its root
+    // (indexing.ts's indexStatus says the same); an empty index folder, made
+    // before the first build, is not, so search and Query name it alike.
+    caps: (c => (primaryIndexed() ? { ...c, index: true } : { index: false, fullText: false, vectors: false }))(capsOf(path.resolve(GUFI_INDEX), () => ({ index: primaryIndexed(), fullText: primaryIndexed(), vectors: false }))),
     meaning: indexSettings(PRIMARY_ID).meaning,
     depth: indexSettings(PRIMARY_ID).depth,
   }

server/src/omissions.ts

@@ -116,6 +116,10 @@ const building = (l: Library) => l.build?.state === 'building' || l.build?.state
 /** Whether the main folder's index is being built (indexing.ts says). */
 let primaryIndexing: () => boolean = () => false
 export function setPrimaryIndexingSource(fn: () => boolean): void { primaryIndexing = fn }
+/** Whether the main folder's index was ever built (indexing.ts says), so
+ *  one with no index is told apart from one whose index has gone. */
+let primaryBuilt: () => boolean = () => false
+export function setPrimaryBuiltSource(fn: () => boolean): void { primaryBuilt = fn }
 
 /**
  * What a library's state leaves out of a search or a query, before
@@ -132,6 +136,7 @@ export function libraryGap(l: Library, want: { words: boolean; meaning: boolean
   if (!l.caps.index) {
     if (l.build?.state === 'building' || (l.primary && primaryIndexing())) return o('being built; searchable once its first pass is done', false)
     if (l.build?.state === 'failed') return o(`its build failed${l.build.error ? `: ${short(l.build.error)}` : ''}`, false)
+    if (l.primary && !primaryBuilt()) return o('not indexed yet', false)
     return o('index not found', false)
   }
   const reasons: string[] = []