From 715eb78f1ef8b902ff35d5200b090c22436a21f2 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Wed, 21 Oct 2026 20:33:28 +0300 Subject: [PATCH] fix(search): long queries no longer throw TooManyNestedClauses A query of many words (a whole sentence, e.g. a question) can fail with IndexSearcher.TooManyNestedClauses: each token brings up to 32 dictionary forms, listed in the presence filter and in each of the four phrase queries, and the synonym boosts add up to 256 forms of each kind, past Lucene's default limit of 1024 clauses. - The forms per token now share a budget across the query's tokens (unchanged up to five tokens, fewer beyond, at least four). - The engine raises Lucene's clause limit to 4096 for the boosts. Checked on 555 question-length queries (Hebrew questions of 6 to 20 words): none fails now, 230 ms on average; before, they overflowed. Signed-off-by: Elie Gambache --- .../search/LuceneSearchEngine.kt | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt index bb21b9eb..68c867a9 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt @@ -51,6 +51,14 @@ import java.util.concurrent.atomic.AtomicInteger private val logger = Logger.withTag("LuceneSearchEngine") // Hard cap on how many synonym/expansion terms we allow per token private const val MAX_SYNONYM_TERMS_PER_TOKEN: Int = 32 + +// The forms of all a query's tokens together, per clause that lists them (the presence filter and each phrase query): +// a long query gets fewer forms per token, or it overflows Lucene's 1024-clause limit (TooManyNestedClauses) +private const val SYNONYM_TERMS_BUDGET: Int = 160 +private const val MIN_SYNONYM_TERMS_PER_TOKEN: Int = 4 + +// Room for a long query's synonym boosts, beyond Lucene's default 1024 clauses +private const val MAX_QUERY_CLAUSES: Int = 4096 // Global cap for boost queries built from dictionary expansions private const val MAX_SYNONYM_BOOST_TERMS: Int = 256 // Constants for snippet source building (must match indexer) @@ -170,6 +178,11 @@ class LuceneSearchEngine( private val dictionaryPath: Path? = null ) : SearchEngine { + init { + // A long query's synonym boosts (up to 256 forms of each kind) overflow Lucene's default 1024 clauses: room for them + if (IndexSearcher.getMaxClauseCount() < MAX_QUERY_CLAUSES) IndexSearcher.setMaxClauseCount(MAX_QUERY_CLAUSES) + } + // Open Lucene directory lazily to avoid any I/O at app startup. // GraalVM native image does not support MMapDirectory (Panama foreign downcalls), use NIOFSDirectory instead. private val dir by lazy { @@ -895,7 +908,7 @@ class LuceneSearchEngine( val outer = BooleanQuery.Builder() for (t in tokens) { val expansions = expansionsByToken[t] ?: emptyList() - val synonymTerms = buildLimitedTermsForToken(t, expansions) + val synonymTerms = buildLimitedTermsForToken(t, expansions, synonymTermsPerToken(tokens.size)) val ngram = if (near > 0) buildNgramPresenceForToken(t) else null val clause = BooleanQuery.Builder().apply { add(TermQuery(Term("text", t)), BooleanClause.Occur.SHOULD) @@ -1111,9 +1124,14 @@ class LuceneSearchEngine( return b.build() } + /** How many forms each of [tokenCount] tokens may bring: all of them up to five tokens, fewer beyond. */ + private fun synonymTermsPerToken(tokenCount: Int): Int = + (SYNONYM_TERMS_BUDGET / tokenCount.coerceAtLeast(1)).coerceIn(MIN_SYNONYM_TERMS_PER_TOKEN, MAX_SYNONYM_TERMS_PER_TOKEN) + private fun buildLimitedTermsForToken( token: String, - expansions: List + expansions: List, + cap: Int = MAX_SYNONYM_TERMS_PER_TOKEN, ): List { if (expansions.isEmpty()) return listOf(token) @@ -1128,7 +1146,7 @@ class LuceneSearchEngine( otherTerms.forEach { ordered += it } val totalSize = ordered.size - val limited = ordered.take(MAX_SYNONYM_TERMS_PER_TOKEN) + val limited = ordered.take(cap) if (totalSize > limited.size) { logger.d { "[DEBUG] Capped synonym terms for token '$token' from $totalSize to ${limited.size}" @@ -1144,7 +1162,7 @@ class LuceneSearchEngine( if (tokens.isEmpty()) return emptyList() return tokens.map { token -> val expansions = expansionsByToken[token] ?: emptyList() - buildLimitedTermsForToken(token, expansions) + buildLimitedTermsForToken(token, expansions, synonymTermsPerToken(tokens.size)) } }