diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt index bb21b9eb..68c867a9 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt @@ -51,6 +51,14 @@ import java.util.concurrent.atomic.AtomicInteger private val logger = Logger.withTag("LuceneSearchEngine") // Hard cap on how many synonym/expansion terms we allow per token private const val MAX_SYNONYM_TERMS_PER_TOKEN: Int = 32 + +// The forms of all a query's tokens together, per clause that lists them (the presence filter and each phrase query): +// a long query gets fewer forms per token, or it overflows Lucene's 1024-clause limit (TooManyNestedClauses) +private const val SYNONYM_TERMS_BUDGET: Int = 160 +private const val MIN_SYNONYM_TERMS_PER_TOKEN: Int = 4 + +// Room for a long query's synonym boosts, beyond Lucene's default 1024 clauses +private const val MAX_QUERY_CLAUSES: Int = 4096 // Global cap for boost queries built from dictionary expansions private const val MAX_SYNONYM_BOOST_TERMS: Int = 256 // Constants for snippet source building (must match indexer) @@ -170,6 +178,11 @@ class LuceneSearchEngine( private val dictionaryPath: Path? = null ) : SearchEngine { + init { + // A long query's synonym boosts (up to 256 forms of each kind) overflow Lucene's default 1024 clauses: room for them + if (IndexSearcher.getMaxClauseCount() < MAX_QUERY_CLAUSES) IndexSearcher.setMaxClauseCount(MAX_QUERY_CLAUSES) + } + // Open Lucene directory lazily to avoid any I/O at app startup. // GraalVM native image does not support MMapDirectory (Panama foreign downcalls), use NIOFSDirectory instead. private val dir by lazy { @@ -895,7 +908,7 @@ class LuceneSearchEngine( val outer = BooleanQuery.Builder() for (t in tokens) { val expansions = expansionsByToken[t] ?: emptyList() - val synonymTerms = buildLimitedTermsForToken(t, expansions) + val synonymTerms = buildLimitedTermsForToken(t, expansions, synonymTermsPerToken(tokens.size)) val ngram = if (near > 0) buildNgramPresenceForToken(t) else null val clause = BooleanQuery.Builder().apply { add(TermQuery(Term("text", t)), BooleanClause.Occur.SHOULD) @@ -1111,9 +1124,14 @@ class LuceneSearchEngine( return b.build() } + /** How many forms each of [tokenCount] tokens may bring: all of them up to five tokens, fewer beyond. */ + private fun synonymTermsPerToken(tokenCount: Int): Int = + (SYNONYM_TERMS_BUDGET / tokenCount.coerceAtLeast(1)).coerceIn(MIN_SYNONYM_TERMS_PER_TOKEN, MAX_SYNONYM_TERMS_PER_TOKEN) + private fun buildLimitedTermsForToken( token: String, - expansions: List + expansions: List, + cap: Int = MAX_SYNONYM_TERMS_PER_TOKEN, ): List { if (expansions.isEmpty()) return listOf(token) @@ -1128,7 +1146,7 @@ class LuceneSearchEngine( otherTerms.forEach { ordered += it } val totalSize = ordered.size - val limited = ordered.take(MAX_SYNONYM_TERMS_PER_TOKEN) + val limited = ordered.take(cap) if (totalSize > limited.size) { logger.d { "[DEBUG] Capped synonym terms for token '$token' from $totalSize to ${limited.size}" @@ -1144,7 +1162,7 @@ class LuceneSearchEngine( if (tokens.isEmpty()) return emptyList() return tokens.map { token -> val expansions = expansionsByToken[token] ?: emptyList() - buildLimitedTermsForToken(token, expansions) + buildLimitedTermsForToken(token, expansions, synonymTermsPerToken(tokens.size)) } }