From 68b71b5f26eb35a57c488b6115ece809361eb9c9 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 11 Nov 2026 08:34:52 +0800 Subject: [PATCH 1/9] Functions: backport Local MATCH AGAINST to release-8.5 Use the Local MATCH protocol backported by pingcap/tipb#440. (cherry picked from commit 608fb566c33148518e2e6f4669beccb1c9a18025) Signed-off-by: Zhigao Tong --- contrib/tipb | 2 +- .../DAGExpressionAnalyzerHelper.cpp | 25 +- .../Coprocessor/DAGExpressionAnalyzerHelper.h | 5 + dbms/src/Flash/Coprocessor/DAGUtils.cpp | 1 + .../Functions/FunctionsLocalMatchAgainst.cpp | 1315 +++++++++++ .../Functions/FunctionsLocalMatchAgainst.h | 22 + .../Functions/LocalMatchAgainstTokenChars.h | 211 ++ dbms/src/Functions/registerFunctions.cpp | 2 + .../tests/gtest_local_match_against.cpp | 1918 +++++++++++++++++ 9 files changed, 3498 insertions(+), 3 deletions(-) create mode 100644 dbms/src/Functions/FunctionsLocalMatchAgainst.cpp create mode 100644 dbms/src/Functions/FunctionsLocalMatchAgainst.h create mode 100644 dbms/src/Functions/LocalMatchAgainstTokenChars.h create mode 100644 dbms/src/Functions/tests/gtest_local_match_against.cpp diff --git a/contrib/tipb b/contrib/tipb index 0607513e7fa..348fa94e545 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit 0607513e7fa40f3b564f0b269da76c1c8dc90032 +Subproject commit 348fa94e545a21e2b1a12752a7d0d1bbccea2efe diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp index 14f5e55c4bb..61cdcfc3d3b 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp @@ -328,7 +328,7 @@ String DAGExpressionAnalyzerHelper::buildSingleParamJsonRelatedFunctions( function_json_unquote) { bool valid_check - = !(isScalarFunctionExpr(input_expr) && input_expr.sig() == tipb::ScalarFuncSig::CastJsonAsString); + = !isScalarFunctionExpr(input_expr) || input_expr.sig() != tipb::ScalarFuncSig::CastJsonAsString; function_json_unquote->setNeedValidCheck(valid_check); } else if (auto * function_cast_json_as_string = dynamic_cast(function_impl); @@ -508,6 +508,26 @@ String DAGExpressionAnalyzerHelper::buildGroupingFunction( return result_name; } +String DAGExpressionAnalyzerHelper::buildLocalMatchAgainstBoolean( + DAGExpressionAnalyzer * analyzer, + const tipb::Expr & expr, + const ExpressionActionsPtr & actions) +{ + const String & func_name = getFunctionName(expr); + Names argument_names; + for (const auto & child : expr.children()) + { + String name = analyzer->getActions(child, actions); + argument_names.push_back(name); + } + // Expr.val carries Local MATCH metadata, not an SQL argument. Adapt it + // to the runtime function's required final constant argument. Keeping it + // in argument_names also distinguishes actions with different metadata. + const auto metadata_expr = constructStringLiteralTiExpr(expr.val()); + argument_names.push_back(analyzer->getActions(metadata_expr, actions)); + return analyzer->applyFunction(func_name, argument_names, actions, getCollatorFromExpr(expr)); +} + String DAGExpressionAnalyzerHelper::buildDefaultFunction( DAGExpressionAnalyzer * analyzer, const tipb::Expr & expr, @@ -570,6 +590,7 @@ DAGExpressionAnalyzerHelper::FunctionBuilderMap DAGExpressionAnalyzerHelper::fun {"regexp", DAGExpressionAnalyzerHelper::buildRegexpFunction}, {"replaceRegexpAll", DAGExpressionAnalyzerHelper::buildRegexpFunction}, {"tidbRound", DAGExpressionAnalyzerHelper::buildRoundFunction}, - {"grouping", DAGExpressionAnalyzerHelper::buildGroupingFunction}}); + {"grouping", DAGExpressionAnalyzerHelper::buildGroupingFunction}, + {"local_match_against_boolean", DAGExpressionAnalyzerHelper::buildLocalMatchAgainstBoolean}}); } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.h b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.h index 529dc204ace..3f59e972637 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.h +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.h @@ -97,6 +97,11 @@ class DAGExpressionAnalyzerHelper const tipb::Expr & expr, const ExpressionActionsPtr & actions); + static String buildLocalMatchAgainstBoolean( + DAGExpressionAnalyzer * analyzer, + const tipb::Expr & expr, + const ExpressionActionsPtr & actions); + static String buildDefaultFunction( DAGExpressionAnalyzer * analyzer, const tipb::Expr & expr, diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index efacecc9b93..79abe601636 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -455,6 +455,7 @@ const std::unordered_map scalar_func_map({ {tipb::ScalarFuncSig::RegexpInStrSig, "regexp_instr"}, {tipb::ScalarFuncSig::RegexpReplaceSig, "regexp_replace"}, {tipb::ScalarFuncSig::RegexpSubstrSig, "regexp_substr"}, + {tipb::ScalarFuncSig::LocalMatchAgainstBoolean, "local_match_against_boolean"}, {tipb::ScalarFuncSig::JsonExtractSig, "json_extract"}, {tipb::ScalarFuncSig::JsonUnquoteSig, "json_unquote"}, diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp new file mode 100644 index 00000000000..39f827553b2 --- /dev/null +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -0,0 +1,1315 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_COLUMN; +} + +namespace +{ +constexpr size_t default_min_token_size = 3; +constexpr size_t default_max_token_size = 84; +constexpr size_t default_ngram_token_size = 2; +// This is the Local MATCH semantic protocol version, not the TiDB/TiFlash +// product version. Version 2 fixes the Boolean AST interpretation, analyzer +// behavior, and built-in stopword set. TiFlash must support a version before +// TiDB emits it; unknown versions are rejected, and existing versions must +// never be reinterpreted with changed semantics. +// Version 2 returns zero for NULL searches and uses MySQL's multibyte NGRAM +// document scanner. STANDARD and the NGRAM Boolean lexer use BMP word +// characters; filtered phrases also verify the unfiltered document stream. +constexpr UInt32 local_match_against_protocol_version = 2; + +struct FullTextToken +{ + // Query terms and the no-collator lowercase path own their spelling. + // Document views refer only to the current input column, which outlives + // both analyzed streams and matching. Never retain them across blocks. + String text; + std::string_view source; + size_t position = 0; + size_t code_points = 0; + + std::string_view value() const { return source.data() ? source : std::string_view{text}; } +}; + +class FullTextColumn +{ +public: + using Container = std::vector; + using iterator = Container::iterator; + using const_iterator = Container::const_iterator; + + void clear() { used = 0; } + + void reserve(size_t size) { tokens.reserve(size); } + + // This is per-token work. Keep it inline even when the surrounding + // analyzer grows: outlining it penalizes the unfiltered/OFF stream too. + ALWAYS_INLINE void append(std::string_view text, size_t position, size_t code_points, bool borrow = false) + { + if (used == tokens.size()) + tokens.emplace_back(); + auto & token = tokens[used++]; + if (borrow) + token.source = text; + else + { + token.source = {}; + token.text.assign(text.data(), text.size()); + } + token.position = position; + token.code_points = code_points; + } + + template + void filterInPlace(Predicate predicate) + { + size_t write = 0; + for (size_t read = 0; read < used; ++read) + { + if (!predicate(tokens[read])) + continue; + if (write != read) + std::swap(tokens[write], tokens[read]); + ++write; + } + used = write; + } + + bool empty() const { return used == 0; } + size_t size() const { return used; } + FullTextToken & front() { return tokens.front(); } + const FullTextToken & front() const { return tokens.front(); } + FullTextToken & operator[](size_t index) { return tokens[index]; } + const FullTextToken & operator[](size_t index) const { return tokens[index]; } + iterator begin() { return tokens.begin(); } + iterator end() { return tokens.begin() + used; } + const_iterator begin() const { return tokens.begin(); } + const_iterator end() const { return tokens.begin() + used; } + +private: + Container tokens; + size_t used = 0; +}; + +struct BooleanClause +{ + enum class Modifier + { + Should, + Must, + MustNot, + }; + + Modifier modifier = Modifier::Should; + bool phrase = false; + bool prefix = false; + std::vector terms; + std::vector offsets; + std::unique_ptr verification; + std::vector> prefix_matchers; +}; + +struct AnalyzerScratch +{ + std::vector char_boundaries; + String lowercase_run; +}; + +struct CompiledBooleanQuery +{ + std::vector clauses; + size_t must_clause_count = 0; + bool has_must_not = false; + bool matches_nothing = false; + bool needs_verification = false; +}; + +// Normalize the already validated wire configuration once per block. Keep +// the tokenizer implementations and their filtered/unfiltered rules intact. +struct LocalMatchAnalyzerConfig +{ + bool use_ngram; + size_t ngram_token_size; + size_t min_token_size; + size_t max_token_size; + bool enable_stopword; + + explicit LocalMatchAnalyzerConfig(const tipb::LocalMatchAgainstBooleanQuery & query) + : use_ngram(query.parser() == tipb::LocalMatchAgainstParserNgram) + , ngram_token_size( + !use_ngram || query.ngram_token_size() == 0 ? default_ngram_token_size : query.ngram_token_size()) + , min_token_size( + query.innodb_ft_min_token_size() != 0 || query.innodb_ft_max_token_size() != 0 + ? query.innodb_ft_min_token_size() + : default_min_token_size) + , max_token_size( + query.innodb_ft_min_token_size() != 0 || query.innodb_ft_max_token_size() != 0 + ? query.innodb_ft_max_token_size() + : default_max_token_size) + , enable_stopword(query.stopword_mode() == tipb::LocalMatchAgainstStopwordModeBuiltin) + {} +}; + +// Query tokens own their text. This cold-path helper only collects the same +// terms and source-relative offsets; document views never enter it. +size_t collectClauseTerms(BooleanClause & clause, const FullTextColumn & tokens) +{ + const size_t first_position = tokens.empty() ? 0 : tokens.front().position; + for (const auto & token : tokens) + { + clause.terms.push_back(token.text); + if (clause.phrase) + clause.offsets.push_back(token.position - first_position); + } + return first_position; +} + +void lowercaseTokenInPlace(String & token) +{ + const bool is_ascii = std::all_of(token.begin(), token.end(), [](unsigned char c) { return c < 0x80; }); + if (is_ascii) + { + for (char & c : token) + if (c >= 'A' && c <= 'Z') + c = static_cast(c + ('a' - 'A')); + return; + } + Poco::UTF8::toLowerInPlace(token); +} + +bool isFullTextToken(UInt32 code_point) +{ + return code_point <= 0xFFFF && LocalMatchAgainst::isTokenChar(code_point); +} + +std::pair decodeCodePoint(std::string_view text, size_t offset) +{ + // Most document bytes are ASCII. Keep the multibyte/invalid-byte + // semantics below, but avoid calling the generic decoder for ASCII. + const auto byte = static_cast(text[offset]); + if (byte < 0x80) + return {byte, 1}; + const auto decoded = UTF8::utf8Decode(text.data() + offset, text.size() - offset); + if (decoded.second == 0 || decoded.first == UTF8::UTF8_Error || decoded.second > text.size() - offset) + return {0xFFFD, 1}; // Go's utf8.DecodeRuneInString also consumes one invalid byte as RuneError. + return {decoded.first, decoded.second}; +} + +template +void scanStandardTokens(std::string_view text, Consumer && consume) +{ + size_t position = 0; + for (size_t i = 0; i < text.size();) + { + const auto [code_point, length] = decodeCodePoint(text, i); + if (!isFullTextToken(code_point)) + { + i += length; + continue; + } + + const size_t token_start = i; + // The first character has already been decoded and classified. + size_t code_points = 1; + i += length; + while (i < text.size()) + { + const auto [code_point, length] = decodeCodePoint(text, i); + if (!isFullTextToken(code_point)) + break; + ++code_points; + i += length; + } + + // Positions include source tokens subsequently removed by filtering. + consume(text.substr(token_start, i - token_start), position++, code_points); + } +} + +void tokenizeTextInto(std::string_view text, FullTextColumn & result, const TiDB::TiDBCollatorPtr & collator = nullptr) +{ + result.clear(); + scanStandardTokens(text, [&](std::string_view token, size_t position, size_t code_points) { + result.append(token, position, code_points); + if (!collator) + lowercaseTokenInPlace(result[result.size() - 1].text); + }); +} + +FullTextColumn tokenizeText(std::string_view text, const TiDB::TiDBCollatorPtr & collator = nullptr) +{ + FullTextColumn result; + tokenizeTextInto(text, result, collator); + return result; +} + +const std::unordered_set & defaultStopwords() +{ + static const std::unordered_set stopwords{ + "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", + "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", + "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; + return stopwords; +} + +// Owned collation keys are compiled once per executeImpl, then reused by +// query analysis and every document in the block. The scratch key is local +// to that execution, never shared across concurrent calls. Use the stopword +// (server) collation, not the MATCH column collation. +class BuiltinStopwordLookup +{ +public: + explicit BuiltinStopwordLookup(TiDB::TiDBCollatorPtr collator_) + : collator(collator_) + { + for (const auto & stopword : defaultStopwords()) + { + const auto key = collator->sortKeyFastPath(stopword.data(), stopword.size(), key_buffer); + // sortKeyFastPath may reference its input or key_buffer; copy it. + String owned_key(key.data, key.size); + keys.insert(owned_key); + // The built-in list is ASCII. Preserve NGRAM's code-point length + // buckets even for collations with expansions/ignorable weights. + keys_by_length[stopword.size()].insert(std::move(owned_key)); + } + } + + bool contains(std::string_view token) const { return contains(token, keys); } + + bool contains(std::string_view token, size_t code_points) const + { + const auto bucket = keys_by_length.find(code_points); + return bucket != keys_by_length.end() && contains(token, bucket->second); + } + +private: + struct KeyHash + { + using is_transparent = void; + size_t operator()(std::string_view key) const { return std::hash{}(key); } + }; + using Keys = std::unordered_set>; + + bool contains(std::string_view token, const Keys & lookup) const + { + // sortKey (rather than sortKeyNoTrim) preserves PAD SPACE/NO PAD + // equality, as well as the collator's case/accent semantics. + const auto key = collator->sortKeyFastPath(token.data(), token.size(), key_buffer); + return lookup.contains(std::string_view(key.data, key.size)); + } + + TiDB::TiDBCollatorPtr collator; + Keys keys; + std::unordered_map keys_by_length; + mutable String key_buffer; +}; + +bool isDefaultStopword(std::string_view token, const BuiltinStopwordLookup * stopwords) +{ + return stopwords ? stopwords->contains(token) : defaultStopwords().contains(Poco::UTF8::toLower(String(token))); +} + +size_t maxBuiltinStopwordLength() +{ + // The supported built-in list is ASCII: byte and code-point lengths are + // identical. Do not examine lengths which cannot match any stopword. + static const size_t max_length = [] { + size_t result = 0; + for (const auto & word : defaultStopwords()) + result = std::max(result, word.size()); + return result; + }(); + return max_length; +} + +bool containsNgramStopword( + std::string_view token, + const std::vector & char_boundaries, + size_t ngram_start, + size_t ngram_token_size, + const BuiltinStopwordLookup * stopwords) +{ + const size_t ngram_end = ngram_start + ngram_token_size; + for (size_t start = ngram_start; start < ngram_end; ++start) + { + for (size_t end = start + 1; end <= ngram_end; ++end) + { + const auto candidate = token.substr(char_boundaries[start], char_boundaries[end] - char_boundaries[start]); + if (!stopwords && defaultStopwords().contains(Poco::UTF8::toLower(String(candidate)))) + return true; + if (stopwords && stopwords->contains(candidate, end - start)) + return true; + } + } + return false; +} + +void analyzeTextInto( + std::string_view text, + FullTextColumn & result, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true, + bool borrow_document = false) +{ + result.clear(); + scanStandardTokens(text, [&](std::string_view token, size_t position, size_t code_points) { + // Filter before materializing. Stopwords use the source spelling and + // server collation, independently of the MATCH column collation. + if (code_points < min_token_size || code_points > max_token_size + || (enable_stopword && isDefaultStopword(token, stopwords))) + return; + result.append(token, position, code_points, borrow_document && collator); + if (!collator) + lowercaseTokenInPlace(result[result.size() - 1].text); + }); +} + +FullTextColumn analyzeText( + std::string_view text, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true) +{ + FullTextColumn result; + analyzeTextInto(text, result, collator, stopwords, min_token_size, max_token_size, enable_stopword); + return result; +} + +void analyzeNgramTextInto( + std::string_view text, + FullTextColumn & result, + AnalyzerScratch & scratch, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + bool enable_stopword = true, + bool borrow_document = false) +{ + result.clear(); + if (ngram_token_size == 0) + return; + + const bool preserve_case = enable_stopword && stopwords; + const size_t max_stopword_length = enable_stopword ? std::min(ngram_token_size, maxBuiltinStopwordLength()) : 0; + size_t next_position_base = 0; + auto & char_boundaries = scratch.char_boundaries; + for (size_t offset = 0; offset < text.size();) + { + const auto [first_code_point, first_length] = decodeCodePoint(text, offset); + // Stop the document on malformed UTF-8, retain all valid multibyte + // characters, and split runs on ASCII nonwords, as MySQL does. + if (first_length == 1 && first_code_point >= 0x80) + break; + if (first_length == 1 && !isFullTextToken(first_code_point)) + { + offset += first_length; + continue; + } + const size_t run_start = offset; + char_boundaries.clear(); + char_boundaries.push_back(0); + offset += first_length; + char_boundaries.push_back(offset - run_start); + while (offset < text.size()) + { + const auto [code_point, length] = decodeCodePoint(text, offset); + if (length == 1 && (code_point >= 0x80 || !isFullTextToken(code_point))) + break; + offset += length; + char_boundaries.push_back(offset - run_start); + } + + std::string_view run = text.substr(run_start, offset - run_start); + const size_t char_count = char_boundaries.size() - 1; + const size_t base_position = next_position_base; + if (char_count < ngram_token_size) + { + // A short document run emits no grams and contributes no position. + continue; + } + + if (!collator && !preserve_case) + { + // The no-collator path lowercases a whole run before splitting. + // Unicode case mapping can change byte widths, so only that path + // needs a copy and another boundary scan. Never borrow scratch. + scratch.lowercase_run.assign(run.data(), run.size()); + lowercaseTokenInPlace(scratch.lowercase_run); + run = scratch.lowercase_run; + char_boundaries.clear(); + char_boundaries.push_back(0); + for (size_t i = 0; i < run.size();) + { + i += decodeCodePoint(run, i).second; + char_boundaries.push_back(i); + } + } + + const auto append_gram = [&](size_t start) ALWAYS_INLINE { + const size_t begin = char_boundaries[start]; + const size_t end = char_boundaries[start + ngram_token_size]; + result.append( + run.substr(begin, end - begin), + base_position + start, + ngram_token_size, + borrow_document && collator); + if (!collator && preserve_case) + lowercaseTokenInPlace(result[result.size() - 1].text); + }; + + // Keep the unfiltered stream and stopwords-OFF path free of per-gram + // stopword branches and sliding-state dependencies. + if (!enable_stopword) + { + for (size_t start = 0; start + ngram_token_size <= char_count; ++start) + append_gram(start); + next_position_base = base_position + char_count - ngram_token_size + 1; + continue; + } + + // Advance candidate endings once per run, instead of enumerating all + // substrings again for every overlapping gram. Keep one past the + // greatest start of a matched stopword ending at/before the gram end. + // The gram [start, end) is filtered iff that value is greater than + // start: such a stopword lies wholly inside the current gram. Once + // start advances beyond it, it expires without a queue or cache. + size_t stopword_end = 0; + size_t filtered_until_start = 0; + for (size_t start = 0; start + ngram_token_size <= char_count; ++start) + { + while (stopword_end < start + ngram_token_size) + { + ++stopword_end; + const size_t max_length = std::min(stopword_end, max_stopword_length); + for (size_t length = 1; length <= max_length; ++length) + { + const size_t candidate_start = stopword_end - length; + // Longer candidates at this ending have earlier starts + // and cannot improve a start already known to match. + if (candidate_start < filtered_until_start) + break; + const auto candidate = run.substr( + char_boundaries[candidate_start], + char_boundaries[stopword_end] - char_boundaries[candidate_start]); + const bool matched = stopwords + ? stopwords->contains(candidate, length) + : defaultStopwords().contains(Poco::UTF8::toLower(String(candidate))); + if (matched) + { + filtered_until_start = candidate_start + 1; + break; + } + } + } + if (filtered_until_start <= start) + append_gram(start); + } + next_position_base = base_position + char_count - ngram_token_size + 1; + } +} + +FullTextColumn analyzeNgramText( + std::string_view text, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + bool enable_stopword = true) +{ + FullTextColumn result; + AnalyzerScratch scratch; + analyzeNgramTextInto(text, result, scratch, ngram_token_size, collator, stopwords, enable_stopword); + return result; +} + +bool textEquals(std::string_view lhs, std::string_view rhs, const TiDB::TiDBCollatorPtr & collator) +{ + if (!collator) + return lhs == rhs; + return collator->compare(lhs.data(), lhs.size(), rhs.data(), rhs.size()) == 0; +} + +String makePrefixPattern(std::string_view prefix) +{ + // Reuse the same collation-aware pattern implementation as LIKE. Escape + // token characters that have LIKE meaning before appending the wildcard. + String pattern; + pattern.reserve(prefix.size() + 1); + for (const char c : prefix) + { + if (c == '\\' || c == '%' || c == '_') + pattern.push_back('\\'); + pattern.push_back(c); + } + pattern.push_back('%'); + return pattern; +} + +std::unique_ptr compilePrefixMatcher( + std::string_view prefix, + const TiDB::TiDBCollatorPtr & collator) +{ + auto matcher = collator->pattern(); + matcher->compile(makePrefixPattern(prefix), '\\'); + return matcher; +} + +bool textStartsWith(std::string_view value, std::string_view prefix, const TiDB::TiDBCollatorPtr & collator) +{ + if (!collator) + return value.starts_with(prefix); + const auto matcher = compilePrefixMatcher(prefix, collator); + return matcher->match(value.data(), value.size()); +} + +bool matchesTermAt( + const BooleanClause & clause, + size_t term_index, + const FullTextToken & token, + const TiDB::TiDBCollatorPtr & collator) +{ + const auto value = token.value(); + if (!clause.prefix) + return textEquals(value, clause.terms[term_index], collator); + if (collator && term_index < clause.prefix_matchers.size()) + return clause.prefix_matchers[term_index]->match(value.data(), value.size()); + return textStartsWith(value, clause.terms[term_index], collator); +} + +struct ClauseMatchState +{ + std::vector matched_terms; + size_t remaining_terms = 0; + bool matched = false; + bool phrase_verification_failed = false; +}; + +void initializeClauseMatchStates(const std::vector & clauses, std::vector & states) +{ + states.resize(clauses.size()); + for (size_t i = 0; i < clauses.size(); ++i) + if (!clauses[i].phrase) + { + states[i].matched_terms.resize(clauses[i].terms.size()); + states[i].remaining_terms = clauses[i].terms.size(); + } +} + +bool matchesPhraseEndingAt( + const BooleanClause & clause, + const FullTextColumn & column, + const FullTextToken & end, + const TiDB::TiDBCollatorPtr & collator) +{ + if (clause.terms.empty() || clause.terms.size() != clause.offsets.size()) + return false; + + const size_t last_term = clause.terms.size() - 1; + const size_t last_offset = clause.offsets[last_term]; + if (end.position < last_offset || !matchesTermAt(clause, last_term, end, collator)) + return false; + + const size_t start_position = end.position - last_offset; + for (size_t i = 0; i < last_term; ++i) + { + const size_t expected_position = start_position + clause.offsets[i]; + const auto it = std::lower_bound( + column.begin(), + column.end(), + expected_position, + [](const FullTextToken & token, size_t position) { return token.position < position; }); + if (it == column.end() || it->position != expected_position + || !textEquals(it->value(), clause.terms[i], collator)) + return false; + } + return true; +} + +void analyzeColumnInto( + std::string_view document, + FullTextColumn & output, + AnalyzerScratch & scratch, + bool use_ngram, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true) +{ + if (use_ngram) + analyzeNgramTextInto(document, output, scratch, ngram_token_size, collator, stopwords, enable_stopword, true); + else + analyzeTextInto(document, output, collator, stopwords, min_token_size, max_token_size, enable_stopword, true); +} + +void resetClauseMatchStates(std::vector & states) +{ + for (auto & state : states) + { + state.matched = false; + state.remaining_terms = state.matched_terms.size(); + std::fill(state.matched_terms.begin(), state.matched_terms.end(), 0); + } +} + +struct BooleanMatchProgress +{ + size_t matched_must_clauses = 0; + bool matched_should_clause = false; +}; + +enum class ColumnMatchResult +{ + Continue, + Accepted, + Rejected, +}; + +ColumnMatchResult updateBooleanMatchStatesForColumn( + const CompiledBooleanQuery & query, + const FullTextColumn & column, + const FullTextColumn & unfiltered_column, + const TiDB::TiDBCollatorPtr & collator, + std::vector & states, + BooleanMatchProgress & progress) +{ + const auto & clauses = query.clauses; + if (query.matches_nothing || clauses.empty()) + return ColumnMatchResult::Continue; + + // Raw phrase verification searches the entire column, so a failed result + // needs no recheck at later anchors. Unlike positive clause matches, this + // negative cache belongs only to this column, not other MATCH columns. + for (auto & state : states) + state.phrase_verification_failed = false; + + const auto mark_clause_matched = [&](size_t clause_index) { + states[clause_index].matched = true; + switch (clauses[clause_index].modifier) + { + case BooleanClause::Modifier::Must: + ++progress.matched_must_clauses; + break; + case BooleanClause::Modifier::Should: + progress.matched_should_clause = true; + break; + case BooleanClause::Modifier::MustNot: + return ColumnMatchResult::Rejected; + } + if (!query.has_must_not + && (query.must_clause_count != 0 ? progress.matched_must_clauses == query.must_clause_count + : progress.matched_should_clause)) + return ColumnMatchResult::Accepted; + return ColumnMatchResult::Continue; + }; + + // Walk this MATCH column's analyzed tokens once and update every clause's + // state. Phrase clauses are checked only when their final term is + // encountered; positional lookups preserve stopword gaps and phrases do + // not cross MATCH columns. + for (const auto & token : column) + { + for (size_t clause_index = 0; clause_index < clauses.size(); ++clause_index) + { + const auto & clause = clauses[clause_index]; + auto & state = states[clause_index]; + if (state.matched || state.phrase_verification_failed || clause.terms.empty()) + continue; + + if (clause.phrase) + { + if (matchesPhraseEndingAt(clause, column, token, collator)) + { + if (clause.verification + && !std::any_of(unfiltered_column.begin(), unfiltered_column.end(), [&](const auto & end) { + return matchesPhraseEndingAt(*clause.verification, unfiltered_column, end, collator); + })) + { + state.phrase_verification_failed = true; + continue; + } + const auto result = mark_clause_matched(clause_index); + if (result != ColumnMatchResult::Continue) + return result; + } + continue; + } + + for (size_t term_index = 0; term_index < clause.terms.size(); ++term_index) + if (!state.matched_terms[term_index] && matchesTermAt(clause, term_index, token, collator)) + { + state.matched_terms[term_index] = 1; + --state.remaining_terms; + } + // STANDARD may split one SQL term into several words. Required + // terms intersect those words, but optional/prohibited terms use + // their union, exactly like TiDB's combineBooleanTermNodes. + state.matched = clause.modifier == BooleanClause::Modifier::Must + ? state.remaining_terms == 0 + : state.remaining_terms < clause.terms.size(); + if (state.matched) + { + const auto result = mark_clause_matched(clause_index); + if (result != ColumnMatchResult::Continue) + return result; + } + } + } + return ColumnMatchResult::Continue; +} + +bool evaluateBooleanMatchResult(const CompiledBooleanQuery & query, const std::vector & states) +{ + if (query.matches_nothing || query.clauses.empty()) + return false; + bool has_positive = false; + bool has_must = false; + bool positive_match = false; + for (size_t i = 0; i < query.clauses.size(); ++i) + { + const auto & clause = query.clauses[i]; + const bool matched = states[i].matched; + switch (clause.modifier) + { + case BooleanClause::Modifier::Must: + has_must = true; + if (!matched) + return false; + break; + case BooleanClause::Modifier::MustNot: + if (matched) + return false; + break; + case BooleanClause::Modifier::Should: + has_positive = true; + positive_match = positive_match || matched; + break; + } + } + + return has_must || !has_positive || positive_match; +} + +CompiledBooleanQuery compileBooleanQuery( + const tipb::LocalMatchAgainstBooleanQuery & query, + const LocalMatchAnalyzerConfig & config, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr) +{ + CompiledBooleanQuery compiled; + if (!config.use_ngram && (config.max_token_size == 0 || config.min_token_size > config.max_token_size)) + { + compiled.matches_nothing = true; + return compiled; + } + auto analyze_query = [&](std::string_view text) { + return config.use_ngram + ? analyzeNgramText(text, config.ngram_token_size, collator, stopwords, config.enable_stopword) + : analyzeText( + text, + collator, + stopwords, + config.min_token_size, + config.max_token_size, + config.enable_stopword); + }; + for (const auto & node : query.nodes()) + { + BooleanClause clause; + size_t first_query_position = 0; + switch (node.occur()) + { + case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurMust: + clause.modifier = BooleanClause::Modifier::Must; + break; + case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurMustNot: + clause.modifier = BooleanClause::Modifier::MustNot; + break; + case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurShould: + clause.modifier = BooleanClause::Modifier::Should; + break; + default: + throw Exception("Invalid Local MATCH Boolean node occurrence", ErrorCodes::ILLEGAL_COLUMN); + } + + switch (node.term_type()) + { + case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermWord: + { + const auto terms = analyze_query(node.text()); + clause.phrase = config.use_ngram; + first_query_position = collectClauseTerms(clause, terms); + break; + } + case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermPrefix: + { + const auto terms = config.use_ngram + ? analyzeNgramText(node.text(), config.ngram_token_size, collator, stopwords, config.enable_stopword) + : tokenizeText(node.text(), collator); + if (config.use_ngram) + { + if (!terms.empty()) + { + // TiDB expands an NGRAM prefix at least as long as one + // gram into a positional phrase of query ngrams. Matching + // only one analyzed term here makes prefixes such as + // "caf*" ("ca", "af" at token size 2) never match. + clause.phrase = true; + first_query_position = collectClauseTerms(clause, terms); + break; + } + + // A query shorter than the configured gram size is retained + // by TiDB as a prefix over document ngrams instead of being + // discarded by the analyzer. + const auto source_terms = tokenizeText(node.text(), collator); + if (source_terms.size() == 1) + { + const auto code_points = UTF8::countCodePoints( + reinterpret_cast(source_terms.front().text.data()), + source_terms.front().text.size()); + if (code_points < config.ngram_token_size) + { + clause.prefix = true; + clause.terms.push_back(source_terms.front().text); + break; + } + } + + if (clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + continue; + } + + clause.prefix = true; + if (terms.size() != 1) + { + if (clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + continue; + } + const auto code_points = UTF8::countCodePoints( + reinterpret_cast(terms.front().text.data()), + terms.front().text.size()); + if (code_points > config.max_token_size) + { + if (clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + continue; + } + clause.terms.push_back(terms.front().text); + break; + } + case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermPhrase: + { + clause.phrase = true; + const auto terms = analyze_query(node.text()); + if (config.use_ngram) + { + const auto words = tokenizeText(node.text(), collator); + bool seen_indexed_word = false; + if (std::any_of(words.begin(), words.end(), [&](const auto & word) { + if (word.code_points >= config.ngram_token_size) + { + seen_indexed_word = seen_indexed_word || !analyze_query(word.text).empty(); + return false; + } + if (!config.enable_stopword || seen_indexed_word) + return true; + std::vector boundaries{0}; + for (size_t offset = 0; offset < word.text.size();) + { + offset += decodeCodePoint(word.text, offset).second; + boundaries.push_back(offset); + } + return !containsNgramStopword(word.text, boundaries, 0, word.code_points, stopwords); + })) + break; // A short query unigram cannot occur in the document's fixed-size ngrams. + } + first_query_position = collectClauseTerms(clause, terms); + break; + } + default: + throw Exception("Invalid Local MATCH Boolean node term type", ErrorCodes::ILLEGAL_COLUMN); + } + + if (clause.phrase && !clause.terms.empty()) + { + auto raw_terms = config.use_ngram + ? analyzeNgramText(node.text(), config.ngram_token_size, collator, nullptr, false) + : analyzeText(node.text(), collator, nullptr, 0, std::numeric_limits::max(), false); + // InnoDB discards leading filtered tokens, but retains every + // following word in the original phrase used for verification. + raw_terms.filterInPlace([&](const auto & token) { return token.position >= first_query_position; }); + if (raw_terms.size() != clause.terms.size()) + { + // InnoDB verifies phrases against the original document, not + // just the index tokens: removing "the" must not turn it into + // an arbitrary positional gap. Ordinary terms/prefixes do not + // pay for this additional document stream. + clause.verification = std::make_unique(); + clause.verification->phrase = true; + collectClauseTerms(*clause.verification, raw_terms); + compiled.needs_verification = true; + } + } + + if (clause.prefix && collator) + { + clause.prefix_matchers.reserve(clause.terms.size()); + for (const auto & term : clause.terms) + clause.prefix_matchers.push_back(compilePrefixMatcher(term, collator)); + } + + if (!clause.terms.empty() || clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + } + + for (const auto & clause : compiled.clauses) + { + if (clause.modifier == BooleanClause::Modifier::Must) + { + // An analyzer-filtered required clause cannot match any document. + // Do this after compiling ALL nodes so malformed later nodes + // retain their existing no-match handling, rather than being + // hidden by an earlier block-level shortcut. + if (clause.terms.empty()) + { + compiled.matches_nothing = true; + return compiled; + } + ++compiled.must_clause_count; + } + else if (clause.modifier == BooleanClause::Modifier::MustNot) + compiled.has_must_not = true; + } + + // A BOOLEAN MODE query containing only prohibited terms has no positive + // branch. TiDB's local evaluator treats it as matching no rows. + if (std::none_of(compiled.clauses.begin(), compiled.clauses.end(), [](const BooleanClause & clause) { + return clause.modifier != BooleanClause::Modifier::MustNot; + })) + compiled.matches_nothing = true; + + return compiled; +} + +Float64 matchBooleanValue(const CompiledBooleanQuery & query, const std::vector & states) +{ + // Local MATCH returns a Boolean 0/1 value, not a relevance score. + return evaluateBooleanMatchResult(query, states) ? 1 : 0; +} + +class FullTextColumnAccessor +{ +public: + explicit FullTextColumnAccessor(const IColumn & column) { initialize(column, false); } + + bool isNull(size_t row) const + { + return nullable_column != nullptr && nullable_column->getNullMapData()[nullable_is_constant ? 0 : row] != 0; + } + + std::string_view getString(size_t row) const + { + const auto & chars = string_column->getChars(); + const auto & offsets = string_column->getOffsets(); + const size_t string_row = string_is_constant ? 0 : row; + const size_t begin = string_row == 0 ? 0 : offsets[string_row - 1]; + const size_t end = offsets[string_row]; + return {reinterpret_cast(&chars[begin]), end - begin - 1}; + } + +private: + void initialize(const IColumn & column, bool row_is_constant) + { + if (const auto * column_const = typeid_cast(&column)) + { + string_is_constant = true; + initialize(column_const->getDataColumn(), true); + return; + } + if (const auto * column_nullable = typeid_cast(&column)) + { + nullable_column = column_nullable; + nullable_is_constant = row_is_constant; + initialize(column_nullable->getNestedColumn(), row_is_constant); + return; + } + string_column = checkAndGetColumn(&column); + if (string_column == nullptr) + throw Exception("Full-text arguments must be string columns", ErrorCodes::ILLEGAL_COLUMN); + } + + const ColumnString * string_column = nullptr; + const ColumnNullable * nullable_column = nullptr; + bool string_is_constant = false; + bool nullable_is_constant = false; +}; + +bool decodeLocalMatchAgainstBooleanQuery(const IColumn & column, tipb::LocalMatchAgainstBooleanQuery & query) +{ + const auto * constant = typeid_cast(&column); + if (constant == nullptr) + return false; + const auto encoded = constant->getValue(); + // Expr.val directly contains the versioned Local MATCH query. Validate the + // semantic version before interpreting parser or stopword fields; never + // guess a default for a version this TiFlash binary does not understand. + if (!query.ParseFromString(encoded) || query.version() != local_match_against_protocol_version) + return false; + if (query.parser() != tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard + && query.parser() != tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram) + return false; + if (query.stopword_mode() != tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled + && query.stopword_mode() != tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin) + return false; + // Validate every node before NULL-search or analyzer no-match shortcuts. + // Missing/unknown proto2 enum values read as Invalid, not as valid clauses. + for (const auto & node : query.nodes()) + { + if (node.occur() != tipb::LocalMatchAgainstBooleanOccurMust + && node.occur() != tipb::LocalMatchAgainstBooleanOccurMustNot + && node.occur() != tipb::LocalMatchAgainstBooleanOccurShould) + return false; + if (node.term_type() != tipb::LocalMatchAgainstBooleanTermWord + && node.term_type() != tipb::LocalMatchAgainstBooleanTermPrefix + && node.term_type() != tipb::LocalMatchAgainstBooleanTermPhrase) + return false; + } + return true; +} + +class FunctionLocalMatchAgainstBoolean final : public IFunction +{ +public: + static constexpr auto name = "local_match_against_boolean"; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + bool useDefaultImplementationForConstants() const override { return false; } + // NULL MATCH columns are empty documents, not NULL MATCH results. The + // implementation below handles nullable columns per row, so the generic + // IFunction NULL wrapper must not short-circuit the whole expression. + bool useDefaultImplementationForNulls() const override { return false; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + void setCollator(const TiDB::TiDBCollatorPtr & collator_) override { collator = collator_; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() < 3) + throw Exception("local_match_against_boolean requires a query, at least one column, and metadata"); + bool nullable = false; + for (size_t i = 0; i < arguments.size(); ++i) + { + const auto & argument = arguments[i]; + const auto nested_type = removeNullable(argument); + // An untyped SQL NULL search arrives as Nullable(Nothing). + // Documents and metadata still require string types. + const bool null_query = i == 0 && nested_type->getTypeId() == TypeIndex::Nothing; + if (!nested_type->isString() && !null_query) + throw Exception( + "Illegal type " + argument->getName() + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + nullable = nullable || argument->isNullable(); + } + DataTypePtr result_type = std::make_shared(); + return nullable ? std::make_shared(result_type) : result_type; + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override + { + const auto * query_column = typeid_cast(&*block.getByPosition(arguments[0]).column); + if (query_column == nullptr) + throw Exception( + "The query argument of local_match_against_boolean must be constant", + ErrorCodes::ILLEGAL_COLUMN); + + const size_t rows = block.getByPosition(arguments[1]).column->size(); + auto output = ColumnFloat64::create(rows, 0); + auto & output_data = output->getData(); + const bool nullable = block.getByPosition(result).type->isNullable(); + auto null_map = nullable ? ColumnUInt8::create(rows, 0) : nullptr; + const auto finish_result = [&] { + ColumnPtr result_column = std::move(output); + if (null_map) + result_column = ColumnNullable::create(std::move(result_column), std::move(null_map)); + block.getByPosition(result).column = std::move(result_column); + }; + + tipb::LocalMatchAgainstBooleanQuery protocol_boolean_query; + if (arguments.size() <= 2 + || !decodeLocalMatchAgainstBooleanQuery( + *block.getByPosition(arguments.back()).column, + protocol_boolean_query)) + throw Exception( + "local_match_against_boolean requires valid Boolean query metadata", + ErrorCodes::ILLEGAL_COLUMN); + if (query_column->isNullAt(0)) + { + // InnoDB interprets NULL AGAINST as an empty search: non-NULL zero. + finish_result(); + return; + } + const size_t document_argument_end = arguments.size() - 1; + const LocalMatchAnalyzerConfig config(protocol_boolean_query); + // Keep scalar values in the row hot path. Passing the config by + // reference changes Release code generation, notably with stopwords + // disabled; the config object belongs to query compilation only. + const bool use_ngram = config.use_ngram; + const size_t ngram_token_size = config.ngram_token_size; + const size_t min_token_size = config.min_token_size; + const size_t max_token_size = config.max_token_size; + const bool enable_stopword = config.enable_stopword; + const auto * const stopword_collator + = enable_stopword ? TiDB::ITiDBCollator::getCollator(protocol_boolean_query.stopword_collation()) : nullptr; + if (enable_stopword && !stopword_collator) + throw Exception( + "local_match_against_boolean requires a supported stopword collation when stopwords are enabled", + ErrorCodes::ILLEGAL_COLUMN); + std::optional stopword_lookup; + if (enable_stopword) + stopword_lookup.emplace(stopword_collator); + const auto * stopwords = stopword_lookup ? &*stopword_lookup : nullptr; + const CompiledBooleanQuery compiled_query + = compileBooleanQuery(protocol_boolean_query, config, collator, stopwords); + if (compiled_query.matches_nothing) + { + finish_result(); + return; + } + std::vector document_columns; + document_columns.reserve(document_argument_end - 1); + for (size_t arg = 1; arg < document_argument_end; ++arg) + document_columns.emplace_back(*block.getByPosition(arguments[arg]).column); + std::vector clause_states; + initializeClauseMatchStates(compiled_query.clauses, clause_states); + FullTextColumn analyzed_column; + FullTextColumn unfiltered_column; + AnalyzerScratch analyzer_scratch; + + for (size_t row = 0; row < rows; ++row) + { + resetClauseMatchStates(clause_states); + BooleanMatchProgress match_progress; + bool result_known = false; + bool row_matches = false; + for (const auto & document_column : document_columns) + { + // A NULL MATCH column contributes no tokens. This is + // intentional: #70485 relies on a row with a NULL body + // still matching a required term from another MATCH column. + if (document_column.isNull(row)) + { + analyzed_column.clear(); + continue; + } + analyzeColumnInto( + document_column.getString(row), + analyzed_column, + analyzer_scratch, + use_ngram, + ngram_token_size, + collator, + stopwords, + min_token_size, + max_token_size, + enable_stopword); + if (compiled_query.needs_verification) + analyzeColumnInto( + document_column.getString(row), + unfiltered_column, + analyzer_scratch, + use_ngram, + ngram_token_size, + collator, + nullptr, + 0, + std::numeric_limits::max(), + false); + const auto match_result = updateBooleanMatchStatesForColumn( + compiled_query, + analyzed_column, + unfiltered_column, + collator, + clause_states, + match_progress); + if (match_result == ColumnMatchResult::Rejected) + { + result_known = true; + break; + } + if (match_result == ColumnMatchResult::Accepted) + { + result_known = true; + row_matches = true; + break; + } + } + output_data[row] = result_known ? (row_matches ? 1 : 0) : matchBooleanValue(compiled_query, clause_states); + } + finish_result(); + } + +private: + TiDB::TiDBCollatorPtr collator = nullptr; +}; +} // namespace + +void registerFunctionsLocalMatchAgainst(FunctionFactory & factory) +{ + factory.registerFunction(FunctionFactory::CaseInsensitive); +} +} // namespace DB diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.h b/dbms/src/Functions/FunctionsLocalMatchAgainst.h new file mode 100644 index 00000000000..064c503718f --- /dev/null +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.h @@ -0,0 +1,22 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace DB +{ +void registerFunctionsLocalMatchAgainst(FunctionFactory & factory); +} diff --git a/dbms/src/Functions/LocalMatchAgainstTokenChars.h b/dbms/src/Functions/LocalMatchAgainstTokenChars.h new file mode 100644 index 00000000000..ec2d3987c7c --- /dev/null +++ b/dbms/src/Functions/LocalMatchAgainstTokenChars.h @@ -0,0 +1,211 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated by TiDB pkg/expression/matchagainst/generator; DO NOT EDIT. +#pragma once + +#include +#include +#include + +namespace DB::LocalMatchAgainst +{ +// Generated from Go's Unicode 15.0.0 categories L and N. TiDB uses its standard +// library and strictly tests that Unicode version and the Unicode 15.0.0 checksum. +// Equivalent Go predicate: +// word := r <= unicode.MaxRune && +// (unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_') +// This helper covers all Unicode code points; the Boolean lexer separately +// restricts word characters to the BMP (r <= 0xFFFF). +// Regenerate/check this header using the paired TiDB generator. Never change +// an existing protocol version's classification; deploy TiFlash support for a +// new version before TiDB emits it. ASCII and '_' use the fast path below. +struct TokenCharRange +{ + std::uint32_t first; + std::uint32_t last; +}; +inline constexpr std::array token_char_ranges{{ + {0xAA, 0xAA}, {0xB2, 0xB3}, {0xB5, 0xB5}, {0xB9, 0xBA}, {0xBC, 0xBE}, + {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x2C1}, {0x2C6, 0x2D1}, {0x2E0, 0x2E4}, + {0x2EC, 0x2EC}, {0x2EE, 0x2EE}, {0x370, 0x374}, {0x376, 0x377}, {0x37A, 0x37D}, + {0x37F, 0x37F}, {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x3A1}, + {0x3A3, 0x3F5}, {0x3F7, 0x481}, {0x48A, 0x52F}, {0x531, 0x556}, {0x559, 0x559}, + {0x560, 0x588}, {0x5D0, 0x5EA}, {0x5EF, 0x5F2}, {0x620, 0x64A}, {0x660, 0x669}, + {0x66E, 0x66F}, {0x671, 0x6D3}, {0x6D5, 0x6D5}, {0x6E5, 0x6E6}, {0x6EE, 0x6FC}, + {0x6FF, 0x6FF}, {0x710, 0x710}, {0x712, 0x72F}, {0x74D, 0x7A5}, {0x7B1, 0x7B1}, + {0x7C0, 0x7EA}, {0x7F4, 0x7F5}, {0x7FA, 0x7FA}, {0x800, 0x815}, {0x81A, 0x81A}, + {0x824, 0x824}, {0x828, 0x828}, {0x840, 0x858}, {0x860, 0x86A}, {0x870, 0x887}, + {0x889, 0x88E}, {0x8A0, 0x8C9}, {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, + {0x958, 0x961}, {0x966, 0x96F}, {0x971, 0x980}, {0x985, 0x98C}, {0x98F, 0x990}, + {0x993, 0x9A8}, {0x9AA, 0x9B0}, {0x9B2, 0x9B2}, {0x9B6, 0x9B9}, {0x9BD, 0x9BD}, + {0x9CE, 0x9CE}, {0x9DC, 0x9DD}, {0x9DF, 0x9E1}, {0x9E6, 0x9F1}, {0x9F4, 0x9F9}, + {0x9FC, 0x9FC}, {0xA05, 0xA0A}, {0xA0F, 0xA10}, {0xA13, 0xA28}, {0xA2A, 0xA30}, + {0xA32, 0xA33}, {0xA35, 0xA36}, {0xA38, 0xA39}, {0xA59, 0xA5C}, {0xA5E, 0xA5E}, + {0xA66, 0xA6F}, {0xA72, 0xA74}, {0xA85, 0xA8D}, {0xA8F, 0xA91}, {0xA93, 0xAA8}, + {0xAAA, 0xAB0}, {0xAB2, 0xAB3}, {0xAB5, 0xAB9}, {0xABD, 0xABD}, {0xAD0, 0xAD0}, + {0xAE0, 0xAE1}, {0xAE6, 0xAEF}, {0xAF9, 0xAF9}, {0xB05, 0xB0C}, {0xB0F, 0xB10}, + {0xB13, 0xB28}, {0xB2A, 0xB30}, {0xB32, 0xB33}, {0xB35, 0xB39}, {0xB3D, 0xB3D}, + {0xB5C, 0xB5D}, {0xB5F, 0xB61}, {0xB66, 0xB6F}, {0xB71, 0xB77}, {0xB83, 0xB83}, + {0xB85, 0xB8A}, {0xB8E, 0xB90}, {0xB92, 0xB95}, {0xB99, 0xB9A}, {0xB9C, 0xB9C}, + {0xB9E, 0xB9F}, {0xBA3, 0xBA4}, {0xBA8, 0xBAA}, {0xBAE, 0xBB9}, {0xBD0, 0xBD0}, + {0xBE6, 0xBF2}, {0xC05, 0xC0C}, {0xC0E, 0xC10}, {0xC12, 0xC28}, {0xC2A, 0xC39}, + {0xC3D, 0xC3D}, {0xC58, 0xC5A}, {0xC5D, 0xC5D}, {0xC60, 0xC61}, {0xC66, 0xC6F}, + {0xC78, 0xC7E}, {0xC80, 0xC80}, {0xC85, 0xC8C}, {0xC8E, 0xC90}, {0xC92, 0xCA8}, + {0xCAA, 0xCB3}, {0xCB5, 0xCB9}, {0xCBD, 0xCBD}, {0xCDD, 0xCDE}, {0xCE0, 0xCE1}, + {0xCE6, 0xCEF}, {0xCF1, 0xCF2}, {0xD04, 0xD0C}, {0xD0E, 0xD10}, {0xD12, 0xD3A}, + {0xD3D, 0xD3D}, {0xD4E, 0xD4E}, {0xD54, 0xD56}, {0xD58, 0xD61}, {0xD66, 0xD78}, + {0xD7A, 0xD7F}, {0xD85, 0xD96}, {0xD9A, 0xDB1}, {0xDB3, 0xDBB}, {0xDBD, 0xDBD}, + {0xDC0, 0xDC6}, {0xDE6, 0xDEF}, {0xE01, 0xE30}, {0xE32, 0xE33}, {0xE40, 0xE46}, + {0xE50, 0xE59}, {0xE81, 0xE82}, {0xE84, 0xE84}, {0xE86, 0xE8A}, {0xE8C, 0xEA3}, + {0xEA5, 0xEA5}, {0xEA7, 0xEB0}, {0xEB2, 0xEB3}, {0xEBD, 0xEBD}, {0xEC0, 0xEC4}, + {0xEC6, 0xEC6}, {0xED0, 0xED9}, {0xEDC, 0xEDF}, {0xF00, 0xF00}, {0xF20, 0xF33}, + {0xF40, 0xF47}, {0xF49, 0xF6C}, {0xF88, 0xF8C}, {0x1000, 0x102A}, {0x103F, 0x1049}, + {0x1050, 0x1055}, {0x105A, 0x105D}, {0x1061, 0x1061}, {0x1065, 0x1066}, {0x106E, 0x1070}, + {0x1075, 0x1081}, {0x108E, 0x108E}, {0x1090, 0x1099}, {0x10A0, 0x10C5}, {0x10C7, 0x10C7}, + {0x10CD, 0x10CD}, {0x10D0, 0x10FA}, {0x10FC, 0x1248}, {0x124A, 0x124D}, {0x1250, 0x1256}, + {0x1258, 0x1258}, {0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0}, + {0x12B2, 0x12B5}, {0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, + {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, {0x1369, 0x137C}, {0x1380, 0x138F}, + {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1401, 0x166C}, {0x166F, 0x167F}, {0x1681, 0x169A}, + {0x16A0, 0x16EA}, {0x16EE, 0x16F8}, {0x1700, 0x1711}, {0x171F, 0x1731}, {0x1740, 0x1751}, + {0x1760, 0x176C}, {0x176E, 0x1770}, {0x1780, 0x17B3}, {0x17D7, 0x17D7}, {0x17DC, 0x17DC}, + {0x17E0, 0x17E9}, {0x17F0, 0x17F9}, {0x1810, 0x1819}, {0x1820, 0x1878}, {0x1880, 0x1884}, + {0x1887, 0x18A8}, {0x18AA, 0x18AA}, {0x18B0, 0x18F5}, {0x1900, 0x191E}, {0x1946, 0x196D}, + {0x1970, 0x1974}, {0x1980, 0x19AB}, {0x19B0, 0x19C9}, {0x19D0, 0x19DA}, {0x1A00, 0x1A16}, + {0x1A20, 0x1A54}, {0x1A80, 0x1A89}, {0x1A90, 0x1A99}, {0x1AA7, 0x1AA7}, {0x1B05, 0x1B33}, + {0x1B45, 0x1B4C}, {0x1B50, 0x1B59}, {0x1B83, 0x1BA0}, {0x1BAE, 0x1BE5}, {0x1C00, 0x1C23}, + {0x1C40, 0x1C49}, {0x1C4D, 0x1C7D}, {0x1C80, 0x1C88}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, + {0x1CE9, 0x1CEC}, {0x1CEE, 0x1CF3}, {0x1CF5, 0x1CF6}, {0x1CFA, 0x1CFA}, {0x1D00, 0x1DBF}, + {0x1E00, 0x1F15}, {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, + {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, + {0x1FB6, 0x1FBC}, {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4}, {0x1FC6, 0x1FCC}, {0x1FD0, 0x1FD3}, + {0x1FD6, 0x1FDB}, {0x1FE0, 0x1FEC}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, {0x2070, 0x2071}, + {0x2074, 0x2079}, {0x207F, 0x2089}, {0x2090, 0x209C}, {0x2102, 0x2102}, {0x2107, 0x2107}, + {0x210A, 0x2113}, {0x2115, 0x2115}, {0x2119, 0x211D}, {0x2124, 0x2124}, {0x2126, 0x2126}, + {0x2128, 0x2128}, {0x212A, 0x212D}, {0x212F, 0x2139}, {0x213C, 0x213F}, {0x2145, 0x2149}, + {0x214E, 0x214E}, {0x2150, 0x2189}, {0x2460, 0x249B}, {0x24EA, 0x24FF}, {0x2776, 0x2793}, + {0x2C00, 0x2CE4}, {0x2CEB, 0x2CEE}, {0x2CF2, 0x2CF3}, {0x2CFD, 0x2CFD}, {0x2D00, 0x2D25}, + {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D6F}, {0x2D80, 0x2D96}, + {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, + {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2E2F, 0x2E2F}, {0x3005, 0x3007}, + {0x3021, 0x3029}, {0x3031, 0x3035}, {0x3038, 0x303C}, {0x3041, 0x3096}, {0x309D, 0x309F}, + {0x30A1, 0x30FA}, {0x30FC, 0x30FF}, {0x3105, 0x312F}, {0x3131, 0x318E}, {0x3192, 0x3195}, + {0x31A0, 0x31BF}, {0x31F0, 0x31FF}, {0x3220, 0x3229}, {0x3248, 0x324F}, {0x3251, 0x325F}, + {0x3280, 0x3289}, {0x32B1, 0x32BF}, {0x3400, 0x4DBF}, {0x4E00, 0xA48C}, {0xA4D0, 0xA4FD}, + {0xA500, 0xA60C}, {0xA610, 0xA62B}, {0xA640, 0xA66E}, {0xA67F, 0xA69D}, {0xA6A0, 0xA6EF}, + {0xA717, 0xA71F}, {0xA722, 0xA788}, {0xA78B, 0xA7CA}, {0xA7D0, 0xA7D1}, {0xA7D3, 0xA7D3}, + {0xA7D5, 0xA7D9}, {0xA7F2, 0xA801}, {0xA803, 0xA805}, {0xA807, 0xA80A}, {0xA80C, 0xA822}, + {0xA830, 0xA835}, {0xA840, 0xA873}, {0xA882, 0xA8B3}, {0xA8D0, 0xA8D9}, {0xA8F2, 0xA8F7}, + {0xA8FB, 0xA8FB}, {0xA8FD, 0xA8FE}, {0xA900, 0xA925}, {0xA930, 0xA946}, {0xA960, 0xA97C}, + {0xA984, 0xA9B2}, {0xA9CF, 0xA9D9}, {0xA9E0, 0xA9E4}, {0xA9E6, 0xA9FE}, {0xAA00, 0xAA28}, + {0xAA40, 0xAA42}, {0xAA44, 0xAA4B}, {0xAA50, 0xAA59}, {0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, + {0xAA7E, 0xAAAF}, {0xAAB1, 0xAAB1}, {0xAAB5, 0xAAB6}, {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, + {0xAAC2, 0xAAC2}, {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, {0xAAF2, 0xAAF4}, {0xAB01, 0xAB06}, + {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, {0xAB30, 0xAB5A}, + {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, {0xABF0, 0xABF9}, {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, + {0xD7CB, 0xD7FB}, {0xF900, 0xFA6D}, {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, + {0xFB1D, 0xFB1D}, {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, + {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFBB1}, {0xFBD3, 0xFD3D}, {0xFD50, 0xFD8F}, + {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, {0xFF10, 0xFF19}, + {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, {0xFF66, 0xFFBE}, {0xFFC2, 0xFFC7}, {0xFFCA, 0xFFCF}, + {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, + {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, {0x10107, 0x10133}, + {0x10140, 0x10178}, {0x1018A, 0x1018B}, {0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x102E1, 0x102FB}, + {0x10300, 0x10323}, {0x1032D, 0x1034A}, {0x10350, 0x10375}, {0x10380, 0x1039D}, {0x103A0, 0x103C3}, + {0x103C8, 0x103CF}, {0x103D1, 0x103D5}, {0x10400, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, + {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, {0x10570, 0x1057A}, {0x1057C, 0x1058A}, + {0x1058C, 0x10592}, {0x10594, 0x10595}, {0x10597, 0x105A1}, {0x105A3, 0x105B1}, {0x105B3, 0x105B9}, + {0x105BB, 0x105BC}, {0x10600, 0x10736}, {0x10740, 0x10755}, {0x10760, 0x10767}, {0x10780, 0x10785}, + {0x10787, 0x107B0}, {0x107B2, 0x107BA}, {0x10800, 0x10805}, {0x10808, 0x10808}, {0x1080A, 0x10835}, + {0x10837, 0x10838}, {0x1083C, 0x1083C}, {0x1083F, 0x10855}, {0x10858, 0x10876}, {0x10879, 0x1089E}, + {0x108A7, 0x108AF}, {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, {0x10920, 0x10939}, + {0x10980, 0x109B7}, {0x109BC, 0x109CF}, {0x109D2, 0x10A00}, {0x10A10, 0x10A13}, {0x10A15, 0x10A17}, + {0x10A19, 0x10A35}, {0x10A40, 0x10A48}, {0x10A60, 0x10A7E}, {0x10A80, 0x10A9F}, {0x10AC0, 0x10AC7}, + {0x10AC9, 0x10AE4}, {0x10AEB, 0x10AEF}, {0x10B00, 0x10B35}, {0x10B40, 0x10B55}, {0x10B58, 0x10B72}, + {0x10B78, 0x10B91}, {0x10BA9, 0x10BAF}, {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, + {0x10CFA, 0x10D23}, {0x10D30, 0x10D39}, {0x10E60, 0x10E7E}, {0x10E80, 0x10EA9}, {0x10EB0, 0x10EB1}, + {0x10F00, 0x10F27}, {0x10F30, 0x10F45}, {0x10F51, 0x10F54}, {0x10F70, 0x10F81}, {0x10FB0, 0x10FCB}, + {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11052, 0x1106F}, {0x11071, 0x11072}, {0x11075, 0x11075}, + {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x110F0, 0x110F9}, {0x11103, 0x11126}, {0x11136, 0x1113F}, + {0x11144, 0x11144}, {0x11147, 0x11147}, {0x11150, 0x11172}, {0x11176, 0x11176}, {0x11183, 0x111B2}, + {0x111C1, 0x111C4}, {0x111D0, 0x111DA}, {0x111DC, 0x111DC}, {0x111E1, 0x111F4}, {0x11200, 0x11211}, + {0x11213, 0x1122B}, {0x1123F, 0x11240}, {0x11280, 0x11286}, {0x11288, 0x11288}, {0x1128A, 0x1128D}, + {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, {0x112F0, 0x112F9}, {0x11305, 0x1130C}, + {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333}, {0x11335, 0x11339}, + {0x1133D, 0x1133D}, {0x11350, 0x11350}, {0x1135D, 0x11361}, {0x11400, 0x11434}, {0x11447, 0x1144A}, + {0x11450, 0x11459}, {0x1145F, 0x11461}, {0x11480, 0x114AF}, {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, + {0x114D0, 0x114D9}, {0x11580, 0x115AE}, {0x115D8, 0x115DB}, {0x11600, 0x1162F}, {0x11644, 0x11644}, + {0x11650, 0x11659}, {0x11680, 0x116AA}, {0x116B8, 0x116B8}, {0x116C0, 0x116C9}, {0x11700, 0x1171A}, + {0x11730, 0x1173B}, {0x11740, 0x11746}, {0x11800, 0x1182B}, {0x118A0, 0x118F2}, {0x118FF, 0x11906}, + {0x11909, 0x11909}, {0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x1192F}, {0x1193F, 0x1193F}, + {0x11941, 0x11941}, {0x11950, 0x11959}, {0x119A0, 0x119A7}, {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, + {0x119E3, 0x119E3}, {0x11A00, 0x11A00}, {0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, + {0x11A5C, 0x11A89}, {0x11A9D, 0x11A9D}, {0x11AB0, 0x11AF8}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, + {0x11C40, 0x11C40}, {0x11C50, 0x11C6C}, {0x11C72, 0x11C8F}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, + {0x11D0B, 0x11D30}, {0x11D46, 0x11D46}, {0x11D50, 0x11D59}, {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, + {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11DA0, 0x11DA9}, {0x11EE0, 0x11EF2}, {0x11F02, 0x11F02}, + {0x11F04, 0x11F10}, {0x11F12, 0x11F33}, {0x11F50, 0x11F59}, {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FD4}, + {0x12000, 0x12399}, {0x12400, 0x1246E}, {0x12480, 0x12543}, {0x12F90, 0x12FF0}, {0x13000, 0x1342F}, + {0x13441, 0x13446}, {0x14400, 0x14646}, {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, + {0x16A70, 0x16ABE}, {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, + {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16E40, 0x16E96}, + {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, + {0x17000, 0x187F7}, {0x18800, 0x18CD5}, {0x18D00, 0x18D08}, {0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, + {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122}, {0x1B132, 0x1B132}, {0x1B150, 0x1B152}, {0x1B155, 0x1B155}, + {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, + {0x1BC90, 0x1BC99}, {0x1D2C0, 0x1D2D3}, {0x1D2E0, 0x1D2F3}, {0x1D360, 0x1D378}, {0x1D400, 0x1D454}, + {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, + {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, + {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, + {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA}, + {0x1D6DC, 0x1D6FA}, {0x1D6FC, 0x1D714}, {0x1D716, 0x1D734}, {0x1D736, 0x1D74E}, {0x1D750, 0x1D76E}, + {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, {0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB}, {0x1D7CE, 0x1D7FF}, + {0x1DF00, 0x1DF1E}, {0x1DF25, 0x1DF2A}, {0x1E030, 0x1E06D}, {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, + {0x1E140, 0x1E149}, {0x1E14E, 0x1E14E}, {0x1E290, 0x1E2AD}, {0x1E2C0, 0x1E2EB}, {0x1E2F0, 0x1E2F9}, + {0x1E4D0, 0x1E4EB}, {0x1E4F0, 0x1E4F9}, {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE}, + {0x1E7F0, 0x1E7FE}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8CF}, {0x1E900, 0x1E943}, {0x1E94B, 0x1E94B}, + {0x1E950, 0x1E959}, {0x1EC71, 0x1ECAB}, {0x1ECAD, 0x1ECAF}, {0x1ECB1, 0x1ECB4}, {0x1ED01, 0x1ED2D}, + {0x1ED2F, 0x1ED3D}, {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, + {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, + {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, + {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, + {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, + {0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, + {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, {0x1F100, 0x1F10C}, + {0x1FBF0, 0x1FBF9}, {0x20000, 0x2A6DF}, {0x2A700, 0x2B739}, {0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, + {0x2CEB0, 0x2EBE0}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, {0x31350, 0x323AF}, +}}; + +inline bool isTokenChar(std::uint32_t code_point) +{ + if (code_point < 128) + return (code_point >= 'a' && code_point <= 'z') || (code_point >= 'A' && code_point <= 'Z') + || (code_point >= '0' && code_point <= '9') || code_point == '_'; + std::size_t lo = 0; + std::size_t hi = token_char_ranges.size(); + while (lo < hi) + { + const std::size_t mid = lo + (hi - lo) / 2; + const auto & span = token_char_ranges[mid]; + if (code_point < span.first) + hi = mid; + else if (code_point > span.last) + lo = mid + 1; + else + return true; + } + return false; +} +} // namespace DB::LocalMatchAgainst diff --git a/dbms/src/Functions/registerFunctions.cpp b/dbms/src/Functions/registerFunctions.cpp index cb43ae7760d..7a9193c392c 100644 --- a/dbms/src/Functions/registerFunctions.cpp +++ b/dbms/src/Functions/registerFunctions.cpp @@ -51,6 +51,7 @@ void registerFunctionsRegexpSubstr(FunctionFactory &); void registerFunctionsRegexpReplace(FunctionFactory &); void registerFunctionsGrouping(FunctionFactory &); void registerFunctionsVector(FunctionFactory &); +void registerFunctionsLocalMatchAgainst(FunctionFactory &); void registerFunctions() { @@ -85,6 +86,7 @@ void registerFunctions() registerFunctionsIsIPAddr(factory); registerFunctionsGrouping(factory); registerFunctionsVector(factory); + registerFunctionsLocalMatchAgainst(factory); } } // namespace DB diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp new file mode 100644 index 00000000000..c90d9968248 --- /dev/null +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -0,0 +1,1918 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB::ErrorCodes +{ +extern const int ILLEGAL_COLUMN; +} + +namespace DB::tests +{ +class TestLocalMatchAgainst : public DB::tests::FunctionTest +{ +}; + +namespace +{ +constexpr std::array builtin_stopwords{ + "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", + "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", + "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; + +// Count token comparisons rather than wall time, so a repeated full-column +// verification regression is deterministic even on a loaded CI host. +class CountingMatchCollator final : public TiDB::ITiDBCollator +{ +public: + CountingMatchCollator() + : ITiDBCollator(UTF8MB4_BIN) + , inner(getCollator(UTF8MB4_BIN)) + { + collator_type = inner->getCollatorType(); + } + + int compare(const char * lhs, size_t lhs_size, const char * rhs, size_t rhs_size) const override + { + ++comparisons; + return inner->compare(lhs, lhs_size, rhs, rhs_size); + } + StringRef convert(const char * s, size_t size, String & buffer, std::vector * lens) const override + { + return inner->convert(s, size, buffer, lens); + } + StringRef sortKeyNoTrim(const char * s, size_t size, String & buffer) const override + { + return inner->sortKeyNoTrim(s, size, buffer); + } + StringRef sortKey(const char * s, size_t size, String & buffer) const override + { + return inner->sortKey(s, size, buffer); + } + std::unique_ptr pattern() const override { return inner->pattern(); } + + mutable size_t comparisons = 0; + +private: + TiDB::TiDBCollatorPtr inner; +}; + +String serializeLocalMatchAgainstBooleanQuery(const tipb::LocalMatchAgainstBooleanQuery & query) +{ + auto encoded_query = query; + if (encoded_query.version() == 0) + encoded_query.set_version(2); // The tests exercise the currently supported Local MATCH semantic protocol. + if (encoded_query.parser() == tipb::LocalMatchAgainstParser::LocalMatchAgainstParserInvalid) + encoded_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + if (encoded_query.stopword_mode() == tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeInvalid) + encoded_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + if (encoded_query.stopword_collation().empty()) + encoded_query.set_stopword_collation("utf8mb4_bin"); + return encoded_query.SerializeAsString(); +} + +tipb::LocalMatchAgainstBooleanQuery makeRequiredLocalMatchQuery( + const String & text, + tipb::LocalMatchAgainstBooleanTermType term_type = tipb::LocalMatchAgainstBooleanTermWord) +{ + tipb::LocalMatchAgainstBooleanQuery query; + query.set_innodb_ft_min_token_size(3); + query.set_innodb_ft_max_token_size(84); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(term_type); + node->set_text(text); + return query; +} + +// Construct the wire expression directly: metadata belongs to Expr.val, not +// SQL children. ColumnRef offsets and signed collations use the real DAG path. +tipb::Expr makeLocalMatchDAGExpr( + const String & search, + const tipb::LocalMatchAgainstBooleanQuery & query, + const NamesAndTypes & inputs, + Int32 collation, + bool null_search = false) +{ + tipb::Expr expr; + expr.set_tp(tipb::ExprType::ScalarFunc); + expr.set_sig(tipb::ScalarFuncSig::LocalMatchAgainstBoolean); + expr.set_val(serializeLocalMatchAgainstBooleanQuery(query)); + auto * result_type = expr.mutable_field_type(); + result_type->set_tp(TiDB::TypeDouble); + result_type->set_collate(-collation); + const bool nullable = null_search + || std::any_of(inputs.begin(), inputs.end(), [](const auto & input) { return input.type->isNullable(); }); + result_type->set_flag(nullable ? 0 : TiDB::ColumnFlagNotNull); + auto search_expr = null_search ? constructNULLLiteralTiExpr() : constructStringLiteralTiExpr(search); + search_expr.mutable_field_type()->set_collate(-collation); + *expr.add_children() = std::move(search_expr); + for (size_t i = 0; i < inputs.size(); ++i) + { + auto column = constructInt64LiteralTiExpr(static_cast(i)); + column.set_tp(tipb::ExprType::ColumnRef); + column.mutable_field_type()->set_tp(TiDB::TypeString); + column.mutable_field_type()->set_flag(inputs[i].type->isNullable() ? 0 : TiDB::ColumnFlagNotNull); + column.mutable_field_type()->set_collate(-collation); + *expr.add_children() = std::move(column); + } + return expr; +} + +std::pair buildLocalMatchDAGActions( + Context & context, + const NamesAndTypes & inputs, + const std::vector & expressions) +{ + DAGExpressionAnalyzer analyzer(inputs, context); + ExpressionActionsChain chain; + auto & step = analyzer.initAndGetLastStep(chain); + Names results; + for (const auto & expr : expressions) + { + results.push_back(analyzer.getActions(expr, step.actions)); + step.required_output.push_back(results.back()); + } + chain.finalize(); + return {step.actions, results}; +} +} // namespace + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGCollationAndMultiColumn) +try +{ + const ColumnsWithTypeAndName columns{ + createColumn>({"CAFE", {}, "cafe", "cafeteria", {}, "cafe"}, "title"), + createColumn>({"", "café", "mysql", "", {}, "MYSQL"}, "body")}; + const NamesAndTypes inputs{{"title", columns[0].type}, {"body", columns[1].type}}; + for (const auto * collation : + {"utf8mb4_bin", "utf8mb4_0900_bin", "utf8mb4_general_ci", "utf8mb4_unicode_ci", "utf8mb4_0900_ai_ci"}) + { + const auto collator = TiDB::ITiDBCollator::getCollator(collation); + ASSERT_NE(collator, nullptr); + const bool binary = std::string_view(collation).ends_with("_bin"); + for (const bool prefix : {false, true}) + { + SCOPED_TRACE(fmt::format("collation={} prefix={}", collation, prefix)); + auto query = makeRequiredLocalMatchQuery( + prefix ? "caf" : "cafe", + prefix ? tipb::LocalMatchAgainstBooleanTermPrefix : tipb::LocalMatchAgainstBooleanTermWord); + auto * prohibited = query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("mysql"); + const auto expr = makeLocalMatchDAGExpr( + prefix ? "+caf* -mysql" : "+cafe -mysql", + query, + inputs, + collator->getCollatorId()); + ASSERT_EQ(expr.children_size(), 3); // Query + two MATCH columns; no metadata SQL child. + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); + Block block(columns); + actions->execute(block); + const auto expected = prefix + ? (binary ? createColumn>({0, 1, 0, 1, 0, 1}) + : createColumn>({1, 1, 0, 1, 0, 0})) + : (binary ? createColumn>({0, 0, 0, 0, 0, 1}) + : createColumn>({1, 1, 0, 0, 0, 0})); + ASSERT_COLUMN_EQ(expected, block.getByName(names[0])); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGMetadataIsolation) +try +{ + const auto documents + = createColumn({"the", "the 中文", "中文", "中文词", "中文测试", "", "中 文", "foo", "thé"}, "body"); + const NamesAndTypes inputs{{"body", documents.type}}; + const auto binary = TiDB::ITiDBCollator::UTF8MB4_BIN; + std::vector expressions; + ColumnsWithTypeAndName expected; + const auto append = [&](const String & search, + const tipb::LocalMatchAgainstBooleanQuery & query, + Int32 collation, + const ColumnWithTypeAndName & result) { + expressions.push_back(makeLocalMatchDAGExpr(search, query, inputs, collation)); + expected.push_back(result); + }; + auto query = makeRequiredLocalMatchQuery("the"); + append("+the", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + append("+the", query, binary, createColumn({1, 1, 0, 0, 0, 0, 0, 0, 0})); + + query = makeRequiredLocalMatchQuery("中文"); + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + query.set_ngram_token_size(2); + append("+中文", query, binary, createColumn({0, 1, 1, 1, 1, 0, 0, 0, 0})); + query.set_ngram_token_size(3); + append("+中文", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + + query = makeRequiredLocalMatchQuery("foo"); + append("+foo", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 1, 0})); + query.set_innodb_ft_min_token_size(4); + append("+foo", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + + query = makeRequiredLocalMatchQuery("thé"); + query.set_stopword_collation("utf8mb4_bin"); + append("+thé", query, TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 1})); + query.set_stopword_collation("utf8mb4_general_ci"); + append("+thé", query, TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, expressions); + ASSERT_EQ(names.size(), expected.size()); + for (size_t i = 0; i < names.size(); i += 2) + ASSERT_NE(names[i], names[i + 1]) << "Different metadata must not alias the same action"; + Block block({documents}); + actions->execute(block); + for (size_t i = 0; i < names.size(); ++i) + { + SCOPED_TRACE(i); + ASSERT_COLUMN_EQ(expected[i], block.getByName(names[i])); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGReusesActionsAcrossBlocks) +try +{ + struct BlockCase + { + ColumnsWithTypeAndName columns; + ColumnWithTypeAndName expected; + }; + for (const bool ngram : {false, true}) + { + SCOPED_TRACE(ngram); + auto query = makeRequiredLocalMatchQuery( + ngram ? "中文测试" : "quick the fox", + tipb::LocalMatchAgainstBooleanTermPhrase); + if (ngram) + { + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(2); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + } + auto * prohibited = query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("mysql"); + const auto first = ngram + ? BlockCase{{createColumn>({"中文测试", "中文", "中文测试", "", {}}, "title"), createColumn>({{}, "测试", "mysql", "中文测试", {}}, "body")}, createColumn>({1, 0, 0, 1, 0})} + : BlockCase{ + {createColumn>( + {"QUICK the fox", "quick the fox", "quick", {}, "quick the fox"}, + "title"), + createColumn>({{}, "mysql", "the fox", {}, "MYSQL"}, "body")}, + createColumn>({1, 0, 0, 0, 0})}; + const auto second = ngram + ? BlockCase{{createColumn>({"中文 test", {}, "中文测试", "中 文测试"}, "title"), createColumn>({"", {}, "", "中文测试"}, "body")}, createColumn>({0, 0, 1, 1})} + : BlockCase{ + {createColumn>( + {"slow the fox", "quick the fox", {}, "quick fox", "quick the fox"}, + "title"), + createColumn>({"", {}, "quick the fox", "", {}}, "body")}, + createColumn>({0, 1, 1, 0, 1})}; + const BlockCase nulls{ + {createColumn>({{}, {}}, "title"), createColumn>({{}, {}}, "body")}, + createColumn>({0, 0})}; + const BlockCase empty{ + {createColumn>({}, "title"), createColumn>({}, "body")}, + createColumn>({})}; + const NamesAndTypes inputs{{"title", first.columns[0].type}, {"body", first.columns[1].type}}; + const auto expr = makeLocalMatchDAGExpr( + ngram ? "+\"中文测试\" -mysql" : "+\"quick the fox\" -mysql", + query, + inputs, + TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); + // The same compiled actions/function sees changing row counts, empty + // input, and old input again. No previous result or NULL map may leak. + for (size_t pass = 0; pass < 2; ++pass) + for (const auto * batch : {&first, &second, &nulls, &empty, &first}) + { + SCOPED_TRACE(fmt::format("pass={} rows={}", pass, batch->expected.column->size())); + Block block(batch->columns); + actions->execute(block); + ASSERT_COLUMN_EQ(batch->expected, block.getByName(names[0])); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGColumnWrappersAndEmptyBlocks) +try +{ + struct ColumnCase + { + ColumnWithTypeAndName document; + ColumnWithTypeAndName expected; + }; + const std::vector cases{ + {createColumn({"quick", "", "quick fox", "slow"}, "body"), createColumn({1, 0, 1, 0})}, + {createColumn>({"quick", {}, "", "slow"}, "body"), + createColumn>({1, 0, 0, 0})}, + {createColumn>({{}, {}, {}, {}}, "body"), createColumn>({0, 0, 0, 0})}, + {createConstColumn(4, "quick", "body"), createColumn({1, 1, 1, 1})}, + {createConstColumn>(4, "quick", "body"), createColumn>({1, 1, 1, 1})}, + {createConstColumn>(4, {}, "body"), createColumn>({0, 0, 0, 0})}, + {createConstColumn(4, "", "body"), createColumn({0, 0, 0, 0})}}; + const auto query = makeRequiredLocalMatchQuery("quick"); + for (size_t i = 0; i < cases.size(); ++i) + for (const bool null_search : {false, true}) + { + SCOPED_TRACE(fmt::format("case={} null_search={}", i, null_search)); + const auto & column_case = cases[i]; + const NamesAndTypes inputs{{"body", column_case.document.type}}; + const auto expr + = makeLocalMatchDAGExpr("+quick", query, inputs, TiDB::ITiDBCollator::UTF8MB4_BIN, null_search); + ASSERT_EQ(expr.children(0).field_type().tp(), null_search ? TiDB::TypeNull : TiDB::TypeString); + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); + const auto expected = null_search ? createColumn>({0, 0, 0, 0}) : column_case.expected; + for (const size_t rows : {4U, 0U, 4U}) + { + SCOPED_TRACE(rows); + auto document = column_case.document; + document.column = document.column->cloneResized(rows); + auto expected_rows = expected; + expected_rows.column = expected_rows.column->cloneResized(rows); + Block block({document}); + actions->execute(block); + ASSERT_COLUMN_EQ(expected_rows, block.getByName(names[0])); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanMultiColumnNullable) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+quick"), + createColumn>({{}, "quick fox", {}, "slow fox"}), + createColumn>({"quick fox", {}, "slow fox", {}}), + createConstColumn(4, metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, FilteredPhraseVerificationComparisonGrowth) +try +{ + tipb::LocalMatchAgainstBooleanQuery query; + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + node->set_text("foo a bar"); + size_t small_comparisons = 0; + for (const size_t repeats : {32, 256}) + { + String document; + for (size_t i = 0; i < repeats; ++i) + document += "foo x bar "; + CountingMatchCollator collator; + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+\"foo a bar\""), + createColumn({document}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}, + &collator, + true)); + if (repeats == 32) + small_comparisons = collator.comparisons; + else + { + EXPECT_GT(small_comparisons, 0); + // Eight times the input may add proportional token work, not + // 64 times the work from rechecking every failed anchor. + EXPECT_LE(collator.comparisons, small_comparisons * 12); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, FilteredPhraseVerificationCacheScope) +try +{ + String miss; + for (size_t i = 0; i < 64; ++i) + miss += "foo x bar "; + for (const auto occur : + {tipb::LocalMatchAgainstBooleanOccurMust, + tipb::LocalMatchAgainstBooleanOccurShould, + tipb::LocalMatchAgainstBooleanOccurMustNot}) + { + tipb::LocalMatchAgainstBooleanQuery query; + if (occur != tipb::LocalMatchAgainstBooleanOccurShould) + { + auto * anchor = query.add_nodes(); + anchor->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + anchor->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + anchor->set_text("foo"); + } + auto * phrase = query.add_nodes(); + phrase->set_occur(occur); + phrase->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + phrase->set_text("foo a bar"); + const Float64 hit = occur == tipb::LocalMatchAgainstBooleanOccurMustNot ? 0 : 1; + const Float64 no_phrase = occur == tipb::LocalMatchAgainstBooleanOccurMustNot ? 1 : 0; + ASSERT_COLUMN_EQ( + createColumn>({hit, hit, no_phrase, hit, no_phrase}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(5, "foo \"foo a bar\""), + createColumn>({miss, "foo a bar", miss, {}, miss}), + createColumn>({"foo a bar", miss, {}, "foo a bar", miss}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } + + tipb::LocalMatchAgainstBooleanQuery query; + auto * required = query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + required->set_text("foo x bar"); + auto * excluded = query.add_nodes(); + excluded->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + excluded->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + excluded->set_text("foo a bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+\"foo x bar\" -\"foo a bar\""), + createColumn({miss, "foo x bar foo a bar", miss}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + + // NGRAM size 3 removes "the" but retains "foo" and "zoo". + // STANDARD exercises the same cache with a filtered stopword. + for (const bool ngram : {false, true}) + { + tipb::LocalMatchAgainstBooleanQuery filtered; + filtered.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + filtered.set_ngram_token_size(3); + auto * node = filtered.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + node->set_text("foo the zoo"); + String repetitive_miss; + for (size_t i = 0; i < 64; ++i) + repetitive_miss += "foo xyz zoo "; + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+\"foo the zoo\""), + createColumn>({repetitive_miss, "foo the zoo", repetitive_miss, {}}), + createColumn>({"foo the zoo", repetitive_miss, {}, "foo the zoo"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(filtered))}, + nullptr, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanMySQLCharacterBoundaries) +try +{ + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(2); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + node->set_text("a"); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1, 1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(6, "+a*"), + createColumn({"a,b", "a,b", "a🙃b", "a𞤀b", "a𝟙b", String("ab") + char(0xff) + "cd"}), + createConstColumn(6, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + + query.set_parser(tipb::LocalMatchAgainstParserStandard); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("foo"); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+foo"), + createColumn({"foo𞤀bar", "foo𝟙bar", "foo🙃bar", "foobar"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanFilteredPhraseVerification) +try +{ + for (const bool ngram : {false, true}) + for (const bool stopwords : {false, true}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + query.set_ngram_token_size(2); + query.set_stopword_mode( + stopwords ? tipb::LocalMatchAgainstStopwordModeBuiltin : tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + node->set_text("quick x fox"); + ASSERT_COLUMN_EQ( + createColumn({ngram ? 0.0 : 1.0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+\"quick x fox\""), + createColumn({"quick x fox", "quick the fox", "quick xx fox"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + node->set_text("foo bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 1, ngram ? 0.0 : 1.0, ngram ? 0.0 : 1.0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(5, "+\"foo bar\""), + createColumn({"foo bar", "foo,bar", "foo,bar", "foo🙃bar", "foobar"}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } + + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(3); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeBuiltin); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + for (const String & text : {String("quick the fox"), String("quick a fox")}) + { + node->set_text(text); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, text), + createColumn({"quick fox", "quick x fox", "fox quick"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } + for (const String & text : {String("foo a zoo"), String("foo zoo")}) + { + node->set_text(text); + const Float64 expected = text == "foo zoo" ? 1.0 : 0.0; + ASSERT_COLUMN_EQ( + createColumn({expected, expected, expected, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, text), + createColumn({"foo zoo", "foo a zoo", "foo x zoo", "zoo foo"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanFilteredRequiredClause) +try +{ + for (const bool ngram : {false, true}) + for (const String & term : {String("the"), String("x")}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + query.set_ngram_token_size(3); + auto * required = query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text(term); + auto * optional = query.add_nodes(); + optional->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + optional->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + optional->set_text("quick"); + const auto columns = ColumnsWithTypeAndName{ + createConstColumn(3, "+" + term + " quick"), + createColumn>({"the x quick", String(4096, 'x'), {}}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}; + ASSERT_COLUMN_EQ( + createColumn>({0, 0, 0}), + executeFunction("local_match_against_boolean", columns, nullptr, true)); + + // NULL AGAINST is an empty search even for an impossible query. + auto null_search = columns; + null_search[0] = createConstColumn>(3, {}); + ASSERT_COLUMN_EQ( + createColumn>({0, 0, 0}), + executeFunction("local_match_against_boolean", null_search, nullptr, true)); + + // Envelope validation must still reject an unknown version. + query.set_version(999); + EXPECT_THROW( + executeFunction( + "local_match_against_boolean", + {columns[0], + columns[1], + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true), + Exception); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanProtocolQuery) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + auto * prohibited = boolean_query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("slow"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+quick -slow"), + createColumn({"quick brown", "slow fox", "brown fox"}), + createConstColumn(3, metadata)}, + nullptr, + true)); + + tipb::LocalMatchAgainstBooleanQuery prohibited_query; + auto * prohibited_only = prohibited_query.add_nodes(); + prohibited_only->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited_only->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited_only->set_text("slow"); + const String prohibited_metadata = serializeLocalMatchAgainstBooleanQuery(prohibited_query); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "-slow"), + createColumn({"quick brown", "slow fox", "brown fox"}), + createConstColumn(3, prohibited_metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanPhrasePositionsWithStopwordGaps) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + required->set_text("quick the fox"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+\"quick the fox\""), + createColumn({"quick the fox", "quick fox", "quick slow the fox", "quick the fox quick the fox"}), + createConstColumn(4, metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanConstantDocumentColumn) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+quick"), + createConstColumn(3, "quick fox"), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(boolean_query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanMultiColumnStateAndPhraseBoundary) +try +{ + const auto make_query = [](tipb::LocalMatchAgainstBooleanTermType term_type) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(term_type); + required->set_text("quick fox"); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+quick +fox"), + createColumn({"quick", "quick"}), + createColumn({"fox", "slow"}), + createConstColumn(2, make_query(tipb::LocalMatchAgainstBooleanTermWord))}, + nullptr, + true)); + + ASSERT_COLUMN_EQ( + createColumn({0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+\"quick fox\""), + createColumn({"quick", "quick fox"}), + createColumn({"fox", "slow"}), + createConstColumn(2, make_query(tipb::LocalMatchAgainstBooleanTermPhrase))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDecidesPositiveMatchEarly) +try +{ + tipb::LocalMatchAgainstBooleanQuery required_query; + auto * required_quick = required_query.add_nodes(); + required_quick->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_quick->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_quick->set_text("quick"); + auto * required_fox = required_query.add_nodes(); + required_fox->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_fox->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_fox->set_text("fox"); + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+quick +fox"), + createColumn({"quick", "quick"}), + createColumn({"fox", "quiet"}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(required_query))}, + nullptr, + true)); + + tipb::LocalMatchAgainstBooleanQuery optional_query; + auto * optional_quick = optional_query.add_nodes(); + optional_quick->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + optional_quick->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + optional_quick->set_text("quick"); + auto * optional_fox = optional_query.add_nodes(); + optional_fox->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + optional_fox->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + optional_fox->set_text("fox"); + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "quick fox"), + createColumn({"quick", "quiet"}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(optional_query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanStillChecksProhibitedTermsAfterPositiveMatch) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required_quick = boolean_query.add_nodes(); + required_quick->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_quick->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_quick->set_text("quick"); + auto * required_fox = boolean_query.add_nodes(); + required_fox->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_fox->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_fox->set_text("fox"); + auto * prohibited_slow = boolean_query.add_nodes(); + prohibited_slow->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited_slow->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited_slow->set_text("slow"); + + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+quick +fox -slow"), + createColumn({"quick", "quick", "quick"}), + createColumn({"fox", "fox", "quiet"}), + createColumn({"calm", "slow", "slow"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(boolean_query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanRejectsInvalidProtocol) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_version(1); // Version 1 must not be silently reinterpreted as version 2. + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + boolean_query.set_stopword_collation("utf8mb4_bin"); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + + ASSERT_THROW( + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, boolean_query.SerializeAsString())}, + nullptr, + true), + Exception); + + boolean_query.set_version(2); + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserInvalid); + ASSERT_THROW( + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, boolean_query.SerializeAsString())}, + nullptr, + true), + Exception); + + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeInvalid); + ASSERT_THROW( + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, boolean_query.SerializeAsString())}, + nullptr, + true), + Exception); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanRejectsInvalidNodeEnums) +try +{ + enum class InvalidEnumEncoding + { + Missing, + Invalid, + Unknown, + }; + for (const bool invalid_occur : {false, true}) + for (const auto encoding : + {InvalidEnumEncoding::Missing, InvalidEnumEncoding::Invalid, InvalidEnumEncoding::Unknown}) + { + auto query = makeRequiredLocalMatchQuery("quick"); + query.set_version(2); + query.set_parser(tipb::LocalMatchAgainstParserStandard); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + // Put the malformed node after a valid one: all nodes must be checked. + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("fox"); + if (invalid_occur) + { + node->clear_occur(); + if (encoding == InvalidEnumEncoding::Invalid) + node->set_occur(tipb::LocalMatchAgainstBooleanOccurInvalid); + } + else + { + node->clear_term_type(); + if (encoding == InvalidEnumEncoding::Invalid) + node->set_term_type(tipb::LocalMatchAgainstBooleanTermInvalid); + } + // Encode unknown values on the wire; generated enum setters may + // assert rather than allow an out-of-range value in debug builds. + if (encoding == InvalidEnumEncoding::Unknown) + ASSERT_TRUE(node->MergeFromString(invalid_occur ? "\x08\x7f" : "\x10\x7f")); + + for (const bool empty_analyzer : {false, true}) + for (const bool null_search : {false, true}) + { + SCOPED_TRACE(fmt::format( + "invalid_occur={} encoding={} empty_analyzer={} null_search={}", + invalid_occur, + static_cast(encoding), + empty_analyzer, + null_search)); + query.set_innodb_ft_max_token_size(empty_analyzer ? 0 : 84); + auto search = createConstColumn(1, "+quick"); + if (null_search) + search = createOnlyNullColumnConst(1); + try + { + executeFunction( + "local_match_against_boolean", + {search, + createColumn({"quick fox"}), + createConstColumn(1, query.SerializeAsString())}, + nullptr, + true); + FAIL() << "Invalid Local MATCH node metadata must be rejected"; + } + catch (const Exception & error) + { + ASSERT_EQ(ErrorCodes::ILLEGAL_COLUMN, error.code()); + } + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanStandardAnalyzerProtocolSettings) +try +{ + const auto make_metadata = [](const String & term, + UInt32 min_token_size, + UInt32 max_token_size, + bool enable_stopword, + const String & stopword_collation) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + boolean_query.set_stopword_collation(stopword_collation); + boolean_query.set_innodb_ft_min_token_size(min_token_size); + boolean_query.set_innodb_ft_max_token_size(max_token_size); + boolean_query.set_stopword_mode( + enable_stopword ? tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin + : tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text(term); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + const auto short_term_metadata = make_metadata("go", 1, 84, true, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+go"), + createColumn({"go", "good"}), + createConstColumn(2, short_term_metadata)}, + nullptr, + true)); + + const auto stopword_metadata = make_metadata("the", 1, 84, false, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+the"), + createColumn({"the", "there"}), + createConstColumn(2, stopword_metadata)}, + nullptr, + true)); + + const auto collated_stopword_metadata = make_metadata("thé", 3, 84, true, "utf8mb4_general_ci"); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+thé"), + createColumn({"thé"}), + createConstColumn(1, collated_stopword_metadata)}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); + + const auto binary_server_metadata = make_metadata("The", 3, 84, true, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+The"), + createColumn({"The"}), + createConstColumn(1, binary_server_metadata)}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); + + const auto max_size_metadata = make_metadata("extraordinary", 1, 10, false, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+extraordinary"), + createColumn({"extraordinary"}), + createConstColumn(1, max_size_metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, AnalyzerDefaultsAndResultFinalization) +try +{ + struct ConfigCase + { + UInt32 min_size; + UInt32 max_size; + InferredDataVector expected; + }; + for (const auto & config : + {ConfigCase{0, 0, {1, 0, 0, 0}}, ConfigCase{0, 4, {0, 1, 0, 0}}, ConfigCase{5, 4, {0, 0, 0, 0}}}) + { + SCOPED_TRACE(fmt::format("min={} max={}", config.min_size, config.max_size)); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_innodb_ft_min_token_size(config.min_size); + query.set_innodb_ft_max_token_size(config.max_size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + for (const auto * term : {"quick", "a"}) + { + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(term); + } + const auto metadata = createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query)); + ASSERT_COLUMN_EQ( + createColumn(config.expected), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "quick a"), createColumn({"quick", "a", "", "none"}), metadata}, + nullptr, + true)); + InferredDataVector> nullable_expected; + for (auto value : config.expected) + nullable_expected.emplace_back(value); + ASSERT_COLUMN_EQ( + createColumn>(nullable_expected), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "quick a"), + createColumn>({"quick", "a", {}, ""}), + metadata}, + nullptr, + true)); + // NULL search must still bypass analyzer setup and return non-NULL + // zeros, even when that setup would reject the stopword collation. + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeBuiltin); + query.set_stopword_collation("unsupported_stopword_collation"); + ASSERT_COLUMN_EQ( + createColumn>({0, 0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn>(4, {}), + createColumn({"quick", "a", "", "none"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanNgramProtocolQuery) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + boolean_query.set_ngram_token_size(2); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("数据库"); + auto * prohibited = boolean_query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("mysql"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+数据库 -mysql"), + createColumn({"数据库系统", "MySQL 数据库", "数据库", "数据科学"}), + createConstColumn(4, metadata)}, + nullptr, + true)); + + const auto ci_collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + tipb::LocalMatchAgainstBooleanQuery case_query; + case_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + case_query.set_ngram_token_size(2); + auto * case_term = case_query.add_nodes(); + case_term->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + case_term->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + case_term->set_text("mysql"); + const String case_metadata = serializeLocalMatchAgainstBooleanQuery(case_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+mysql"), + createColumn({"MySQL", "PostgreSQL"}), + createConstColumn(2, case_metadata)}, + ci_collator, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanNgramStopwordSettings) +try +{ + const auto make_metadata = [](UInt32 token_size, bool enable_stopword) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + boolean_query.set_ngram_token_size(token_size); + boolean_query.set_stopword_mode( + enable_stopword ? tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin + : tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("the"); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + const auto query = createConstColumn(3, "+the"); + const auto documents = createColumn({"the", "other", "there"}); + + // A stopword longer than ngram_token_size is ignored by the NGRAM parser. + const auto size_two_metadata = createConstColumn(3, make_metadata(2, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, size_two_metadata}, nullptr, true)); + + // At size 3, the gram "the" is removed from both query and document. + const auto enabled_metadata = createConstColumn(3, make_metadata(3, true)); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 0}), + executeFunction("local_match_against_boolean", {query, documents, enabled_metadata}, nullptr, true)); + + tipb::LocalMatchAgainstBooleanQuery filtered_prefix; + filtered_prefix.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + filtered_prefix.set_ngram_token_size(2); + filtered_prefix.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + auto * prefix_node = filtered_prefix.add_nodes(); + prefix_node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + prefix_node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + prefix_node->set_text("caf"); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+caf*"), + createColumn({"cafe"}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(filtered_prefix))}, + nullptr, + true)); + + // Explicit OFF keeps that gram and restores the matches. + const auto disabled_metadata = createConstColumn(3, make_metadata(3, false)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, disabled_metadata}, nullptr, true)); + + // Stopword matching follows collation_server independently from the + // MATCH column collation. + tipb::LocalMatchAgainstBooleanQuery collated_query; + collated_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + collated_query.set_ngram_token_size(2); + collated_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + collated_query.set_stopword_collation("utf8mb4_general_ci"); + auto * collated_required = collated_query.add_nodes(); + collated_required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + collated_required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + collated_required->set_text("xá"); + const auto collated_metadata = createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(collated_query)); + const auto document = createColumn({"xá"}); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+xá"), document, collated_metadata}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); + + // The MATCH column collation remains accent-insensitive, but the binary + // server collation does not consider "á" to contain stopword "a". + collated_query.set_stopword_collation("utf8mb4_bin"); + const auto binary_stopword_metadata + = createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(collated_query)); + ASSERT_COLUMN_EQ( + createColumn({1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+xá"), document, binary_stopword_metadata}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanNgramPrefix) +try +{ + const auto make_metadata = [](UInt32 token_size, const String & prefix) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + boolean_query.set_ngram_token_size(token_size); + boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled); + auto * node = boolean_query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + node->set_text(prefix); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + const auto query = createConstColumn(4, "+caf*"); + const auto documents = createColumn({"CAFE", "café", "cafe", "decaf"}); + const auto metadata = createConstColumn(4, make_metadata(2, "caf")); + const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, metadata}, utf8mb4_bin, true)); + + const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, metadata}, utf8mb4_general_ci, true)); + + const auto short_query = createConstColumn(4, "+c*"); + const auto short_metadata = createConstColumn(4, make_metadata(2, "c")); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {short_query, documents, short_metadata}, + utf8mb4_general_ci, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanCollationMatrix) +try +{ + const auto query = createConstColumn(3, "+cafe"); + const auto documents = createColumn({"CAFE", "café", "cafe"}); + const auto prefix_query = createConstColumn(3, "+caf*"); + const auto make_metadata = [](tipb::LocalMatchAgainstBooleanTermType term_type, const String & term) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + auto * node = boolean_query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(term_type); + node->set_text(term); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + const auto word_metadata + = createConstColumn(3, make_metadata(tipb::LocalMatchAgainstBooleanTermWord, "cafe")); + const auto prefix_metadata + = createConstColumn(3, make_metadata(tipb::LocalMatchAgainstBooleanTermPrefix, "caf")); + + const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_bin, true)); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1}), + executeFunction("local_match_against_boolean", {prefix_query, documents, prefix_metadata}, utf8mb4_bin, true)); + + const auto utf8mb4_0900_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_0900_bin, true)); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1}), + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_0900_bin, + true)); + + const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_general_ci, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_general_ci, + true)); + + const auto utf8mb4_unicode_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_unicode_ci, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_unicode_ci, + true)); + + const auto utf8mb4_0900_ai_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_0900_ai_ci, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_0900_ai_ci, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanSplitWordModifiers) +try +{ + const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + const auto documents = createColumn({"foo only", "bar only", "foo bar", "baz foo", "baz bar", "baz qux"}); + for (const auto occur : + {tipb::LocalMatchAgainstBooleanOccurShould, + tipb::LocalMatchAgainstBooleanOccurMust, + tipb::LocalMatchAgainstBooleanOccurMustNot}) + { + tipb::LocalMatchAgainstBooleanQuery query; + auto * node = query.add_nodes(); + node->set_occur(occur); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("foo.bar"); + if (occur == tipb::LocalMatchAgainstBooleanOccurMustNot) + { + auto * positive = query.add_nodes(); + positive->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + positive->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + positive->set_text("baz"); + } + const auto expected = occur == tipb::LocalMatchAgainstBooleanOccurShould + ? createColumn({1, 1, 1, 1, 1, 0}) + : occur == tipb::LocalMatchAgainstBooleanOccurMust ? createColumn({0, 0, 1, 0, 0, 0}) + : createColumn({0, 0, 0, 0, 0, 1}); + const auto metadata = createConstColumn(6, serializeLocalMatchAgainstBooleanQuery(query)); + ASSERT_COLUMN_EQ( + expected, + executeFunction( + "local_match_against_boolean", + {createConstColumn(6, "foo.bar"), documents, metadata}, + collator, + true)); + } + // The intersection for a required split word may span MATCH columns. + tipb::LocalMatchAgainstBooleanQuery query; + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("foo.bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+foo.bar"), + createColumn({"foo", "foo"}), + createColumn({"bar", "qux"}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, BuiltinStopwordLookupMatchesComparisons) +try +{ + // Independent reference: retain the old compare-based stopword rule, + // including NGRAM's substring and source-code-point-length restriction. + const auto & stopwords = builtin_stopwords; + std::vector candidates; + for (const auto word : stopwords) + candidates.emplace_back(word); + for (const String & word : + {"ABOUT", "The", "THE", "WWW", "WITH", "ábout", "thé", "tĥe", "with", "www", "aß", "ß", + "xá", "áx", "xA", "Ax", "xáy", "aaa", "AAA", "xy", "xyz", "foo", "数据库"}) + candidates.push_back(word); + for (const auto collation : + {"binary", + "ascii_bin", + "latin1_bin", + "utf8_bin", + "utf8mb4_bin", + "utf8mb4_0900_bin", + "utf8_general_ci", + "utf8mb4_general_ci", + "utf8_unicode_ci", + "utf8mb4_unicode_ci", + "utf8mb4_0900_ai_ci"}) + { + const auto stopword_collator = TiDB::ITiDBCollator::getCollator(collation); + ASSERT_NE(stopword_collator, nullptr); + // Check the key-equality premise directly, including padding and + // combining marks which the STANDARD tokenizer may split away. + String key_buffer; + auto key_copy = [&](const String & text) { + const auto key = stopword_collator->sortKeyFastPath(text.data(), text.size(), key_buffer); + return String(key.data, key.size); + }; + for (const auto word : stopwords) + { + const String source(word); + const auto source_key = key_copy(source); + for (const auto & candidate : candidates) + for (const String & suffix : {"", " ", " ", "\u0301"}) + { + const String text = candidate + suffix; + EXPECT_EQ( + source_key == key_copy(text), + stopword_collator->compare(source.data(), source.size(), text.data(), text.size()) == 0) + << collation << " source=" << source << " candidate=" << text; + } + } + auto equals_stopword = [&](std::string_view token, size_t length, bool restrict_length) { + return std::any_of(stopwords.begin(), stopwords.end(), [&](const auto word) { + return (!restrict_length || length == word.size()) + && stopword_collator->compare(token.data(), token.size(), word.data(), word.size()) == 0; + }); + }; + for (const auto size : {0U, 1U, 2U, 3U}) + for (const auto & candidate : candidates) + { + SCOPED_TRACE(fmt::format("stopword_collation={} size={} candidate={}", collation, size, candidate)); + std::vector boundaries{0}; + for (size_t offset = 0; offset < candidate.size();) + { + offset += UTF8::utf8Decode(candidate.data() + offset, candidate.size() - offset).second; + boundaries.push_back(offset); + } + bool retained = size == 0 && !equals_stopword(candidate, boundaries.size() - 1, false); + if (size != 0) + for (size_t start = 0; start + size < boundaries.size(); ++start) + { + bool filtered = false; + for (size_t begin = start; begin < start + size; ++begin) + for (size_t end = begin + 1; end <= start + size; ++end) + filtered = filtered + || equals_stopword( + std::string_view(candidate).substr( + boundaries[begin], + boundaries[end] - boundaries[begin]), + end - begin, + true); + retained = retained || !filtered; + } + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser( + size == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_innodb_ft_min_token_size(1); + query.set_innodb_ft_max_token_size(84); + query.set_stopword_collation(collation); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(candidate); + // Repeated rows exercise key-buffer reuse. Column and server + // collations are deliberately independent. + for (const auto match_collation : {"utf8mb4_bin", "utf8mb4_general_ci"}) + ASSERT_COLUMN_EQ( + createColumn({retained ? 1.0 : 0.0, retained ? 1.0 : 0.0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, candidate), + createColumn({candidate, candidate}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(query))}, + TiDB::ITiDBCollator::getCollator(match_collation), + true)); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, NgramSlidingStopwordWindows) +try +{ + InferredDataVector> documents{ + "", + "z", + "zzaz", + "zzazzzzzzzzzzz", + "zthe", + "zzthezz", + "zzthezzzzzzzzzz", + "zaboutzz", + "zaboutzaboutzzzzzz", + "zzazzizzz", + "za zzzzzzzzzzz", + "zzz!zzz", + "zThezz", + "zTHEzz", + "zthézz", + "ztĥezz", + "zwithzz", + "zwwwzz", + "zázz", + "zAzz", + "zßzz", + "z\u0301azz", + "z🙃a🙃zz", + "z数据azzz", + "ZTHEZZ", + String("za\0zzzzzzzzzz", 13), + String("zthe\xffzzzzzzzzzz"), + String("\xffzzzzzzzzzz"), + "zzazzazzzzzzzzzzzzz", + "zaboutzzzzzzzzzz", + "z", + {}, + "zzzzzzzzzzzz"}; + // Deterministic mixed runs, ending candidates and repeated rows. The + // reference below never uses the optimized lookup or sliding state. + UInt32 seed = 70485; + const std::array pieces{"z", "a", "i", "the", "about", "thé", "🙃", "!"}; + for (size_t row = 0; row < 24; ++row) + { + String text; + for (size_t i = 0; i < 12; ++i) + { + seed = seed * 1664525 + 1013904223; + text += pieces[(seed >> 16) % pieces.size()]; + } + documents.emplace_back(std::move(text)); + } + + for (const auto stopword_collation : + {"binary", + "ascii_bin", + "latin1_bin", + "utf8_bin", + "utf8mb4_bin", + "utf8mb4_0900_bin", + "utf8_general_ci", + "utf8mb4_general_ci", + "utf8_unicode_ci", + "utf8mb4_unicode_ci", + "utf8mb4_0900_ai_ci"}) + for (const auto match_collation : {"utf8mb4_bin", "utf8mb4_general_ci"}) + for (size_t size = 1; size <= 10; ++size) + { + SCOPED_TRACE(fmt::format("stopwords={} match={} size={}", stopword_collation, match_collation, size)); + const auto stopword_collator = TiDB::ITiDBCollator::getCollator(stopword_collation); + const auto match_collator = TiDB::ITiDBCollator::getCollator(match_collation); + auto prefix = match_collator->pattern(); + prefix->compile("z%", '\\'); + InferredDataVector> expected; + for (const auto & document : documents) + { + bool retained = false; + if (document) + { + const String & text = *document; + size_t run_start = 0; + std::vector boundaries{0}; + auto check_run = [&] { + for (size_t start = 0; start + size < boundaries.size() && !retained; ++start) + { + bool filtered = false; + for (size_t begin = start; begin < start + size && !filtered; ++begin) + for (size_t end = begin + 1; end <= start + size && !filtered; ++end) + { + const auto candidate = std::string_view(text).substr( + run_start + boundaries[begin], + boundaries[end] - boundaries[begin]); + filtered = std::any_of( + builtin_stopwords.begin(), + builtin_stopwords.end(), + [&](auto word) { + return word.size() == end - begin + && stopword_collator->compare( + candidate.data(), + candidate.size(), + word.data(), + word.size()) + == 0; + }); + } + const auto gram = std::string_view(text).substr( + run_start + boundaries[start], + boundaries[start + size] - boundaries[start]); + retained = !filtered && prefix->match(gram.data(), gram.size()); + } + }; + for (size_t offset = 0; offset < text.size();) + { + const auto byte = static_cast(text[offset]); + const auto decoded = UTF8::utf8Decode(text.data() + offset, text.size() - offset); + if (decoded.second == 0 || decoded.first == UTF8::UTF8_Error + || decoded.second > text.size() - offset || (byte >= 0x80 && decoded.second == 1)) + break; + if (byte < 0x80 && !LocalMatchAgainst::isTokenChar(byte)) + { + check_run(); + run_start = ++offset; + boundaries.assign(1, 0); + } + else + { + offset += decoded.second; + boundaries.push_back(offset - run_start); + } + } + check_run(); + } + expected.emplace_back(retained ? 1.0 : 0.0); + } + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_collation(stopword_collation); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + node->set_text("z"); + ASSERT_COLUMN_EQ( + createColumn>(expected), + executeFunction( + "local_match_against_boolean", + {createConstColumn(documents.size(), "+z*"), + createColumn>(documents), + createConstColumn(documents.size(), serializeLocalMatchAgainstBooleanQuery(query))}, + match_collator, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanASCIIAndMalformedUTF8) +try +{ + const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + const auto documents = createColumn( + {"bar", + "foo bar", + "foobar", + "bar!", + String("bar\0foo", 7), + "bar\xff" + "foo", + "foo\xff" + "bar", + "foo中bar", + ""}); + for (const auto size : {0U, 2U, 3U}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(size == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("bar"); + ASSERT_COLUMN_EQ( + size == 0 ? createColumn({1, 1, 0, 1, 1, 1, 1, 0, 0}) + : createColumn({1, 1, 1, 1, 1, 1, 0, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(9, "+bar"), + documents, + createConstColumn(9, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, DocumentViewsPreserveFilteredPhrase) +try +{ + String miss; + for (size_t i = 0; i < 4096; ++i) + miss += "foo xyz zoo "; + for (const bool ngram : {false, true}) + for (const bool stopwords : {false, true}) + for (const auto collation : {"utf8mb4_bin", "utf8mb4_general_ci"}) + { + SCOPED_TRACE(fmt::format("ngram={} stopwords={} collation={}", ngram, stopwords, collation)); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + query.set_ngram_token_size(3); + query.set_stopword_mode( + stopwords ? tipb::LocalMatchAgainstStopwordModeBuiltin + : tipb::LocalMatchAgainstStopwordModeDisabled); + auto * phrase = query.add_nodes(); + phrase->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + phrase->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + phrase->set_text("foo the zoo"); + auto * excluded = query.add_nodes(); + excluded->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + excluded->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + excluded->set_text("blocked"); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(5, "+\"foo the zoo\" -blocked"), + createColumn>({miss, "foo the zoo", {}, "", miss}), + createColumn>( + {"foo the zoo", miss, "foo the zoo", "foo the zoo blocked", miss}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}, + TiDB::ITiDBCollator::getCollator(collation), + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, NgramDocumentViewsAndPositions) +try +{ + // Overlapping grams in separate runs still have consecutive positions; + // short runs contribute none. Exercise vector growth, scratch reuse and + // successive large/small/NULL documents without borrowing scratch memory. + for (const auto size : {1U, 2U, 3U, 4U, 5U, 6U, 7U, 8U, 9U, 10U}) + for (const bool chinese : {false, true}) + for (const auto collation : {"", "utf8mb4_bin", "utf8mb4_general_ci"}) + { + SCOPED_TRACE(fmt::format("size={} chinese={} collation={}", size, chinese, collation)); + const String alphabet = chinese ? "甲乙丙丁戊己庚辛壬癸子丑" : "ABCDEFGHIJKL"; + const size_t width = chinese ? 3 : 1; + const String query_text = alphabet.substr(0, (size + 1) * width); + const String first = query_text.substr(0, size * width); + const String last = query_text.substr(width); + const String short_run(size - 1, 'z'); + const String padding(4096, 'z'); + const auto documents = createColumn>( + {padding + " " + query_text, + query_text, + first + "!" + last, + first + "!" + String(size, 'z') + "!" + last, + first + "!" + short_run + "!" + last, + query_text + "\xff" + padding, + first + "\xff" + last, + {}, + "", + query_text + "!" + padding}); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(query_text); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 1, 0, 1, 1, 0, 0, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(10, query_text), + documents, + createConstColumn(10, serializeLocalMatchAgainstBooleanQuery(query))}, + String(collation).empty() ? nullptr : TiDB::ITiDBCollator::getCollator(collation), + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanUnicodeTokenBoundaries) +try +{ + const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + const auto documents = createColumn( + {"foo🙃bar", + "foo👁bar", + "foo bar", + "foobar", + "foo𞤀bar", + "foo𝟙bar", + "foo\U0002EBF0bar", + "foo\xff" + "bar"}); + for (const auto size : {0U, 2U, 3U}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(size == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(size == 0 ? "foo" : "foob"); + ASSERT_COLUMN_EQ( + size == 0 ? createColumn({1, 1, 1, 0, 1, 1, 1, 1}) + : createColumn({0, 0, 0, 1, 0, 0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(8, node->text()), + documents, + createConstColumn(8, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); + // TiDB's version-2 NGRAM Boolean lexer emits "bar" for this search. + node->set_text(size == 0 ? "𞤀bar" : "bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+𞤀bar"), + createColumn({"foo𞤀bar", "𞤀bar", "foo bar"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); + } +} +CATCH + +TEST(LocalMatchAgainstTokenChars, ProtocolV1Classification) +{ + for (const UInt32 code_point : {0x1F643U, 0x1F441U, 0x301U, 0xFFFDU, 0x2EBF0U, 0x110000U}) + EXPECT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + for (const UInt32 code_point : {0x5FU, 0x4E2DU, 0xE9U, 0x1D7D9U, 0x1E900U}) + EXPECT_TRUE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + UInt32 previous = 127; + for (const auto & span : LocalMatchAgainst::token_char_ranges) + { + ASSERT_GT(span.first, previous); + ASSERT_LE(span.first, span.last); + for (UInt32 code_point = previous + 1; code_point < span.first; ++code_point) + ASSERT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + for (UInt32 code_point = span.first; code_point <= span.last; ++code_point) + ASSERT_TRUE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + previous = span.last; + } + for (UInt32 code_point = previous + 1; code_point <= 0x10FFFFU; ++code_point) + ASSERT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + + // FNV-1a over one 0/1 byte per code point, U+0000..U+10FFFF. The paired + // TiDB tests hash Go's Unicode 15.0.0 IsLetter/IsNumber/'_' classification + // identically. This fixed regression checksum must not change in v1, + // even when the header is regenerated or either toolchain is upgraded. + std::uint64_t fingerprint = 14695981039346656037ULL; + for (UInt32 code_point = 0; code_point <= 0x10FFFFU; ++code_point) + { + fingerprint ^= static_cast(LocalMatchAgainst::isTokenChar(code_point)); + fingerprint *= 1099511628211ULL; + } + EXPECT_EQ(0x71f51f3810b3b529ULL, fingerprint); +} + +} // namespace DB::tests From d84a12034d9c6103dcc74b461d86f8c224b56cbb Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 11 Nov 2026 08:34:52 +0800 Subject: [PATCH 2/9] Functions: optimize large Local MATCH exact-term queries (cherry picked from commit 896f96e73b57a5baa52a71786693d870928b0758) Signed-off-by: Zhigao Tong --- .../Functions/FunctionsLocalMatchAgainst.cpp | 548 +++++++++++++++--- .../Functions/FunctionsLocalMatchAgainst.h | 22 +- .../tests/gtest_local_match_against.cpp | 538 ++++++++++++++++- 3 files changed, 1020 insertions(+), 88 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index 39f827553b2..761eb06dea6 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -16,6 +16,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -160,6 +164,82 @@ struct AnalyzerScratch String lowercase_run; }; +// Short first-character misses and long UCA expansions can make key creation +// costlier than a small linear scan, even above the document token threshold. +// Reserve automatic routing for large queries; forced routing stays test-only. +constexpr size_t exact_route_min_terms = 100; +constexpr size_t exact_route_min_tokens = 32; +constexpr size_t exact_route_budget = 4 * 1024 * 1024; +constexpr size_t exact_route_max_token_bytes = 4096; + +// sortKey's reusable std::string is not allocated through DB::Allocator. +// Account its heap capacity separately; all persistent route storage below +// uses the existing tracked Arena/HashMap/PODArray allocators. +class MatchKeyBuffer +{ +public: + MatchKeyBuffer() = default; + MatchKeyBuffer(const MatchKeyBuffer &) = delete; + MatchKeyBuffer & operator=(const MatchKeyBuffer &) = delete; + ~MatchKeyBuffer() { CurrentMemoryTracker::free(charged); } + StringRef key(std::string_view text, const TiDB::TiDBCollatorPtr & collator) + { + const auto result = collator->sortKeyFastPath(text.data(), text.size(), buffer); + const size_t bytes = buffer.capacity() > String{}.capacity() ? buffer.capacity() + 1 : 0; + if (bytes > charged) + { + CurrentMemoryTracker::alloc(bytes - charged); + charged = bytes; + } + return result; + } + +private: + String buffer; + size_t charged = 0; +}; + +struct ExactTermRoute +{ + struct Reference + { + size_t bucket; + size_t clause; + size_t term; + }; + struct Bucket + { + size_t offset; + size_t count; + }; + // Keys own their bytes, including empty keys and embedded zero weights. + Arena keys; + HashMap dictionary; + PODArray buckets; + PODArray references; +}; + +struct ExactMatchScratch +{ + PODArray seen; + UInt32 generation; + MatchKeyBuffer keys; + + ExactMatchScratch(size_t buckets, UInt32 initial_generation) + : generation(initial_generation) + { + seen.resize_fill(buckets, 0); + } + void nextRow() + { + if (++generation == 0) + { + std::fill(seen.begin(), seen.end(), 0); + generation = 1; + } + } +}; + struct CompiledBooleanQuery { std::vector clauses; @@ -167,6 +247,7 @@ struct CompiledBooleanQuery bool has_must_not = false; bool matches_nothing = false; bool needs_verification = false; + std::unique_ptr exact_route; }; // Normalize the already validated wire configuration once per block. Keep @@ -611,7 +692,8 @@ bool textStartsWith(std::string_view value, std::string_view prefix, const TiDB: return matcher->match(value.data(), value.size()); } -bool matchesTermAt( +// This call is per unmatched term/token, including small-query fallbacks. +ALWAYS_INLINE bool matchesTermAt( const BooleanClause & clause, size_t term_index, const FullTextToken & token, @@ -644,7 +726,8 @@ void initializeClauseMatchStates(const std::vector & clauses, std } } -bool matchesPhraseEndingAt( +// Preserve the baseline's inlining after adding indexed/test call sites. +ALWAYS_INLINE bool matchesPhraseEndingAt( const BooleanClause & clause, const FullTextColumn & column, const FullTextToken & end, @@ -715,7 +798,51 @@ enum class ColumnMatchResult Rejected, }; -ColumnMatchResult updateBooleanMatchStatesForColumn( +ALWAYS_INLINE ColumnMatchResult markClauseMatched( + const CompiledBooleanQuery & query, + size_t index, + std::vector & states, + BooleanMatchProgress & progress) +{ + states[index].matched = true; + switch (query.clauses[index].modifier) + { + case BooleanClause::Modifier::Must: + ++progress.matched_must_clauses; + break; + case BooleanClause::Modifier::Should: + progress.matched_should_clause = true; + break; + case BooleanClause::Modifier::MustNot: + return ColumnMatchResult::Rejected; + } + if (!query.has_must_not + && (query.must_clause_count != 0 ? progress.matched_must_clauses == query.must_clause_count + : progress.matched_should_clause)) + return ColumnMatchResult::Accepted; + return ColumnMatchResult::Continue; +} + +ALWAYS_INLINE ColumnMatchResult markTermMatched( + const CompiledBooleanQuery & query, + size_t clause, + size_t term, + std::vector & states, + BooleanMatchProgress & progress) +{ + auto & state = states[clause]; + if (state.matched || state.matched_terms[term]) + return ColumnMatchResult::Continue; + state.matched_terms[term] = 1; + --state.remaining_terms; + // A STANDARD SQL word may split into several terms: MUST intersects them, + // while SHOULD/MUST NOT use their union. Both matchers share this rule. + if (query.clauses[clause].modifier != BooleanClause::Modifier::Must || state.remaining_terms == 0) + return markClauseMatched(query, clause, states, progress); + return ColumnMatchResult::Continue; +} + +ALWAYS_INLINE ColumnMatchResult updateBooleanMatchStatesForColumn( const CompiledBooleanQuery & query, const FullTextColumn & column, const FullTextColumn & unfiltered_column, @@ -733,26 +860,6 @@ ColumnMatchResult updateBooleanMatchStatesForColumn( for (auto & state : states) state.phrase_verification_failed = false; - const auto mark_clause_matched = [&](size_t clause_index) { - states[clause_index].matched = true; - switch (clauses[clause_index].modifier) - { - case BooleanClause::Modifier::Must: - ++progress.matched_must_clauses; - break; - case BooleanClause::Modifier::Should: - progress.matched_should_clause = true; - break; - case BooleanClause::Modifier::MustNot: - return ColumnMatchResult::Rejected; - } - if (!query.has_must_not - && (query.must_clause_count != 0 ? progress.matched_must_clauses == query.must_clause_count - : progress.matched_should_clause)) - return ColumnMatchResult::Accepted; - return ColumnMatchResult::Continue; - }; - // Walk this MATCH column's analyzed tokens once and update every clause's // state. Phrase clauses are checked only when their final term is // encountered; positional lookups preserve stopword gaps and phrases do @@ -778,7 +885,7 @@ ColumnMatchResult updateBooleanMatchStatesForColumn( state.phrase_verification_failed = true; continue; } - const auto result = mark_clause_matched(clause_index); + const auto result = markClauseMatched(query, clause_index, states, progress); if (result != ColumnMatchResult::Continue) return result; } @@ -791,15 +898,12 @@ ColumnMatchResult updateBooleanMatchStatesForColumn( state.matched_terms[term_index] = 1; --state.remaining_terms; } - // STANDARD may split one SQL term into several words. Required - // terms intersect those words, but optional/prohibited terms use - // their union, exactly like TiDB's combineBooleanTermNodes. state.matched = clause.modifier == BooleanClause::Modifier::Must ? state.remaining_terms == 0 : state.remaining_terms < clause.terms.size(); if (state.matched) { - const auto result = mark_clause_matched(clause_index); + const auto result = markClauseMatched(query, clause_index, states, progress); if (result != ColumnMatchResult::Continue) return result; } @@ -808,6 +912,44 @@ ColumnMatchResult updateBooleanMatchStatesForColumn( return ColumnMatchResult::Continue; } +template +ColumnMatchResult updateExactMatchStatesForColumn( + const CompiledBooleanQuery & query, + const FullTextColumn & column, + const TiDB::TiDBCollatorPtr & collator, + std::vector & states, + BooleanMatchProgress & progress, + ExactMatchScratch & scratch, + LocalMatchAgainstTest::Work * work) +{ + const auto & route = *query.exact_route; + for (const auto & token : column) + { + if constexpr (count_work) + ++work->key_lookups; + const auto * const it = route.dictionary.find(scratch.keys.key(token.value(), collator)); + if (!it) + continue; + const auto bucket_id = it->getMapped(); + if (scratch.seen[bucket_id] == scratch.generation) + continue; + scratch.seen[bucket_id] = scratch.generation; + const auto & bucket = route.buckets[bucket_id]; + for (size_t i = bucket.offset; i < bucket.offset + bucket.count; ++i) + { + if constexpr (count_work) + ++work->term_references; + const auto & ref = route.references[i]; + const auto result = markTermMatched(query, ref.clause, ref.term, states, progress); + // has_must_not prevents accepting before a later prohibited + // association in this same bucket (e.g. +apple -APPLE). + if (result != ColumnMatchResult::Continue) + return result; + } + } + return ColumnMatchResult::Continue; +} + bool evaluateBooleanMatchResult(const CompiledBooleanQuery & query, const std::vector & states) { if (query.matches_nothing || query.clauses.empty()) @@ -1044,6 +1186,109 @@ CompiledBooleanQuery compileBooleanQuery( return compiled; } +size_t eligibleExactRouteTermCount( + const CompiledBooleanQuery & query, + const LocalMatchAnalyzerConfig & config, + const TiDB::TiDBCollatorPtr & collator, + LocalMatchAgainstTest::Matcher matcher, + size_t budget) +{ + if (matcher == LocalMatchAgainstTest::Matcher::Linear || config.use_ngram || !collator) + return {}; + // Compare and sortKey in these implementations encode the same weight + // stream and use identical PAD/NO PAD preprocessing. GeneralCI emits one + // 16-bit weight per rune; UCA emits the same expansion/ignorable-weight + // stream consumed by compare. Binary keys preserve byte equality. + using Type = TiDB::ITiDBCollator::CollatorType; + switch (collator->getCollatorType()) + { + case Type::UTF8_GENERAL_CI: + case Type::UTF8MB4_GENERAL_CI: + case Type::UTF8_UNICODE_CI: + case Type::UTF8MB4_UNICODE_CI: + case Type::UTF8MB4_0900_AI_CI: + case Type::UTF8MB4_BIN: + case Type::UTF8_BIN: + case Type::UTF8MB4_0900_BIN: + case Type::BINARY: + case Type::LATIN1_BIN: + case Type::ASCII_BIN: + break; + default: + return {}; + } + + size_t terms = 0; + // UCA may resize a document key buffer to 16 bytes per input byte; + // allow string capacity growth independently of the query term sizes. + size_t estimate = 32 * 1024 + 2 * 16 * exact_route_max_token_bytes; + if (estimate > budget) + return {}; + for (const auto & clause : query.clauses) + { + if (clause.phrase || clause.prefix || clause.verification) + return {}; + for (const auto & term : clause.terms) + { + if (term.size() > exact_route_max_token_bytes) + return {}; + // Conservative bound for power-of-two hash/POD capacities, + // Arena growth and up to 16 key bytes per input byte. Charge + // scratch growth too. Check before constructing any route. + const size_t cost = 256 + 96 * term.size(); + if (cost > budget - estimate) + return {}; + estimate += cost; + ++terms; + } + } + if (terms == 0 || (matcher == LocalMatchAgainstTest::Matcher::Automatic && terms < exact_route_min_terms)) + return {}; + + return terms; +} + +std::unique_ptr buildExactTermRoute( + const CompiledBooleanQuery & query, + const TiDB::TiDBCollatorPtr & collator, + size_t terms) +{ + auto route = std::make_unique(); + route->references.reserve(terms); + route->buckets.reserve(terms); + MatchKeyBuffer keys; + for (size_t clause = 0; clause < query.clauses.size(); ++clause) + for (size_t term = 0; term < query.clauses[clause].terms.size(); ++term) + { + const auto key = keys.key(query.clauses[clause].terms[term], collator); + auto * it = route->dictionary.find(key); + size_t bucket; + if (it) + bucket = it->getMapped(); + else + { + bucket = route->buckets.size(); + const StringRef owned(route->keys.insert(key.data, key.size), key.size); + bool inserted; + route->dictionary.emplace(owned, it, inserted); + it->getMapped() = bucket; + route->buckets.push_back(ExactTermRoute::Bucket{0, 0}); + } + route->references.push_back(ExactTermRoute::Reference{bucket, clause, term}); + } + std::sort(route->references.begin(), route->references.end(), [](const auto & a, const auto & b) { + return a.bucket < b.bucket; + }); + for (size_t i = 0; i < route->references.size(); ++i) + { + auto & bucket = route->buckets[route->references[i].bucket]; + if (bucket.count == 0) + bucket.offset = i; + ++bucket.count; + } + return route; +} + Float64 matchBooleanValue(const CompiledBooleanQuery & query, const std::vector & states) { // Local MATCH returns a Boolean 0/1 value, not a relevance score. @@ -1133,6 +1378,16 @@ bool decodeLocalMatchAgainstBooleanQuery(const IColumn & column, tipb::LocalMatc class FunctionLocalMatchAgainstBoolean final : public IFunction { public: + explicit FunctionLocalMatchAgainstBoolean( + LocalMatchAgainstTest::Matcher matcher_ = LocalMatchAgainstTest::Matcher::Automatic, + LocalMatchAgainstTest::Work * work_ = nullptr, + size_t budget_ = exact_route_budget, + UInt32 generation_ = 0) + : matcher(matcher_) + , work(work_) + , budget(budget_) + , generation(generation_) + {} static constexpr auto name = "local_match_against_boolean"; static FunctionPtr create(const Context &) { return std::make_shared(); } @@ -1170,6 +1425,15 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction } void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override + { + if (work) + executeInternal(block, arguments, result); + else + executeInternal(block, arguments, result); + } + + template + void executeInternal(Block & block, const ColumnNumbers & arguments, size_t result) const { const auto * query_column = typeid_cast(&*block.getByPosition(arguments[0]).column); if (query_column == nullptr) @@ -1223,8 +1487,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction if (enable_stopword) stopword_lookup.emplace(stopword_collator); const auto * stopwords = stopword_lookup ? &*stopword_lookup : nullptr; - const CompiledBooleanQuery compiled_query - = compileBooleanQuery(protocol_boolean_query, config, collator, stopwords); + CompiledBooleanQuery compiled_query = compileBooleanQuery(protocol_boolean_query, config, collator, stopwords); if (compiled_query.matches_nothing) { finish_result(); @@ -1234,80 +1497,195 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction document_columns.reserve(document_argument_end - 1); for (size_t arg = 1; arg < document_argument_end; ++arg) document_columns.emplace_back(*block.getByPosition(arguments[arg]).column); - std::vector clause_states; - initializeClauseMatchStates(compiled_query.clauses, clause_states); - FullTextColumn analyzed_column; - FullTextColumn unfiltered_column; - AnalyzerScratch analyzer_scratch; - - for (size_t row = 0; row < rows; ++row) + const size_t route_terms = eligibleExactRouteTermCount(compiled_query, config, collator, matcher, budget); + bool may_use_index = matcher == LocalMatchAgainstTest::Matcher::Indexed; + if (route_terms && !may_use_index) { - resetClauseMatchStates(clause_states); - BooleanMatchProgress match_progress; - bool result_known = false; - bool row_matches = false; - for (const auto & document_column : document_columns) + // A STANDARD token consumes at least one raw byte, and consecutive + // tokens need a separator. This cheap lower bound bypasses the + // routing row loop for wholly short blocks without tokenizing twice. + // Use one byte even for unusual analyzer bounds to avoid overflow. + constexpr size_t min_document_bytes = exact_route_min_tokens * 2 - 1; + for (const auto & column : document_columns) + { + for (size_t row = 0; row < rows; ++row) + if (!column.isNull(row) && column.getString(row).size() >= min_document_bytes) + { + may_use_index = true; + break; + } + if (may_use_index) + break; + } + } + // Compile separate row paths so ineligible queries discard optional + // scratch and routing branches from their analyzer/matcher hot loop. + const auto execute_rows = [&, + use_ngram, + ngram_token_size, + min_token_size, + max_token_size, + enable_stopword]() ALWAYS_INLINE { + std::vector clause_states; + initializeClauseMatchStates(compiled_query.clauses, clause_states); + FullTextColumn analyzed_column; + FullTextColumn unfiltered_column; + AnalyzerScratch analyzer_scratch; + + std::optional exact_scratch; + bool route_attempted = false; + for (size_t row = 0; row < rows; ++row) { - // A NULL MATCH column contributes no tokens. This is - // intentional: #70485 relies on a row with a NULL body - // still matching a required term from another MATCH column. - if (document_column.isNull(row)) + if constexpr (enable_index) + if (exact_scratch) + exact_scratch->nextRow(); + resetClauseMatchStates(clause_states); + BooleanMatchProgress match_progress; + bool result_known = false; + bool row_matches = false; + for (const auto & document_column : document_columns) { - analyzed_column.clear(); - continue; - } - analyzeColumnInto( - document_column.getString(row), - analyzed_column, - analyzer_scratch, - use_ngram, - ngram_token_size, - collator, - stopwords, - min_token_size, - max_token_size, - enable_stopword); - if (compiled_query.needs_verification) + // A NULL MATCH column contributes no tokens. This is + // intentional: #70485 relies on a row with a NULL body + // still matching a required term from another MATCH column. + if (document_column.isNull(row)) + { + analyzed_column.clear(); + continue; + } analyzeColumnInto( document_column.getString(row), - unfiltered_column, + analyzed_column, analyzer_scratch, use_ngram, ngram_token_size, collator, - nullptr, - 0, - std::numeric_limits::max(), - false); - const auto match_result = updateBooleanMatchStatesForColumn( - compiled_query, - analyzed_column, - unfiltered_column, - collator, - clause_states, - match_progress); - if (match_result == ColumnMatchResult::Rejected) - { - result_known = true; - break; - } - if (match_result == ColumnMatchResult::Accepted) - { - result_known = true; - row_matches = true; - break; + stopwords, + min_token_size, + max_token_size, + enable_stopword); + if constexpr (enable_index) + { + // Do not allocate a query dictionary for blocks containing + // only short documents. Build at most once, after metadata + // validation and complete clause compilation. Earlier short + // columns may already have contributed to this row's state. + if (!route_attempted + && (matcher == LocalMatchAgainstTest::Matcher::Indexed + || analyzed_column.size() >= exact_route_min_tokens)) + { + route_attempted = true; + compiled_query.exact_route = buildExactTermRoute(compiled_query, collator, route_terms); + if (compiled_query.exact_route) + { + exact_scratch.emplace(compiled_query.exact_route->buckets.size(), generation); + exact_scratch->nextRow(); + if constexpr (count_work) + ++work->route_builds; + } + } + } + if (compiled_query.needs_verification) + analyzeColumnInto( + document_column.getString(row), + unfiltered_column, + analyzer_scratch, + use_ngram, + ngram_token_size, + collator, + nullptr, + 0, + std::numeric_limits::max(), + false); + ColumnMatchResult match_result; + if constexpr (enable_index) + { + // STANDARD only emits valid UTF-8 BMP word tokens: malformed + // bytes become RuneError separators. Do not re-interpret the + // raw document. An oversized analyzed token falls back for + // this whole column before any state is modified. + const bool indexed = exact_scratch + && (matcher == LocalMatchAgainstTest::Matcher::Indexed + || analyzed_column.size() >= exact_route_min_tokens) + // Valid BMP tokens use at most three bytes per code + // point. The usual analyzer bound proves this without + // another column pass, including early-hit OR queries. + && (max_token_size <= exact_route_max_token_bytes / 3 + || std::all_of(analyzed_column.begin(), analyzed_column.end(), [](const auto & token) { + return token.value().size() <= exact_route_max_token_bytes; + })); + if constexpr (count_work) + { + if (indexed) + ++work->indexed_columns; + else + ++work->linear_columns; + } + match_result = indexed ? updateExactMatchStatesForColumn( + compiled_query, + analyzed_column, + collator, + clause_states, + match_progress, + *exact_scratch, + work) + : updateBooleanMatchStatesForColumn( + compiled_query, + analyzed_column, + unfiltered_column, + collator, + clause_states, + match_progress); + } + else + { + if constexpr (count_work) + ++work->linear_columns; + match_result = updateBooleanMatchStatesForColumn( + compiled_query, + analyzed_column, + unfiltered_column, + collator, + clause_states, + match_progress); + } + if (match_result == ColumnMatchResult::Rejected) + { + result_known = true; + break; + } + if (match_result == ColumnMatchResult::Accepted) + { + result_known = true; + row_matches = true; + break; + } } + output_data[row] + = result_known ? (row_matches ? 1 : 0) : matchBooleanValue(compiled_query, clause_states); } - output_data[row] = result_known ? (row_matches ? 1 : 0) : matchBooleanValue(compiled_query, clause_states); - } + }; + if (route_terms && may_use_index) + execute_rows.template operator()(); + else + execute_rows.template operator()(); finish_result(); } private: TiDB::TiDBCollatorPtr collator = nullptr; + LocalMatchAgainstTest::Matcher matcher; + LocalMatchAgainstTest::Work * work; + size_t budget; + UInt32 generation; }; } // namespace +FunctionPtr LocalMatchAgainstTest::create(Matcher matcher, Work & work, size_t budget, UInt32 generation) +{ + return std::make_shared(matcher, &work, budget, generation); +} + void registerFunctionsLocalMatchAgainst(FunctionFactory & factory) { factory.registerFunction(FunctionFactory::CaseInsensitive); diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.h b/dbms/src/Functions/FunctionsLocalMatchAgainst.h index 064c503718f..5f0928cf896 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.h +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.h @@ -19,4 +19,24 @@ namespace DB { void registerFunctionsLocalMatchAgainst(FunctionFactory & factory); -} + +// Internal test seam: no SQL setting, global state, or request-shared scratch. +namespace LocalMatchAgainstTest +{ +enum class Matcher +{ + Automatic, + Linear, + Indexed +}; +struct Work +{ + size_t route_builds = 0; + size_t indexed_columns = 0; + size_t linear_columns = 0; + size_t key_lookups = 0; + size_t term_references = 0; +}; +FunctionPtr create(Matcher matcher, Work & work, size_t budget = 4 * 1024 * 1024, UInt32 generation = 0); +} // namespace LocalMatchAgainstTest +} // namespace DB diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index c90d9968248..164f2803a70 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,9 @@ #include #include +#include +#include +#include namespace DB::ErrorCodes { @@ -38,6 +42,19 @@ class TestLocalMatchAgainst : public DB::tests::FunctionTest namespace { +constexpr std::array exact_route_collations{ + TiDB::ITiDBCollator::UTF8_GENERAL_CI, + TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI, + TiDB::ITiDBCollator::UTF8_UNICODE_CI, + TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI, + TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI, + TiDB::ITiDBCollator::UTF8MB4_BIN, + TiDB::ITiDBCollator::UTF8_BIN, + TiDB::ITiDBCollator::UTF8MB4_0900_BIN, + TiDB::ITiDBCollator::BINARY, + TiDB::ITiDBCollator::LATIN1_BIN, + TiDB::ITiDBCollator::ASCII_BIN}; + constexpr std::array builtin_stopwords{ "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", @@ -48,11 +65,11 @@ constexpr std::array builtin_stopwords{ class CountingMatchCollator final : public TiDB::ITiDBCollator { public: - CountingMatchCollator() + explicit CountingMatchCollator(bool supported = true) : ITiDBCollator(UTF8MB4_BIN) , inner(getCollator(UTF8MB4_BIN)) { - collator_type = inner->getCollatorType(); + collator_type = supported ? inner->getCollatorType() : CollatorType::MAX_; } int compare(const char * lhs, size_t lhs_size, const char * rhs, size_t rhs_size) const override @@ -1915,4 +1932,521 @@ TEST(LocalMatchAgainstTokenChars, ProtocolV1Classification) EXPECT_EQ(0x71f51f3810b3b529ULL, fingerprint); } +namespace +{ +ColumnWithTypeAndName runMatcher( + const ColumnsWithTypeAndName & columns, + TiDB::TiDBCollatorPtr collator, + LocalMatchAgainstTest::Matcher strategy, + LocalMatchAgainstTest::Work & work, + size_t budget = 4 * 1024 * 1024, + UInt32 generation = 0, + FunctionPtr reuse = {}) +{ + auto function = reuse ? reuse : LocalMatchAgainstTest::create(strategy, work, budget, generation); + function->setCollator(collator); + Block block; + DataTypes types; + ColumnNumbers arguments; + for (size_t i = 0; i < columns.size(); ++i) + { + auto column = columns[i]; + column.name = "argument" + std::to_string(i); + block.insert(column); + types.push_back(column.type); + arguments.push_back(i); + } + block.insert({nullptr, function->getReturnTypeImpl(types), "result"}); + function->executeImpl(block, arguments, columns.size()); + return block.getByPosition(columns.size()); +} + +tipb::LocalMatchAgainstBooleanQuery exactQuery(size_t terms, bool duplicate = false) +{ + tipb::LocalMatchAgainstBooleanQuery query; + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + for (size_t i = 0; i < terms; ++i) + { + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(duplicate ? "apple" : "query" + std::to_string(i)); + } + return query; +} +} // namespace + +TEST_F(TestLocalMatchAgainst, ExactRouteWorkAndBucketDeduplication) +{ + using namespace LocalMatchAgainstTest; + CountingMatchCollator collator; + auto query = exactQuery(100); + String document; + for (size_t i = 0; i < 1000; ++i) + document += "document "; + const auto columns = ColumnsWithTypeAndName{ + createConstColumn(1, "query"), + createColumn({document}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}; + Work linear, indexed; + const auto expected = runMatcher(columns, &collator, Matcher::Linear, linear); + EXPECT_EQ(100000, collator.comparisons); + collator.comparisons = 0; + ASSERT_COLUMN_EQ(expected, runMatcher(columns, &collator, Matcher::Automatic, indexed)); + EXPECT_EQ(0, collator.comparisons); + EXPECT_EQ(1, indexed.indexed_columns); + EXPECT_EQ(1000, indexed.key_lookups); + EXPECT_EQ(0, indexed.term_references); + + query = exactQuery(100, true); + auto * prohibited = query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("absent"); + document.clear(); + for (size_t i = 0; i < 500; ++i) + document += "apple "; + const auto multi = ColumnsWithTypeAndName{ + createConstColumn(2, "query"), + createColumn({document, "absent"}), + createColumn({document, document}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(query))}; + Work dedup; + ASSERT_COLUMN_EQ(createColumn({1, 0}), runMatcher(multi, &collator, Matcher::Indexed, dedup)); + EXPECT_EQ(101, dedup.term_references); // 100 positive references, then one rejection in row 2. + EXPECT_EQ(1001, dedup.key_lookups); +} + +TEST_F(TestLocalMatchAgainst, ExactRouteDifferentialAndFallbacks) +{ + using namespace LocalMatchAgainstTest; + std::mt19937 random(20261030); + const std::array words{ + "apple", + "APPLE", + "banana", + "cafe", + "café", + "Straße", + "strasse", + "ᅠᅠᅠ", + "中文文本", + "foo.bar", + "abc", + "ABC"}; + for (const auto id : + {TiDB::ITiDBCollator::UTF8MB4_BIN, + TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI, + TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI, + TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI}) + { + const auto * const collator = TiDB::ITiDBCollator::getCollator(id); + for (size_t iteration = 0; iteration < 64; ++iteration) + { + auto query = exactQuery(0); + for (size_t i = 0; i < 25; ++i) + { + auto * node = query.add_nodes(); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + const auto mode = iteration % 4; + node->set_occur( + mode == 0 ? tipb::LocalMatchAgainstBooleanOccurShould + : mode == 1 ? tipb::LocalMatchAgainstBooleanOccurMust + : static_cast(1 + random() % 3)); + node->set_text(words[random() % words.size()]); + if (mode == 3 && node->occur() == tipb::LocalMatchAgainstBooleanOccurMustNot) + node->set_text("absent"); + } + std::vector left(8), right(8); + for (size_t row = 0; row < left.size(); ++row) + for (size_t i = 0; i < 60; ++i) + { + left[row] += words[random() % words.size()] + ' '; + right[row] += words[random() % words.size()] + ' '; + } + const ColumnsWithTypeAndName columns{ + createConstColumn(8, "query"), + createColumn(left), + createColumn(right), + createConstColumn(8, serializeLocalMatchAgainstBooleanQuery(query))}; + Work old_work, new_work; + ASSERT_COLUMN_EQ( + runMatcher(columns, collator, Matcher::Linear, old_work), + runMatcher( + columns, + collator, + Matcher::Indexed, + new_work, + 4 * 1024 * 1024, + std::numeric_limits::max() - 1)); + } + } + auto query = exactQuery(100, true); + String document; + for (size_t i = 0; i < 100; ++i) + document += "apple "; + const auto check = [&](const auto & q, TiDB::TiDBCollatorPtr collator, size_t budget) { + const ColumnsWithTypeAndName columns{ + createConstColumn(1, "query"), + createColumn({document}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(q))}; + Work old_work, new_work; + ASSERT_COLUMN_EQ( + runMatcher(columns, collator, Matcher::Linear, old_work), + runMatcher(columns, collator, Matcher::Indexed, new_work, budget)); + EXPECT_EQ(0, new_work.indexed_columns); + }; + const auto * collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + check(query, collator, 1); + check(query, nullptr, 4 * 1024 * 1024); + CountingMatchCollator unsupported(false); + check(query, &unsupported, 4 * 1024 * 1024); + query.mutable_nodes(0)->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + check(query, collator, 4 * 1024 * 1024); + query.mutable_nodes(0)->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + check(query, collator, 4 * 1024 * 1024); + query.set_parser(tipb::LocalMatchAgainstParserNgram); + check(query, collator, 4 * 1024 * 1024); +} + +TEST_F(TestLocalMatchAgainst, ExactRouteEquivalentProhibitedAndColumnFallback) +{ + using namespace LocalMatchAgainstTest; + const auto * const collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); + auto query = exactQuery(100, true); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("APPLE"); + Work work; + ASSERT_COLUMN_EQ( + createColumn({0}), + runMatcher( + {createConstColumn(1, "query"), + createColumn({"apple"}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + Matcher::Indexed, + work)); + + query = exactQuery(100, true); + query.set_innodb_ft_max_token_size(10000); + node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("banana"); + String oversized(5000, 'x'); + Work fallback; + const auto result = runMatcher( + {createConstColumn(2, "query"), + createColumn({"apple", "apple"}), + createColumn({oversized + " banana", oversized}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + Matcher::Indexed, + fallback); + ASSERT_COLUMN_EQ(createColumn({1, 0}), result); + EXPECT_EQ(2, fallback.indexed_columns); + EXPECT_EQ(2, fallback.linear_columns); +} + +TEST_F(TestLocalMatchAgainst, ExactRouteThresholdNullableAndReusedBlocks) +{ + using namespace LocalMatchAgainstTest; + const auto * const collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + String document; + for (size_t i = 0; i < 100; ++i) + document += "apple "; + for (size_t count : {1, 4, 15, 16, 17, 99, 100, 101}) + { + const auto query = exactQuery(count, true); + Work work; + ASSERT_COLUMN_EQ( + createColumn({1}), + runMatcher( + {createConstColumn(1, "query"), + createColumn({document}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + Matcher::Automatic, + work)); + EXPECT_EQ(count >= 100 ? 1 : 0, work.indexed_columns); + } + for (size_t tokens : {31, 32}) + { + auto query = exactQuery(100, true); + query.set_innodb_ft_min_token_size(1); + query.set_innodb_ft_max_token_size(84); + String short_tokens; + for (size_t i = 0; i < tokens; ++i) + short_tokens += i == 0 ? "a" : " a"; + Work linear, automatic; + const ColumnsWithTypeAndName columns{ + createConstColumn(1, "query"), + createColumn({short_tokens}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}; + ASSERT_COLUMN_EQ( + runMatcher(columns, collator, Matcher::Linear, linear), + runMatcher(columns, collator, Matcher::Automatic, automatic)); + EXPECT_EQ(tokens == 32 ? 1 : 0, automatic.indexed_columns); + } + { + const auto query = exactQuery(100, true); + Work short_work; + ASSERT_COLUMN_EQ( + createColumn({1}), + runMatcher( + {createConstColumn(1, "query"), + createColumn({"apple"}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + Matcher::Automatic, + short_work)); + EXPECT_EQ(0, short_work.route_builds); + EXPECT_EQ(1, short_work.linear_columns); + } + Work work; + auto function = LocalMatchAgainstTest::create(Matcher::Indexed, work); + for (size_t rows : {4, 0, 4}) + { + const auto query = exactQuery(100, true); + auto column = createColumn>({document, {}, "absent", document}); + column.column = column.column->cloneResized(rows); + auto expected = createColumn>({1, 0, 0, 1}); + expected.column = expected.column->cloneResized(rows); + ASSERT_COLUMN_EQ( + expected, + runMatcher( + {createConstColumn(rows, "query"), + column, + createConstColumn(rows, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + Matcher::Indexed, + work, + 4 * 1024 * 1024, + 0, + function)); + } + auto query = exactQuery(100, true); + query.mutable_nodes(0)->set_text("banana"); + ASSERT_COLUMN_EQ( + createColumn({0}), + runMatcher( + {createConstColumn(1, "query"), + createColumn({document}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + Matcher::Indexed, + work, + 4 * 1024 * 1024, + 0, + function)); +} + +TEST_F(TestLocalMatchAgainst, ExactRouteIndependentConcurrentExecutions) +{ + using namespace LocalMatchAgainstTest; + const auto * const collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); + const auto query = exactQuery(100, true); + String document; + for (size_t i = 0; i < 100; ++i) + document += "apple "; + const ColumnsWithTypeAndName columns{ + createConstColumn(4, "query"), + createColumn({document, "absent", document, "absent"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}; + std::vector> tasks; + for (size_t i = 0; i < 8; ++i) + tasks.push_back(std::async(std::launch::async, [&] { + Work work; + return runMatcher(columns, collator, Matcher::Indexed, work); + })); + for (auto & task : tasks) + ASSERT_COLUMN_EQ(createColumn({1, 0, 1, 0}), task.get()); +} + +TEST(LocalMatchAgainstTokenChars, ExactRouteKeyEquality) +{ + std::mt19937 random(20261030); + for (const auto id : exact_route_collations) + { + SCOPED_TRACE(id); + const auto * const collator = TiDB::ITiDBCollator::getCollator(id); + std::unordered_map representatives; + std::vector words{"café", "CAFE", "Straße", "strasse", "fi", "fi", "ᅠ", "", "abc", "ABC"}; + String buffer; + for (const auto & a : words) + for (const auto & b : words) + { + auto key = collator->sortKeyFastPath(a.data(), a.size(), buffer); + const String owned(key.data, key.size); + key = collator->sortKeyFastPath(b.data(), b.size(), buffer); + ASSERT_EQ( + collator->compare(a.data(), a.size(), b.data(), b.size()) == 0, + owned == String(key.data, key.size)); + } + for (UInt32 cp = 128; cp <= 0xFFFF; ++cp) + { + if (!LocalMatchAgainst::isTokenChar(cp)) + continue; + String word; + if (cp < 0x800) + { + word += static_cast(0xC0 | cp >> 6); + word += static_cast(0x80 | (cp & 63)); + } + else + { + word += static_cast(0xE0 | cp >> 12); + word += static_cast(0x80 | ((cp >> 6) & 63)); + word += static_cast(0x80 | (cp & 63)); + } + const auto key = collator->sortKeyFastPath(word.data(), word.size(), buffer); + auto [it, inserted] = representatives.emplace(String(key.data, key.size), word); + if (!inserted) + ASSERT_EQ(0, collator->compare(word.data(), word.size(), it->second.data(), it->second.size())); + if (random() % 32 == 0) + words.push_back(word); + } + for (size_t i = 0; i < 10000; ++i) + { + const String a = words[random() % words.size()] + words[random() % words.size()]; + const String b = i % 2 == 0 ? a : words[random() % words.size()] + words[random() % words.size()]; + auto key = collator->sortKeyFastPath(a.data(), a.size(), buffer); + const String owned(key.data, key.size); + key = collator->sortKeyFastPath(b.data(), b.size(), buffer); + ASSERT_EQ( + collator->compare(a.data(), a.size(), b.data(), b.size()) == 0, + owned == String(key.data, key.size)); + } + } +} + +TEST(LocalMatchAgainstTokenChars, ExactRouteKeyEqualitySpecialCases) +{ + // Include raw collation boundaries as well as valid STANDARD tokens: + // trailing spaces, embedded NUL, UCA expansions, zero weights and accents. + // The two calls deliberately reuse one buffer; retain the returned size + // and copy each key before a later call can overwrite its storage. + const std::vector words{ + "", + " ", + " ", + "abc", + "ABC", + "abc ", + "abc ", + "abc\t", + "cafe", + "CAFÉ", + "café", + "cafe\u0301", + "Straße", + "strasse", + "fi", + "fi", + "ffi", + "ffi", + "æ", + "ae", + "\uFDFA", + "\uFDFB", + "جل جلاله", + "㌖", + "キロメートル", + "x㌖y", + "xキロメートルy", + "㌖\u034F", + "\u034F", + "\u00AD", + "\u200D", + "\u1160", + "\uFE0F", + "a\u034Fb", + "ab", + "a\u00ADb", + "a\u200Db", + "a\u1160b", + "a\uFE0Fb", + "中文", + "ABC", + "𠀀", + "😀", + String(1, '\0'), + String("abc\0", 4), + String("abc\0 ", 5), + String("a\0b", 3), + String(512, 'a'), + "a"}; + for (const auto id : exact_route_collations) + { + const auto * const collator = TiDB::ITiDBCollator::getCollator(id); + ASSERT_NE(collator, nullptr); + String buffer; + const auto key = [&](const String & word) { + const auto result = collator->sortKeyFastPath(word.data(), word.size(), buffer); + return String(result.data, result.size); + }; + for (size_t i = 0; i < words.size(); ++i) + for (size_t j = 0; j < words.size(); ++j) + { + SCOPED_TRACE(fmt::format("collation={} left={} right={}", id, i, j)); + const auto left = key(words[i]); + const auto right = key(words[j]); + ASSERT_EQ( + collator->compare(words[i].data(), words[i].size(), words[j].data(), words[j].size()) == 0, + left == right); + } + + // Explicit expected semantics prevent a shared trimming error from + // making the compare/key differential pass unnoticed. + const bool padded = id != TiDB::ITiDBCollator::BINARY && id != TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI + && id != TiDB::ITiDBCollator::UTF8MB4_0900_BIN; + SCOPED_TRACE(id); + EXPECT_EQ(padded, collator->compare("abc", 3, "abc ", 4) == 0); + const auto unpadded_key = key("abc"); + EXPECT_EQ(padded, unpadded_key == key("abc ")); + } +} + +TEST_F(TestLocalMatchAgainst, ExactRouteMalformedUTF8AfterTokenization) +{ + using namespace LocalMatchAgainstTest; + const std::array malformed{ + "\xff", + "\x80", + "\xc0\xaf", + "\xe0\x80\xaf", + "\xed\xa0\x80", + "\xf4\x90\x80\x80", + "\xc2", + "\xe4\xb8", + "\xf0\x9f"}; + String padding; + for (size_t i = 0; i < 40; ++i) + padding += " padding"; + for (const auto id : exact_route_collations) + for (size_t i = 0; i < malformed.size(); ++i) + { + SCOPED_TRACE(fmt::format("collation={} malformed={}", id, i)); + const auto * const collator = TiDB::ITiDBCollator::getCollator(id); + auto query = exactQuery(100); + const auto broken = "foo" + malformed[i] + "bar"; + for (auto & node : *query.mutable_nodes()) + node.set_text(broken); + const ColumnsWithTypeAndName columns{ + createConstColumn(5, "query"), + createColumn( + {broken + padding, "foo bar" + padding, "foobar" + padding, "foo" + padding, padding}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}; + Work linear, indexed, automatic; + const auto expected = runMatcher(columns, collator, Matcher::Linear, linear); + ASSERT_COLUMN_EQ(createColumn({1, 1, 0, 0, 0}), expected); + ASSERT_COLUMN_EQ(expected, runMatcher(columns, collator, Matcher::Indexed, indexed)); + ASSERT_COLUMN_EQ(expected, runMatcher(columns, collator, Matcher::Automatic, automatic)); + EXPECT_EQ(5, indexed.indexed_columns); + EXPECT_EQ(5, automatic.indexed_columns); + } +} + } // namespace DB::tests From f600a2aac7a6ef5acfaa46c9c532c1732bfe3b62 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 11 Nov 2026 00:04:25 +0800 Subject: [PATCH 3/9] Functions: defer Local MATCH raw phrase analysis Generate the unfiltered token stream only after a filtered phrase matches, and reuse it for later clauses in the same document column. ref pingcap/tiflash#10233 Signed-off-by: Zhigao Tong (cherry picked from commit 6304660547b4f0b935a47cb1f3ed3daed76be3ee) --- .../Functions/FunctionsLocalMatchAgainst.cpp | 62 ++++++++++++------- .../Functions/FunctionsLocalMatchAgainst.h | 1 + .../tests/gtest_local_match_against.cpp | 34 ++++++++++ 3 files changed, 73 insertions(+), 24 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index 761eb06dea6..7db02f15390 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -246,7 +246,6 @@ struct CompiledBooleanQuery size_t must_clause_count = 0; bool has_must_not = false; bool matches_nothing = false; - bool needs_verification = false; std::unique_ptr exact_route; }; @@ -842,10 +841,11 @@ ALWAYS_INLINE ColumnMatchResult markTermMatched( return ColumnMatchResult::Continue; } +template ALWAYS_INLINE ColumnMatchResult updateBooleanMatchStatesForColumn( const CompiledBooleanQuery & query, const FullTextColumn & column, - const FullTextColumn & unfiltered_column, + GetUnfilteredColumn && get_unfiltered_column, const TiDB::TiDBCollatorPtr & collator, std::vector & states, BooleanMatchProgress & progress) @@ -877,13 +877,16 @@ ALWAYS_INLINE ColumnMatchResult updateBooleanMatchStatesForColumn( { if (matchesPhraseEndingAt(clause, column, token, collator)) { - if (clause.verification - && !std::any_of(unfiltered_column.begin(), unfiltered_column.end(), [&](const auto & end) { - return matchesPhraseEndingAt(*clause.verification, unfiltered_column, end, collator); - })) + if (clause.verification) { - state.phrase_verification_failed = true; - continue; + const auto & unfiltered_column = get_unfiltered_column(); + if (!std::any_of(unfiltered_column.begin(), unfiltered_column.end(), [&](const auto & end) { + return matchesPhraseEndingAt(*clause.verification, unfiltered_column, end, collator); + })) + { + state.phrase_verification_failed = true; + continue; + } } const auto result = markClauseMatched(query, clause_index, states, progress); if (result != ColumnMatchResult::Continue) @@ -1142,7 +1145,6 @@ CompiledBooleanQuery compileBooleanQuery( clause.verification = std::make_unique(); clause.verification->phrase = true; collectClauseTerms(*clause.verification, raw_terms); - compiled.needs_verification = true; } } @@ -1553,8 +1555,9 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction analyzed_column.clear(); continue; } + const auto document = document_column.getString(row); analyzeColumnInto( - document_column.getString(row), + document, analyzed_column, analyzer_scratch, use_ngram, @@ -1585,18 +1588,29 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction } } } - if (compiled_query.needs_verification) - analyzeColumnInto( - document_column.getString(row), - unfiltered_column, - analyzer_scratch, - use_ngram, - ngram_token_size, - collator, - nullptr, - 0, - std::numeric_limits::max(), - false); + // A filtered phrase hit is the first point that needs the + // raw tokens. Share the stream across clauses in this column. + bool unfiltered_ready = false; + const auto get_unfiltered_column = [&]() -> const FullTextColumn & { + if (!unfiltered_ready) + { + analyzeColumnInto( + document, + unfiltered_column, + analyzer_scratch, + use_ngram, + ngram_token_size, + collator, + nullptr, + 0, + std::numeric_limits::max(), + false); + unfiltered_ready = true; + if constexpr (count_work) + ++work->unfiltered_column_analyses; + } + return unfiltered_column; + }; ColumnMatchResult match_result; if constexpr (enable_index) { @@ -1632,7 +1646,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction : updateBooleanMatchStatesForColumn( compiled_query, analyzed_column, - unfiltered_column, + get_unfiltered_column, collator, clause_states, match_progress); @@ -1644,7 +1658,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction match_result = updateBooleanMatchStatesForColumn( compiled_query, analyzed_column, - unfiltered_column, + get_unfiltered_column, collator, clause_states, match_progress); diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.h b/dbms/src/Functions/FunctionsLocalMatchAgainst.h index 5f0928cf896..cd39439e5fb 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.h +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.h @@ -36,6 +36,7 @@ struct Work size_t linear_columns = 0; size_t key_lookups = 0; size_t term_references = 0; + size_t unfiltered_column_analyses = 0; }; FunctionPtr create(Matcher matcher, Work & work, size_t budget = 4 * 1024 * 1024, UInt32 generation = 0); } // namespace LocalMatchAgainstTest diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index 164f2803a70..506a4a57527 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -1976,6 +1976,40 @@ tipb::LocalMatchAgainstBooleanQuery exactQuery(size_t terms, bool duplicate = fa } } // namespace +TEST_F(TestLocalMatchAgainst, FilteredPhraseAnalyzesRawColumnOnDemand) +try +{ + const auto * collator = TiDB::ITiDBCollator::getCollator("utf8mb4_bin"); + for (const auto parser : {tipb::LocalMatchAgainstParserStandard, tipb::LocalMatchAgainstParserNgram}) + { + SCOPED_TRACE(parser); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(parser); + query.set_ngram_token_size(3); + for (const auto * text : {"foo the zoo", "red the qux"}) + { + auto * phrase = query.add_nodes(); + phrase->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + phrase->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + phrase->set_text(text); + } + const ColumnsWithTypeAndName columns{ + createConstColumn(6, "+\"foo the zoo\" +\"red the qux\""), + createColumn>( + {"unrelated", "foo the zoo red the qux", "foo xyz zoo red the qux", "foo xyz zoo", "foo the zoo", {}}), + createColumn>( + {"filler", "filler", "foo the zoo", "filler", "red the qux", "foo the zoo red the qux"}), + createConstColumn(6, serializeLocalMatchAgainstBooleanQuery(query))}; + LocalMatchAgainstTest::Work work; + ASSERT_COLUMN_EQ( + createColumn>({0, 1, 1, 0, 1, 1}), + runMatcher(columns, collator, LocalMatchAgainstTest::Matcher::Automatic, work)); + // Raw scans by row: 0, 1 (two clauses share a scan), 2, 1, 2, 1. + EXPECT_EQ(work.unfiltered_column_analyses, 7); + } +} +CATCH + TEST_F(TestLocalMatchAgainst, ExactRouteWorkAndBucketDeduplication) { using namespace LocalMatchAgainstTest; From d2db3513955f3208553cbfa7864a635db28dc61a Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 11 Nov 2026 08:41:51 +0800 Subject: [PATCH 4/9] Functions: optimize Local MATCH stopword lookup Share the collation key table between STANDARD and NGRAM. Cache repeated short token results per block and stop probing when the initial hit rate is low. Signed-off-by: Zhigao Tong (cherry picked from commit 0a7545bc01e2b9c29d743c40a0b163c972c5fb92) --- .../Functions/FunctionsLocalMatchAgainst.cpp | 136 ++++++++++++------ 1 file changed, 93 insertions(+), 43 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index 7db02f15390..b7b974c412d 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -35,12 +35,12 @@ #include #include +#include #include #include #include #include #include -#include #include #include #include @@ -368,65 +368,128 @@ FullTextColumn tokenizeText(std::string_view text, const TiDB::TiDBCollatorPtr & return result; } +constexpr auto builtin_stopwords = std::to_array( + {"a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", + "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", + "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}); + +// The built-in list is ASCII, so byte and code-point lengths agree. +constexpr size_t max_builtin_stopword_length = [] { + size_t result = 0; + for (const auto word : builtin_stopwords) + result = std::max(result, word.size()); + return result; +}(); +static_assert(max_builtin_stopword_length < std::numeric_limits::digits); + const std::unordered_set & defaultStopwords() { - static const std::unordered_set stopwords{ - "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", - "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", - "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; + static const auto stopwords = [] { + std::unordered_set result; + result.reserve(builtin_stopwords.size()); + for (const auto word : builtin_stopwords) + result.emplace(word); + return result; + }(); return stopwords; } -// Owned collation keys are compiled once per executeImpl, then reused by -// query analysis and every document in the block. The scratch key is local -// to that execution, never shared across concurrent calls. Use the stopword -// (server) collation, not the MATCH column collation. +// Collation keys own their bytes and are reused across the block. Use the +// stopword (server) collation independently of the MATCH column collation. class BuiltinStopwordLookup { public: explicit BuiltinStopwordLookup(TiDB::TiDBCollatorPtr collator_) : collator(collator_) { - for (const auto & stopword : defaultStopwords()) + for (const auto stopword : builtin_stopwords) { const auto key = collator->sortKeyFastPath(stopword.data(), stopword.size(), key_buffer); - // sortKeyFastPath may reference its input or key_buffer; copy it. - String owned_key(key.data, key.size); - keys.insert(owned_key); - // The built-in list is ASCII. Preserve NGRAM's code-point length - // buckets even for collations with expansions/ignorable weights. - keys_by_length[stopword.size()].insert(std::move(owned_key)); + max_key_size = std::max(max_key_size, key.size); + auto * entry = keys.find(key); + if (!entry) + { + // The sort key may reference the reusable buffer. + const StringRef owned(key_storage.insert(key.data, key.size), key.size); + bool inserted; + keys.emplace(owned, entry, inserted); + entry->getMapped() = 0; + } + entry->getMapped() |= lengthBit(stopword.size()); } } - bool contains(std::string_view token) const { return contains(token, keys); } + bool contains(std::string_view token) const { return matchingLengths(token) != 0; } bool contains(std::string_view token, size_t code_points) const { - const auto bucket = keys_by_length.find(code_points); - return bucket != keys_by_length.end() && contains(token, bucket->second); + // NGRAM also requires the original code-point count, even when a + // collation makes words of different lengths share a key. + return code_points <= max_builtin_stopword_length && (matchingLengths(token) & lengthBit(code_points)) != 0; } private: - struct KeyHash + static UInt8 lengthBit(size_t length) { return static_cast(1U << length); } + + struct CacheEntry { - using is_transparent = void; - size_t operator()(std::string_view key) const { return std::hash{}(key); } + UInt64 bytes = 0; + UInt8 size = 0; + UInt8 lengths = 0; }; - using Keys = std::unordered_set>; - bool contains(std::string_view token, const Keys & lookup) const + UInt8 matchingLengths(std::string_view token) const + { + const size_t size = token.size(); + if (size == 0 || size > cache_token_bytes) + return lookupMatchingLengths(token); + + // The first and last loads cover the complete token. The size + // disambiguates equal packed values from tokens of different lengths. + const char * data = token.data(); + UInt64 bytes; + if (size >= 4) + bytes = (static_cast(unalignedLoad(data)) << 32) | unalignedLoad(data + size - 4); + else if (size >= 2) + bytes = (static_cast(unalignedLoad(data)) << 16) | unalignedLoad(data + size - 2); + else + bytes = static_cast(*data); + + auto & entry = cache[intHashCRC32(bytes) & (cache.size() - 1)]; + const bool hit = entry.size == size && entry.bytes == bytes; + if (unlikely(cache_samples_remaining != 0)) + { + cache_sample_hits += hit; + if (--cache_samples_remaining == 0 && cache_sample_hits < cache.size() / 4) + cache_token_bytes = 0; + } + if (!hit) + entry = {bytes, static_cast(size), lookupMatchingLengths(token)}; + return entry.lengths; + } + + UInt8 lookupMatchingLengths(std::string_view token) const { - // sortKey (rather than sortKeyNoTrim) preserves PAD SPACE/NO PAD - // equality, as well as the collator's case/accent semantics. const auto key = collator->sortKeyFastPath(token.data(), token.size(), key_buffer); - return lookup.contains(std::string_view(key.data, key.size)); + if (key.size > max_key_size) + return 0; + const auto * entry = keys.find(key); + return entry ? entry->getMapped() : 0; } TiDB::TiDBCollatorPtr collator; - Keys keys; - std::unordered_map keys_by_length; + Arena key_storage; + HashMap> keys; + size_t max_key_size = 0; mutable String key_buffer; + // A lookup and its collation are local to one block. Cache exact source + // bytes, including misses, without keeping references to document data. + mutable std::array cache; + // Sample the first cache-sized batch of short tokens. Avoid the cost of + // probing the cache for the rest of the block when most probes miss. + mutable size_t cache_samples_remaining = cache.size(); + mutable size_t cache_sample_hits = 0; + mutable size_t cache_token_bytes = sizeof(UInt64); }; bool isDefaultStopword(std::string_view token, const BuiltinStopwordLookup * stopwords) @@ -434,19 +497,6 @@ bool isDefaultStopword(std::string_view token, const BuiltinStopwordLookup * sto return stopwords ? stopwords->contains(token) : defaultStopwords().contains(Poco::UTF8::toLower(String(token))); } -size_t maxBuiltinStopwordLength() -{ - // The supported built-in list is ASCII: byte and code-point lengths are - // identical. Do not examine lengths which cannot match any stopword. - static const size_t max_length = [] { - size_t result = 0; - for (const auto & word : defaultStopwords()) - result = std::max(result, word.size()); - return result; - }(); - return max_length; -} - bool containsNgramStopword( std::string_view token, const std::vector & char_boundaries, @@ -520,7 +570,7 @@ void analyzeNgramTextInto( return; const bool preserve_case = enable_stopword && stopwords; - const size_t max_stopword_length = enable_stopword ? std::min(ngram_token_size, maxBuiltinStopwordLength()) : 0; + const size_t max_stopword_length = enable_stopword ? std::min(ngram_token_size, max_builtin_stopword_length) : 0; size_t next_position_base = 0; auto & char_boundaries = scratch.char_boundaries; for (size_t offset = 0; offset < text.size();) From 6be59f6d1382aafbc7bb8556f428029478bf2969 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 11 Nov 2026 10:22:07 +0800 Subject: [PATCH 5/9] Functions: align Local MATCH token classification with Unicode 17 Regenerate the character table from the TiDB release-8.5 Go toolchain, cover a newly classified BMP letter, and retain Local MATCH protocol version 2. Signed-off-by: Zhigao Tong --- .../Functions/FunctionsLocalMatchAgainst.cpp | 7 +- .../Functions/LocalMatchAgainstTokenChars.h | 180 +++++++++--------- .../tests/gtest_local_match_against.cpp | 24 +-- 3 files changed, 107 insertions(+), 104 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index b7b974c412d..3dc0d736b1e 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -57,11 +57,8 @@ namespace constexpr size_t default_min_token_size = 3; constexpr size_t default_max_token_size = 84; constexpr size_t default_ngram_token_size = 2; -// This is the Local MATCH semantic protocol version, not the TiDB/TiFlash -// product version. Version 2 fixes the Boolean AST interpretation, analyzer -// behavior, and built-in stopword set. TiFlash must support a version before -// TiDB emits it; unknown versions are rejected, and existing versions must -// never be reinterpreted with changed semantics. +// Local MATCH protocol version. TiFlash rejects unknown versions. TiDB and +// TiFlash must use the same Unicode classification when evaluating a query. // Version 2 returns zero for NULL searches and uses MySQL's multibyte NGRAM // document scanner. STANDARD and the NGRAM Boolean lexer use BMP word // characters; filtered phrases also verify the unfiltered document stream. diff --git a/dbms/src/Functions/LocalMatchAgainstTokenChars.h b/dbms/src/Functions/LocalMatchAgainstTokenChars.h index ec2d3987c7c..abb9e63d045 100644 --- a/dbms/src/Functions/LocalMatchAgainstTokenChars.h +++ b/dbms/src/Functions/LocalMatchAgainstTokenChars.h @@ -21,22 +21,22 @@ namespace DB::LocalMatchAgainst { -// Generated from Go's Unicode 15.0.0 categories L and N. TiDB uses its standard -// library and strictly tests that Unicode version and the Unicode 15.0.0 checksum. +// Generated from Go's Unicode 17.0.0 categories L and N. TiDB uses its standard +// library and strictly tests that Unicode version and the Unicode 17.0.0 checksum. // Equivalent Go predicate: // word := r <= unicode.MaxRune && // (unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_') // This helper covers all Unicode code points; the Boolean lexer separately // restricts word characters to the BMP (r <= 0xFFFF). -// Regenerate/check this header using the paired TiDB generator. Never change -// an existing protocol version's classification; deploy TiFlash support for a -// new version before TiDB emits it. ASCII and '_' use the fast path below. +// Regenerate/check this header using the paired TiDB generator. TiDB and +// TiFlash must use the same character classification for Local MATCH. +// ASCII and '_' use the fast path below. struct TokenCharRange { std::uint32_t first; std::uint32_t last; }; -inline constexpr std::array token_char_ranges{{ +inline constexpr std::array token_char_ranges{{ {0xAA, 0xAA}, {0xB2, 0xB3}, {0xB5, 0xB5}, {0xB9, 0xBA}, {0xBC, 0xBE}, {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x2C1}, {0x2C6, 0x2D1}, {0x2E0, 0x2E4}, {0x2EC, 0x2EC}, {0x2EE, 0x2EE}, {0x370, 0x374}, {0x376, 0x377}, {0x37A, 0x37D}, @@ -47,7 +47,7 @@ inline constexpr std::array token_char_ranges{{ {0x6FF, 0x6FF}, {0x710, 0x710}, {0x712, 0x72F}, {0x74D, 0x7A5}, {0x7B1, 0x7B1}, {0x7C0, 0x7EA}, {0x7F4, 0x7F5}, {0x7FA, 0x7FA}, {0x800, 0x815}, {0x81A, 0x81A}, {0x824, 0x824}, {0x828, 0x828}, {0x840, 0x858}, {0x860, 0x86A}, {0x870, 0x887}, - {0x889, 0x88E}, {0x8A0, 0x8C9}, {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, + {0x889, 0x88F}, {0x8A0, 0x8C9}, {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, {0x958, 0x961}, {0x966, 0x96F}, {0x971, 0x980}, {0x985, 0x98C}, {0x98F, 0x990}, {0x993, 0x9A8}, {0x9AA, 0x9B0}, {0x9B2, 0x9B2}, {0x9B6, 0x9B9}, {0x9BD, 0x9BD}, {0x9CE, 0x9CE}, {0x9DC, 0x9DD}, {0x9DF, 0x9E1}, {0x9E6, 0x9F1}, {0x9F4, 0x9F9}, @@ -61,9 +61,9 @@ inline constexpr std::array token_char_ranges{{ {0xB85, 0xB8A}, {0xB8E, 0xB90}, {0xB92, 0xB95}, {0xB99, 0xB9A}, {0xB9C, 0xB9C}, {0xB9E, 0xB9F}, {0xBA3, 0xBA4}, {0xBA8, 0xBAA}, {0xBAE, 0xBB9}, {0xBD0, 0xBD0}, {0xBE6, 0xBF2}, {0xC05, 0xC0C}, {0xC0E, 0xC10}, {0xC12, 0xC28}, {0xC2A, 0xC39}, - {0xC3D, 0xC3D}, {0xC58, 0xC5A}, {0xC5D, 0xC5D}, {0xC60, 0xC61}, {0xC66, 0xC6F}, + {0xC3D, 0xC3D}, {0xC58, 0xC5A}, {0xC5C, 0xC5D}, {0xC60, 0xC61}, {0xC66, 0xC6F}, {0xC78, 0xC7E}, {0xC80, 0xC80}, {0xC85, 0xC8C}, {0xC8E, 0xC90}, {0xC92, 0xCA8}, - {0xCAA, 0xCB3}, {0xCB5, 0xCB9}, {0xCBD, 0xCBD}, {0xCDD, 0xCDE}, {0xCE0, 0xCE1}, + {0xCAA, 0xCB3}, {0xCB5, 0xCB9}, {0xCBD, 0xCBD}, {0xCDC, 0xCDE}, {0xCE0, 0xCE1}, {0xCE6, 0xCEF}, {0xCF1, 0xCF2}, {0xD04, 0xD0C}, {0xD0E, 0xD10}, {0xD12, 0xD3A}, {0xD3D, 0xD3D}, {0xD4E, 0xD4E}, {0xD54, 0xD56}, {0xD58, 0xD61}, {0xD66, 0xD78}, {0xD7A, 0xD7F}, {0xD85, 0xD96}, {0xD9A, 0xDB1}, {0xDB3, 0xDBB}, {0xDBD, 0xDBD}, @@ -86,7 +86,7 @@ inline constexpr std::array token_char_ranges{{ {0x1970, 0x1974}, {0x1980, 0x19AB}, {0x19B0, 0x19C9}, {0x19D0, 0x19DA}, {0x1A00, 0x1A16}, {0x1A20, 0x1A54}, {0x1A80, 0x1A89}, {0x1A90, 0x1A99}, {0x1AA7, 0x1AA7}, {0x1B05, 0x1B33}, {0x1B45, 0x1B4C}, {0x1B50, 0x1B59}, {0x1B83, 0x1BA0}, {0x1BAE, 0x1BE5}, {0x1C00, 0x1C23}, - {0x1C40, 0x1C49}, {0x1C4D, 0x1C7D}, {0x1C80, 0x1C88}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, + {0x1C40, 0x1C49}, {0x1C4D, 0x1C7D}, {0x1C80, 0x1C8A}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, {0x1CE9, 0x1CEC}, {0x1CEE, 0x1CF3}, {0x1CF5, 0x1CF6}, {0x1CFA, 0x1CFA}, {0x1D00, 0x1DBF}, {0x1E00, 0x1F15}, {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, @@ -105,87 +105,93 @@ inline constexpr std::array token_char_ranges{{ {0x31A0, 0x31BF}, {0x31F0, 0x31FF}, {0x3220, 0x3229}, {0x3248, 0x324F}, {0x3251, 0x325F}, {0x3280, 0x3289}, {0x32B1, 0x32BF}, {0x3400, 0x4DBF}, {0x4E00, 0xA48C}, {0xA4D0, 0xA4FD}, {0xA500, 0xA60C}, {0xA610, 0xA62B}, {0xA640, 0xA66E}, {0xA67F, 0xA69D}, {0xA6A0, 0xA6EF}, - {0xA717, 0xA71F}, {0xA722, 0xA788}, {0xA78B, 0xA7CA}, {0xA7D0, 0xA7D1}, {0xA7D3, 0xA7D3}, - {0xA7D5, 0xA7D9}, {0xA7F2, 0xA801}, {0xA803, 0xA805}, {0xA807, 0xA80A}, {0xA80C, 0xA822}, - {0xA830, 0xA835}, {0xA840, 0xA873}, {0xA882, 0xA8B3}, {0xA8D0, 0xA8D9}, {0xA8F2, 0xA8F7}, - {0xA8FB, 0xA8FB}, {0xA8FD, 0xA8FE}, {0xA900, 0xA925}, {0xA930, 0xA946}, {0xA960, 0xA97C}, - {0xA984, 0xA9B2}, {0xA9CF, 0xA9D9}, {0xA9E0, 0xA9E4}, {0xA9E6, 0xA9FE}, {0xAA00, 0xAA28}, - {0xAA40, 0xAA42}, {0xAA44, 0xAA4B}, {0xAA50, 0xAA59}, {0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, - {0xAA7E, 0xAAAF}, {0xAAB1, 0xAAB1}, {0xAAB5, 0xAAB6}, {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, - {0xAAC2, 0xAAC2}, {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, {0xAAF2, 0xAAF4}, {0xAB01, 0xAB06}, - {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, {0xAB30, 0xAB5A}, - {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, {0xABF0, 0xABF9}, {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, - {0xD7CB, 0xD7FB}, {0xF900, 0xFA6D}, {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, - {0xFB1D, 0xFB1D}, {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, - {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFBB1}, {0xFBD3, 0xFD3D}, {0xFD50, 0xFD8F}, - {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, {0xFF10, 0xFF19}, - {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, {0xFF66, 0xFFBE}, {0xFFC2, 0xFFC7}, {0xFFCA, 0xFFCF}, - {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, - {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, {0x10107, 0x10133}, - {0x10140, 0x10178}, {0x1018A, 0x1018B}, {0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x102E1, 0x102FB}, - {0x10300, 0x10323}, {0x1032D, 0x1034A}, {0x10350, 0x10375}, {0x10380, 0x1039D}, {0x103A0, 0x103C3}, - {0x103C8, 0x103CF}, {0x103D1, 0x103D5}, {0x10400, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, - {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, {0x10570, 0x1057A}, {0x1057C, 0x1058A}, - {0x1058C, 0x10592}, {0x10594, 0x10595}, {0x10597, 0x105A1}, {0x105A3, 0x105B1}, {0x105B3, 0x105B9}, - {0x105BB, 0x105BC}, {0x10600, 0x10736}, {0x10740, 0x10755}, {0x10760, 0x10767}, {0x10780, 0x10785}, - {0x10787, 0x107B0}, {0x107B2, 0x107BA}, {0x10800, 0x10805}, {0x10808, 0x10808}, {0x1080A, 0x10835}, - {0x10837, 0x10838}, {0x1083C, 0x1083C}, {0x1083F, 0x10855}, {0x10858, 0x10876}, {0x10879, 0x1089E}, - {0x108A7, 0x108AF}, {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, {0x10920, 0x10939}, - {0x10980, 0x109B7}, {0x109BC, 0x109CF}, {0x109D2, 0x10A00}, {0x10A10, 0x10A13}, {0x10A15, 0x10A17}, - {0x10A19, 0x10A35}, {0x10A40, 0x10A48}, {0x10A60, 0x10A7E}, {0x10A80, 0x10A9F}, {0x10AC0, 0x10AC7}, - {0x10AC9, 0x10AE4}, {0x10AEB, 0x10AEF}, {0x10B00, 0x10B35}, {0x10B40, 0x10B55}, {0x10B58, 0x10B72}, - {0x10B78, 0x10B91}, {0x10BA9, 0x10BAF}, {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, - {0x10CFA, 0x10D23}, {0x10D30, 0x10D39}, {0x10E60, 0x10E7E}, {0x10E80, 0x10EA9}, {0x10EB0, 0x10EB1}, - {0x10F00, 0x10F27}, {0x10F30, 0x10F45}, {0x10F51, 0x10F54}, {0x10F70, 0x10F81}, {0x10FB0, 0x10FCB}, - {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11052, 0x1106F}, {0x11071, 0x11072}, {0x11075, 0x11075}, - {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x110F0, 0x110F9}, {0x11103, 0x11126}, {0x11136, 0x1113F}, - {0x11144, 0x11144}, {0x11147, 0x11147}, {0x11150, 0x11172}, {0x11176, 0x11176}, {0x11183, 0x111B2}, - {0x111C1, 0x111C4}, {0x111D0, 0x111DA}, {0x111DC, 0x111DC}, {0x111E1, 0x111F4}, {0x11200, 0x11211}, - {0x11213, 0x1122B}, {0x1123F, 0x11240}, {0x11280, 0x11286}, {0x11288, 0x11288}, {0x1128A, 0x1128D}, - {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, {0x112F0, 0x112F9}, {0x11305, 0x1130C}, - {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333}, {0x11335, 0x11339}, - {0x1133D, 0x1133D}, {0x11350, 0x11350}, {0x1135D, 0x11361}, {0x11400, 0x11434}, {0x11447, 0x1144A}, - {0x11450, 0x11459}, {0x1145F, 0x11461}, {0x11480, 0x114AF}, {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, - {0x114D0, 0x114D9}, {0x11580, 0x115AE}, {0x115D8, 0x115DB}, {0x11600, 0x1162F}, {0x11644, 0x11644}, - {0x11650, 0x11659}, {0x11680, 0x116AA}, {0x116B8, 0x116B8}, {0x116C0, 0x116C9}, {0x11700, 0x1171A}, + {0xA717, 0xA71F}, {0xA722, 0xA788}, {0xA78B, 0xA7DC}, {0xA7F1, 0xA801}, {0xA803, 0xA805}, + {0xA807, 0xA80A}, {0xA80C, 0xA822}, {0xA830, 0xA835}, {0xA840, 0xA873}, {0xA882, 0xA8B3}, + {0xA8D0, 0xA8D9}, {0xA8F2, 0xA8F7}, {0xA8FB, 0xA8FB}, {0xA8FD, 0xA8FE}, {0xA900, 0xA925}, + {0xA930, 0xA946}, {0xA960, 0xA97C}, {0xA984, 0xA9B2}, {0xA9CF, 0xA9D9}, {0xA9E0, 0xA9E4}, + {0xA9E6, 0xA9FE}, {0xAA00, 0xAA28}, {0xAA40, 0xAA42}, {0xAA44, 0xAA4B}, {0xAA50, 0xAA59}, + {0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, {0xAA7E, 0xAAAF}, {0xAAB1, 0xAAB1}, {0xAAB5, 0xAAB6}, + {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, {0xAAC2, 0xAAC2}, {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, + {0xAAF2, 0xAAF4}, {0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, + {0xAB28, 0xAB2E}, {0xAB30, 0xAB5A}, {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, {0xABF0, 0xABF9}, + {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xF900, 0xFA6D}, {0xFA70, 0xFAD9}, + {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB1D}, {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, + {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFBB1}, + {0xFBD3, 0xFD3D}, {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, + {0xFE76, 0xFEFC}, {0xFF10, 0xFF19}, {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, {0xFF66, 0xFFBE}, + {0xFFC2, 0xFFC7}, {0xFFCA, 0xFFCF}, {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, {0x10000, 0x1000B}, + {0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, + {0x10080, 0x100FA}, {0x10107, 0x10133}, {0x10140, 0x10178}, {0x1018A, 0x1018B}, {0x10280, 0x1029C}, + {0x102A0, 0x102D0}, {0x102E1, 0x102FB}, {0x10300, 0x10323}, {0x1032D, 0x1034A}, {0x10350, 0x10375}, + {0x10380, 0x1039D}, {0x103A0, 0x103C3}, {0x103C8, 0x103CF}, {0x103D1, 0x103D5}, {0x10400, 0x1049D}, + {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, + {0x10570, 0x1057A}, {0x1057C, 0x1058A}, {0x1058C, 0x10592}, {0x10594, 0x10595}, {0x10597, 0x105A1}, + {0x105A3, 0x105B1}, {0x105B3, 0x105B9}, {0x105BB, 0x105BC}, {0x105C0, 0x105F3}, {0x10600, 0x10736}, + {0x10740, 0x10755}, {0x10760, 0x10767}, {0x10780, 0x10785}, {0x10787, 0x107B0}, {0x107B2, 0x107BA}, + {0x10800, 0x10805}, {0x10808, 0x10808}, {0x1080A, 0x10835}, {0x10837, 0x10838}, {0x1083C, 0x1083C}, + {0x1083F, 0x10855}, {0x10858, 0x10876}, {0x10879, 0x1089E}, {0x108A7, 0x108AF}, {0x108E0, 0x108F2}, + {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, {0x10920, 0x10939}, {0x10940, 0x10959}, {0x10980, 0x109B7}, + {0x109BC, 0x109CF}, {0x109D2, 0x10A00}, {0x10A10, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A35}, + {0x10A40, 0x10A48}, {0x10A60, 0x10A7E}, {0x10A80, 0x10A9F}, {0x10AC0, 0x10AC7}, {0x10AC9, 0x10AE4}, + {0x10AEB, 0x10AEF}, {0x10B00, 0x10B35}, {0x10B40, 0x10B55}, {0x10B58, 0x10B72}, {0x10B78, 0x10B91}, + {0x10BA9, 0x10BAF}, {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, {0x10CFA, 0x10D23}, + {0x10D30, 0x10D39}, {0x10D40, 0x10D65}, {0x10D6F, 0x10D85}, {0x10E60, 0x10E7E}, {0x10E80, 0x10EA9}, + {0x10EB0, 0x10EB1}, {0x10EC2, 0x10EC7}, {0x10F00, 0x10F27}, {0x10F30, 0x10F45}, {0x10F51, 0x10F54}, + {0x10F70, 0x10F81}, {0x10FB0, 0x10FCB}, {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11052, 0x1106F}, + {0x11071, 0x11072}, {0x11075, 0x11075}, {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x110F0, 0x110F9}, + {0x11103, 0x11126}, {0x11136, 0x1113F}, {0x11144, 0x11144}, {0x11147, 0x11147}, {0x11150, 0x11172}, + {0x11176, 0x11176}, {0x11183, 0x111B2}, {0x111C1, 0x111C4}, {0x111D0, 0x111DA}, {0x111DC, 0x111DC}, + {0x111E1, 0x111F4}, {0x11200, 0x11211}, {0x11213, 0x1122B}, {0x1123F, 0x11240}, {0x11280, 0x11286}, + {0x11288, 0x11288}, {0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, + {0x112F0, 0x112F9}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330}, + {0x11332, 0x11333}, {0x11335, 0x11339}, {0x1133D, 0x1133D}, {0x11350, 0x11350}, {0x1135D, 0x11361}, + {0x11380, 0x11389}, {0x1138B, 0x1138B}, {0x1138E, 0x1138E}, {0x11390, 0x113B5}, {0x113B7, 0x113B7}, + {0x113D1, 0x113D1}, {0x113D3, 0x113D3}, {0x11400, 0x11434}, {0x11447, 0x1144A}, {0x11450, 0x11459}, + {0x1145F, 0x11461}, {0x11480, 0x114AF}, {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, {0x114D0, 0x114D9}, + {0x11580, 0x115AE}, {0x115D8, 0x115DB}, {0x11600, 0x1162F}, {0x11644, 0x11644}, {0x11650, 0x11659}, + {0x11680, 0x116AA}, {0x116B8, 0x116B8}, {0x116C0, 0x116C9}, {0x116D0, 0x116E3}, {0x11700, 0x1171A}, {0x11730, 0x1173B}, {0x11740, 0x11746}, {0x11800, 0x1182B}, {0x118A0, 0x118F2}, {0x118FF, 0x11906}, {0x11909, 0x11909}, {0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x1192F}, {0x1193F, 0x1193F}, {0x11941, 0x11941}, {0x11950, 0x11959}, {0x119A0, 0x119A7}, {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, {0x119E3, 0x119E3}, {0x11A00, 0x11A00}, {0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, - {0x11A5C, 0x11A89}, {0x11A9D, 0x11A9D}, {0x11AB0, 0x11AF8}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, - {0x11C40, 0x11C40}, {0x11C50, 0x11C6C}, {0x11C72, 0x11C8F}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, - {0x11D0B, 0x11D30}, {0x11D46, 0x11D46}, {0x11D50, 0x11D59}, {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, - {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11DA0, 0x11DA9}, {0x11EE0, 0x11EF2}, {0x11F02, 0x11F02}, - {0x11F04, 0x11F10}, {0x11F12, 0x11F33}, {0x11F50, 0x11F59}, {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FD4}, - {0x12000, 0x12399}, {0x12400, 0x1246E}, {0x12480, 0x12543}, {0x12F90, 0x12FF0}, {0x13000, 0x1342F}, - {0x13441, 0x13446}, {0x14400, 0x14646}, {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, - {0x16A70, 0x16ABE}, {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, - {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16E40, 0x16E96}, - {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, - {0x17000, 0x187F7}, {0x18800, 0x18CD5}, {0x18D00, 0x18D08}, {0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, - {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122}, {0x1B132, 0x1B132}, {0x1B150, 0x1B152}, {0x1B155, 0x1B155}, - {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, - {0x1BC90, 0x1BC99}, {0x1D2C0, 0x1D2D3}, {0x1D2E0, 0x1D2F3}, {0x1D360, 0x1D378}, {0x1D400, 0x1D454}, - {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, - {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, - {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, - {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA}, - {0x1D6DC, 0x1D6FA}, {0x1D6FC, 0x1D714}, {0x1D716, 0x1D734}, {0x1D736, 0x1D74E}, {0x1D750, 0x1D76E}, - {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, {0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB}, {0x1D7CE, 0x1D7FF}, - {0x1DF00, 0x1DF1E}, {0x1DF25, 0x1DF2A}, {0x1E030, 0x1E06D}, {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, - {0x1E140, 0x1E149}, {0x1E14E, 0x1E14E}, {0x1E290, 0x1E2AD}, {0x1E2C0, 0x1E2EB}, {0x1E2F0, 0x1E2F9}, - {0x1E4D0, 0x1E4EB}, {0x1E4F0, 0x1E4F9}, {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE}, - {0x1E7F0, 0x1E7FE}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8CF}, {0x1E900, 0x1E943}, {0x1E94B, 0x1E94B}, - {0x1E950, 0x1E959}, {0x1EC71, 0x1ECAB}, {0x1ECAD, 0x1ECAF}, {0x1ECB1, 0x1ECB4}, {0x1ED01, 0x1ED2D}, - {0x1ED2F, 0x1ED3D}, {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, - {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, - {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, - {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, - {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, - {0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, - {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, {0x1F100, 0x1F10C}, - {0x1FBF0, 0x1FBF9}, {0x20000, 0x2A6DF}, {0x2A700, 0x2B739}, {0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, - {0x2CEB0, 0x2EBE0}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, {0x31350, 0x323AF}, + {0x11A5C, 0x11A89}, {0x11A9D, 0x11A9D}, {0x11AB0, 0x11AF8}, {0x11BC0, 0x11BE0}, {0x11BF0, 0x11BF9}, + {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, {0x11C40, 0x11C40}, {0x11C50, 0x11C6C}, {0x11C72, 0x11C8F}, + {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, {0x11D0B, 0x11D30}, {0x11D46, 0x11D46}, {0x11D50, 0x11D59}, + {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11DA0, 0x11DA9}, + {0x11DB0, 0x11DDB}, {0x11DE0, 0x11DE9}, {0x11EE0, 0x11EF2}, {0x11F02, 0x11F02}, {0x11F04, 0x11F10}, + {0x11F12, 0x11F33}, {0x11F50, 0x11F59}, {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FD4}, {0x12000, 0x12399}, + {0x12400, 0x1246E}, {0x12480, 0x12543}, {0x12F90, 0x12FF0}, {0x13000, 0x1342F}, {0x13441, 0x13446}, + {0x13460, 0x143FA}, {0x14400, 0x14646}, {0x16100, 0x1611D}, {0x16130, 0x16139}, {0x16800, 0x16A38}, + {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, {0x16A70, 0x16ABE}, {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, + {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, {0x16B63, 0x16B77}, + {0x16B7D, 0x16B8F}, {0x16D40, 0x16D6C}, {0x16D70, 0x16D79}, {0x16E40, 0x16E96}, {0x16EA0, 0x16EB8}, + {0x16EBB, 0x16ED3}, {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, + {0x16FE3, 0x16FE3}, {0x16FF2, 0x16FF6}, {0x17000, 0x18CD5}, {0x18CFF, 0x18D1E}, {0x18D80, 0x18DF2}, + {0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122}, {0x1B132, 0x1B132}, + {0x1B150, 0x1B152}, {0x1B155, 0x1B155}, {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, + {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1CCF0, 0x1CCF9}, {0x1D2C0, 0x1D2D3}, + {0x1D2E0, 0x1D2F3}, {0x1D360, 0x1D378}, {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, + {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, + {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, + {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, + {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA}, {0x1D6DC, 0x1D6FA}, {0x1D6FC, 0x1D714}, + {0x1D716, 0x1D734}, {0x1D736, 0x1D74E}, {0x1D750, 0x1D76E}, {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, + {0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB}, {0x1D7CE, 0x1D7FF}, {0x1DF00, 0x1DF1E}, {0x1DF25, 0x1DF2A}, + {0x1E030, 0x1E06D}, {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, {0x1E140, 0x1E149}, {0x1E14E, 0x1E14E}, + {0x1E290, 0x1E2AD}, {0x1E2C0, 0x1E2EB}, {0x1E2F0, 0x1E2F9}, {0x1E4D0, 0x1E4EB}, {0x1E4F0, 0x1E4F9}, + {0x1E5D0, 0x1E5ED}, {0x1E5F0, 0x1E5FA}, {0x1E6C0, 0x1E6DE}, {0x1E6E0, 0x1E6E2}, {0x1E6E4, 0x1E6E5}, + {0x1E6E7, 0x1E6ED}, {0x1E6F0, 0x1E6F4}, {0x1E6FE, 0x1E6FF}, {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, + {0x1E7ED, 0x1E7EE}, {0x1E7F0, 0x1E7FE}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8CF}, {0x1E900, 0x1E943}, + {0x1E94B, 0x1E94B}, {0x1E950, 0x1E959}, {0x1EC71, 0x1ECAB}, {0x1ECAD, 0x1ECAF}, {0x1ECB1, 0x1ECB4}, + {0x1ED01, 0x1ED2D}, {0x1ED2F, 0x1ED3D}, {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, + {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, + {0x1EE3B, 0x1EE3B}, {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, + {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, + {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, + {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, + {0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, + {0x1F100, 0x1F10C}, {0x1FBF0, 0x1FBF9}, {0x20000, 0x2A6DF}, {0x2A700, 0x2B81D}, {0x2B820, 0x2CEAD}, + {0x2CEB0, 0x2EBE0}, {0x2EBF0, 0x2EE5D}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, {0x31350, 0x33479}, }}; inline bool isTokenChar(std::uint32_t code_point) diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index 506a4a57527..acbd1c04847 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -1863,7 +1863,8 @@ try "foo𝟙bar", "foo\U0002EBF0bar", "foo\xff" - "bar"}); + "bar", + "foo\u088Fbar"}); for (const auto size : {0U, 2U, 3U}) { tipb::LocalMatchAgainstBooleanQuery query; @@ -1875,13 +1876,13 @@ try node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); node->set_text(size == 0 ? "foo" : "foob"); ASSERT_COLUMN_EQ( - size == 0 ? createColumn({1, 1, 1, 0, 1, 1, 1, 1}) - : createColumn({0, 0, 0, 1, 0, 0, 0, 0}), + size == 0 ? createColumn({1, 1, 1, 0, 1, 1, 1, 1, 0}) + : createColumn({0, 0, 0, 1, 0, 0, 0, 0, 0}), executeFunction( "local_match_against_boolean", - {createConstColumn(8, node->text()), + {createConstColumn(documents.column->size(), node->text()), documents, - createConstColumn(8, serializeLocalMatchAgainstBooleanQuery(query))}, + createConstColumn(documents.column->size(), serializeLocalMatchAgainstBooleanQuery(query))}, collator, true)); // TiDB's version-2 NGRAM Boolean lexer emits "bar" for this search. @@ -1899,11 +1900,11 @@ try } CATCH -TEST(LocalMatchAgainstTokenChars, ProtocolV1Classification) +TEST(LocalMatchAgainstTokenChars, Unicode17Classification) { - for (const UInt32 code_point : {0x1F643U, 0x1F441U, 0x301U, 0xFFFDU, 0x2EBF0U, 0x110000U}) + for (const UInt32 code_point : {0x1F643U, 0x1F441U, 0x301U, 0xFFFDU, 0x110000U}) EXPECT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; - for (const UInt32 code_point : {0x5FU, 0x4E2DU, 0xE9U, 0x1D7D9U, 0x1E900U}) + for (const UInt32 code_point : {0x5FU, 0x88FU, 0x4E2DU, 0xE9U, 0x1D7D9U, 0x1E900U, 0x2EBF0U}) EXPECT_TRUE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; UInt32 previous = 127; for (const auto & span : LocalMatchAgainst::token_char_ranges) @@ -1920,16 +1921,15 @@ TEST(LocalMatchAgainstTokenChars, ProtocolV1Classification) ASSERT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; // FNV-1a over one 0/1 byte per code point, U+0000..U+10FFFF. The paired - // TiDB tests hash Go's Unicode 15.0.0 IsLetter/IsNumber/'_' classification - // identically. This fixed regression checksum must not change in v1, - // even when the header is regenerated or either toolchain is upgraded. + // TiDB tests hash Go's Unicode 17.0.0 IsLetter/IsNumber/'_' classification + // identically. The fingerprint catches changes to the shared classification. std::uint64_t fingerprint = 14695981039346656037ULL; for (UInt32 code_point = 0; code_point <= 0x10FFFFU; ++code_point) { fingerprint ^= static_cast(LocalMatchAgainst::isTokenChar(code_point)); fingerprint *= 1099511628211ULL; } - EXPECT_EQ(0x71f51f3810b3b529ULL, fingerprint); + EXPECT_EQ(0x0b93995fa724afa0ULL, fingerprint); } namespace From 702ae7030ae3fc9f6169dd254f7ff6a463cc57b4 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 11 Nov 2026 14:12:41 +0800 Subject: [PATCH 6/9] Functions: prune long Local MATCH stopword candidates Skip collation key generation for STANDARD tokens longer than any built-in stopword. Use byte lengths for binary collations and code-point counts for General CI. Preserve weight-based matching for Unicode CI and cover multibyte and ignorable-weight cases. Signed-off-by: Zhigao Tong --- .../Functions/FunctionsLocalMatchAgainst.cpp | 50 ++++++++++++++++--- .../tests/gtest_local_match_against.cpp | 5 +- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index 3dc0d736b1e..2e2506d18bc 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -398,6 +398,7 @@ class BuiltinStopwordLookup public: explicit BuiltinStopwordLookup(TiDB::TiDBCollatorPtr collator_) : collator(collator_) + , standard_length_limit(standardLengthLimit(collator_->getCollatorType())) { for (const auto stopword : builtin_stopwords) { @@ -416,9 +417,13 @@ class BuiltinStopwordLookup } } - bool contains(std::string_view token) const { return matchingLengths(token) != 0; } + bool containsStandard(std::string_view token, size_t code_points) const + { + const size_t length = standard_length_limit.count_code_points ? code_points : token.size(); + return length <= standard_length_limit.maximum && matchingLengths(token) != 0; + } - bool contains(std::string_view token, size_t code_points) const + bool containsNgram(std::string_view token, size_t code_points) const { // NGRAM also requires the original code-point count, even when a // collation makes words of different lengths share a key. @@ -426,6 +431,35 @@ class BuiltinStopwordLookup } private: + struct StandardLengthLimit + { + size_t maximum; + bool count_code_points; + }; + + static StandardLengthLimit standardLengthLimit(TiDB::ITiDBCollator::CollatorType type) + { + using Type = TiDB::ITiDBCollator::CollatorType; + switch (type) + { + case Type::UTF8MB4_BIN: + case Type::UTF8_BIN: + case Type::LATIN1_BIN: + case Type::ASCII_BIN: + case Type::BINARY: + case Type::UTF8MB4_0900_BIN: + // STANDARD tokens have no trailing spaces, so PAD SPACE cannot shorten them. + return {max_builtin_stopword_length, false}; + case Type::UTF8_GENERAL_CI: + case Type::UTF8MB4_GENERAL_CI: + // General CI emits exactly one weight per code point. + return {max_builtin_stopword_length, true}; + default: + // Collations with ignorable weights can match longer source tokens. + return {std::numeric_limits::max(), false}; + } + } + static UInt8 lengthBit(size_t length) { return static_cast(1U << length); } struct CacheEntry @@ -475,6 +509,7 @@ class BuiltinStopwordLookup } TiDB::TiDBCollatorPtr collator; + const StandardLengthLimit standard_length_limit; Arena key_storage; HashMap> keys; size_t max_key_size = 0; @@ -489,9 +524,10 @@ class BuiltinStopwordLookup mutable size_t cache_token_bytes = sizeof(UInt64); }; -bool isDefaultStopword(std::string_view token, const BuiltinStopwordLookup * stopwords) +bool isDefaultStopword(std::string_view token, size_t code_points, const BuiltinStopwordLookup * stopwords) { - return stopwords ? stopwords->contains(token) : defaultStopwords().contains(Poco::UTF8::toLower(String(token))); + return stopwords ? stopwords->containsStandard(token, code_points) + : defaultStopwords().contains(Poco::UTF8::toLower(String(token))); } bool containsNgramStopword( @@ -509,7 +545,7 @@ bool containsNgramStopword( const auto candidate = token.substr(char_boundaries[start], char_boundaries[end] - char_boundaries[start]); if (!stopwords && defaultStopwords().contains(Poco::UTF8::toLower(String(candidate)))) return true; - if (stopwords && stopwords->contains(candidate, end - start)) + if (stopwords && stopwords->containsNgram(candidate, end - start)) return true; } } @@ -531,7 +567,7 @@ void analyzeTextInto( // Filter before materializing. Stopwords use the source spelling and // server collation, independently of the MATCH column collation. if (code_points < min_token_size || code_points > max_token_size - || (enable_stopword && isDefaultStopword(token, stopwords))) + || (enable_stopword && isDefaultStopword(token, code_points, stopwords))) return; result.append(token, position, code_points, borrow_document && collator); if (!collator) @@ -669,7 +705,7 @@ void analyzeNgramTextInto( char_boundaries[candidate_start], char_boundaries[stopword_end] - char_boundaries[candidate_start]); const bool matched = stopwords - ? stopwords->contains(candidate, length) + ? stopwords->containsNgram(candidate, length) : defaultStopwords().contains(Poco::UTF8::toLower(String(candidate))); if (matched) { diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index acbd1c04847..12a9a016d2f 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -1467,8 +1467,9 @@ try for (const auto word : stopwords) candidates.emplace_back(word); for (const String & word : - {"ABOUT", "The", "THE", "WWW", "WITH", "ábout", "thé", "tĥe", "with", "www", "aß", "ß", - "xá", "áx", "xA", "Ax", "xáy", "aaa", "AAA", "xy", "xyz", "foo", "数据库"}) + {"ABOUT", "The", "THE", "WWW", "WITH", "ábout", "thé", "tĥe", "with", + "www", "aß", "ß", "xá", "áx", "xA", "Ax", "xáy", "aaa", + "AAA", "xy", "xyz", "foo", "数据库", "aboutx", "about", "ab\u0640out", "ab\uFC5Eout"}) candidates.push_back(word); for (const auto collation : {"binary", From aa5dd2d0e21223754553686ae55627fa7a71a99e Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 12 Nov 2026 11:57:54 +0800 Subject: [PATCH 7/9] Functions: lower Local MATCH token threshold for large blocks Use a 16-token threshold for blocks with at least 256 rows and verify the automatic route at the row, token, and query-term boundaries. Signed-off-by: Zhigao Tong --- .../Functions/FunctionsLocalMatchAgainst.cpp | 10 ++++-- .../tests/gtest_local_match_against.cpp | 34 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index 2e2506d18bc..ebcbe7a2a22 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -166,6 +166,8 @@ struct AnalyzerScratch // Reserve automatic routing for large queries; forced routing stays test-only. constexpr size_t exact_route_min_terms = 100; constexpr size_t exact_route_min_tokens = 32; +constexpr size_t exact_route_min_tokens_large_block = 16; +constexpr size_t exact_route_large_block_min_rows = 256; constexpr size_t exact_route_budget = 4 * 1024 * 1024; constexpr size_t exact_route_max_token_bytes = 4096; @@ -1583,6 +1585,8 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction for (size_t arg = 1; arg < document_argument_end; ++arg) document_columns.emplace_back(*block.getByPosition(arguments[arg]).column); const size_t route_terms = eligibleExactRouteTermCount(compiled_query, config, collator, matcher, budget); + const size_t min_route_tokens + = rows >= exact_route_large_block_min_rows ? exact_route_min_tokens_large_block : exact_route_min_tokens; bool may_use_index = matcher == LocalMatchAgainstTest::Matcher::Indexed; if (route_terms && !may_use_index) { @@ -1590,7 +1594,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction // tokens need a separator. This cheap lower bound bypasses the // routing row loop for wholly short blocks without tokenizing twice. // Use one byte even for unusual analyzer bounds to avoid overflow. - constexpr size_t min_document_bytes = exact_route_min_tokens * 2 - 1; + const size_t min_document_bytes = min_route_tokens * 2 - 1; for (const auto & column : document_columns) { for (size_t row = 0; row < rows; ++row) @@ -1658,7 +1662,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction // columns may already have contributed to this row's state. if (!route_attempted && (matcher == LocalMatchAgainstTest::Matcher::Indexed - || analyzed_column.size() >= exact_route_min_tokens)) + || analyzed_column.size() >= min_route_tokens)) { route_attempted = true; compiled_query.exact_route = buildExactTermRoute(compiled_query, collator, route_terms); @@ -1703,7 +1707,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction // this whole column before any state is modified. const bool indexed = exact_scratch && (matcher == LocalMatchAgainstTest::Matcher::Indexed - || analyzed_column.size() >= exact_route_min_tokens) + || analyzed_column.size() >= min_route_tokens) // Valid BMP tokens use at most three bytes per code // point. The usual analyzer bound proves this without // another column pass, including early-hit OR queries. diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index 12a9a016d2f..714d1a87c41 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -2185,6 +2185,40 @@ TEST_F(TestLocalMatchAgainst, ExactRouteEquivalentProhibitedAndColumnFallback) EXPECT_EQ(2, fallback.linear_columns); } +TEST_F(TestLocalMatchAgainst, ExactRouteTokenThresholdForLargeBlocks) +{ + using namespace LocalMatchAgainstTest; + const auto * const collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + for (size_t rows : {255, 256}) + { + for (size_t tokens : {15, 16, 31, 32}) + { + String document = "a"; + for (size_t i = 1; i < tokens; ++i) + document += " a"; + for (size_t terms : {99, 100}) + { + SCOPED_TRACE(::testing::Message() << "rows=" << rows << " tokens=" << tokens << " terms=" << terms); + auto query = exactQuery(terms, true); + query.set_innodb_ft_min_token_size(1); + query.set_innodb_ft_max_token_size(84); + const ColumnsWithTypeAndName columns{ + createConstColumn(rows, "query"), + createColumn(std::vector(rows, document)), + createConstColumn(rows, serializeLocalMatchAgainstBooleanQuery(query))}; + Work linear, automatic; + ASSERT_COLUMN_EQ( + runMatcher(columns, collator, Matcher::Linear, linear), + runMatcher(columns, collator, Matcher::Automatic, automatic)); + const bool indexed = terms == 100 && (tokens == 32 || (rows == 256 && tokens >= 16)); + EXPECT_EQ(indexed ? 1 : 0, automatic.route_builds); + EXPECT_EQ(indexed ? rows : 0, automatic.indexed_columns); + EXPECT_EQ(indexed ? 0 : rows, automatic.linear_columns); + } + } + } +} + TEST_F(TestLocalMatchAgainst, ExactRouteThresholdNullableAndReusedBlocks) { using namespace LocalMatchAgainstTest; From 069b9029654410c1f549b3e0de0f7c69c9633229 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 12 Nov 2026 13:13:53 +0800 Subject: [PATCH 8/9] Functions: enable short-document Local MATCH routing from 128 rows Signed-off-by: Zhigao Tong --- dbms/src/Functions/FunctionsLocalMatchAgainst.cpp | 2 +- dbms/src/Functions/tests/gtest_local_match_against.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index ebcbe7a2a22..a5ac5df283b 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -167,7 +167,7 @@ struct AnalyzerScratch constexpr size_t exact_route_min_terms = 100; constexpr size_t exact_route_min_tokens = 32; constexpr size_t exact_route_min_tokens_large_block = 16; -constexpr size_t exact_route_large_block_min_rows = 256; +constexpr size_t exact_route_large_block_min_rows = 128; constexpr size_t exact_route_budget = 4 * 1024 * 1024; constexpr size_t exact_route_max_token_bytes = 4096; diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index 714d1a87c41..253ce6c129b 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -2189,7 +2189,7 @@ TEST_F(TestLocalMatchAgainst, ExactRouteTokenThresholdForLargeBlocks) { using namespace LocalMatchAgainstTest; const auto * const collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); - for (size_t rows : {255, 256}) + for (size_t rows : {127, 128}) { for (size_t tokens : {15, 16, 31, 32}) { @@ -2210,7 +2210,7 @@ TEST_F(TestLocalMatchAgainst, ExactRouteTokenThresholdForLargeBlocks) ASSERT_COLUMN_EQ( runMatcher(columns, collator, Matcher::Linear, linear), runMatcher(columns, collator, Matcher::Automatic, automatic)); - const bool indexed = terms == 100 && (tokens == 32 || (rows == 256 && tokens >= 16)); + const bool indexed = terms == 100 && (tokens == 32 || (rows == 128 && tokens >= 16)); EXPECT_EQ(indexed ? 1 : 0, automatic.route_builds); EXPECT_EQ(indexed ? rows : 0, automatic.indexed_columns); EXPECT_EQ(indexed ? 0 : rows, automatic.linear_columns); From f4020aed9e34c4749fe85cb8c6e283c504f633de Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 12 Nov 2026 15:50:15 +0800 Subject: [PATCH 9/9] Functions: lower Local MATCH term threshold for large blocks Signed-off-by: Zhigao Tong --- dbms/src/Functions/FunctionsLocalMatchAgainst.cpp | 12 ++++++++---- .../Functions/tests/gtest_local_match_against.cpp | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index a5ac5df283b..dd6a3d39178 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -165,6 +165,7 @@ struct AnalyzerScratch // costlier than a small linear scan, even above the document token threshold. // Reserve automatic routing for large queries; forced routing stays test-only. constexpr size_t exact_route_min_terms = 100; +constexpr size_t exact_route_min_terms_large_block = 64; constexpr size_t exact_route_min_tokens = 32; constexpr size_t exact_route_min_tokens_large_block = 16; constexpr size_t exact_route_large_block_min_rows = 128; @@ -1278,6 +1279,7 @@ size_t eligibleExactRouteTermCount( const LocalMatchAnalyzerConfig & config, const TiDB::TiDBCollatorPtr & collator, LocalMatchAgainstTest::Matcher matcher, + size_t min_terms, size_t budget) { if (matcher == LocalMatchAgainstTest::Matcher::Linear || config.use_ngram || !collator) @@ -1329,7 +1331,7 @@ size_t eligibleExactRouteTermCount( ++terms; } } - if (terms == 0 || (matcher == LocalMatchAgainstTest::Matcher::Automatic && terms < exact_route_min_terms)) + if (terms == 0 || (matcher == LocalMatchAgainstTest::Matcher::Automatic && terms < min_terms)) return {}; return terms; @@ -1584,9 +1586,11 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction document_columns.reserve(document_argument_end - 1); for (size_t arg = 1; arg < document_argument_end; ++arg) document_columns.emplace_back(*block.getByPosition(arguments[arg]).column); - const size_t route_terms = eligibleExactRouteTermCount(compiled_query, config, collator, matcher, budget); - const size_t min_route_tokens - = rows >= exact_route_large_block_min_rows ? exact_route_min_tokens_large_block : exact_route_min_tokens; + const bool large_block = rows >= exact_route_large_block_min_rows; + const size_t min_route_terms = large_block ? exact_route_min_terms_large_block : exact_route_min_terms; + const size_t route_terms + = eligibleExactRouteTermCount(compiled_query, config, collator, matcher, min_route_terms, budget); + const size_t min_route_tokens = large_block ? exact_route_min_tokens_large_block : exact_route_min_tokens; bool may_use_index = matcher == LocalMatchAgainstTest::Matcher::Indexed; if (route_terms && !may_use_index) { diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index 253ce6c129b..80e02ddbc16 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -2196,7 +2196,7 @@ TEST_F(TestLocalMatchAgainst, ExactRouteTokenThresholdForLargeBlocks) String document = "a"; for (size_t i = 1; i < tokens; ++i) document += " a"; - for (size_t terms : {99, 100}) + for (size_t terms : {63, 64, 99, 100}) { SCOPED_TRACE(::testing::Message() << "rows=" << rows << " tokens=" << tokens << " terms=" << terms); auto query = exactQuery(terms, true); @@ -2210,7 +2210,7 @@ TEST_F(TestLocalMatchAgainst, ExactRouteTokenThresholdForLargeBlocks) ASSERT_COLUMN_EQ( runMatcher(columns, collator, Matcher::Linear, linear), runMatcher(columns, collator, Matcher::Automatic, automatic)); - const bool indexed = terms == 100 && (tokens == 32 || (rows == 128 && tokens >= 16)); + const bool indexed = rows == 128 ? terms >= 64 && tokens >= 16 : terms >= 100 && tokens >= 32; EXPECT_EQ(indexed ? 1 : 0, automatic.route_builds); EXPECT_EQ(indexed ? rows : 0, automatic.indexed_columns); EXPECT_EQ(indexed ? 0 : rows, automatic.linear_columns);