diff --git a/contrib/tipb b/contrib/tipb index 0607513e7fa..348fa94e545 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit 0607513e7fa40f3b564f0b269da76c1c8dc90032 +Subproject commit 348fa94e545a21e2b1a12752a7d0d1bbccea2efe diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp index c001d495f13..543494a60a0 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp @@ -513,6 +513,26 @@ String DAGExpressionAnalyzerHelper::buildGroupingFunction( return result_name; } +String DAGExpressionAnalyzerHelper::buildLocalMatchAgainstBoolean( + DAGExpressionAnalyzer * analyzer, + const tipb::Expr & expr, + const ExpressionActionsPtr & actions) +{ + const String & func_name = getFunctionName(expr); + Names argument_names; + for (const auto & child : expr.children()) + { + String name = analyzer->getActions(child, actions); + argument_names.push_back(name); + } + // Expr.val carries Local MATCH metadata, not an SQL argument. Adapt it + // to the runtime function's required final constant argument. Keeping it + // in argument_names also distinguishes actions with different metadata. + const auto metadata_expr = constructStringLiteralTiExpr(expr.val()); + argument_names.push_back(analyzer->getActions(metadata_expr, actions)); + return analyzer->applyFunction(func_name, argument_names, actions, getCollatorFromExpr(expr)); +} + String DAGExpressionAnalyzerHelper::buildDefaultFunction( DAGExpressionAnalyzer * analyzer, const tipb::Expr & expr, @@ -575,6 +595,7 @@ DAGExpressionAnalyzerHelper::FunctionBuilderMap DAGExpressionAnalyzerHelper::fun {"regexp", DAGExpressionAnalyzerHelper::buildRegexpFunction}, {"replaceRegexpAll", DAGExpressionAnalyzerHelper::buildRegexpFunction}, {"tidbRound", DAGExpressionAnalyzerHelper::buildRoundFunction}, - {"grouping", DAGExpressionAnalyzerHelper::buildGroupingFunction}}); + {"grouping", DAGExpressionAnalyzerHelper::buildGroupingFunction}, + {"local_match_against_boolean", DAGExpressionAnalyzerHelper::buildLocalMatchAgainstBoolean}}); } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.h b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.h index 529dc204ace..3f59e972637 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.h +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.h @@ -97,6 +97,11 @@ class DAGExpressionAnalyzerHelper const tipb::Expr & expr, const ExpressionActionsPtr & actions); + static String buildLocalMatchAgainstBoolean( + DAGExpressionAnalyzer * analyzer, + const tipb::Expr & expr, + const ExpressionActionsPtr & actions); + static String buildDefaultFunction( DAGExpressionAnalyzer * analyzer, const tipb::Expr & expr, diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index d8d086d6af6..eb565a307c0 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -455,6 +455,7 @@ const std::unordered_map scalar_func_map({ {tipb::ScalarFuncSig::RegexpInStrSig, "regexp_instr"}, {tipb::ScalarFuncSig::RegexpReplaceSig, "regexp_replace"}, {tipb::ScalarFuncSig::RegexpSubstrSig, "regexp_substr"}, + {tipb::ScalarFuncSig::LocalMatchAgainstBoolean, "local_match_against_boolean"}, {tipb::ScalarFuncSig::JsonExtractSig, "json_extract"}, {tipb::ScalarFuncSig::JsonUnquoteSig, "json_unquote"}, diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp new file mode 100644 index 00000000000..dd6a3d39178 --- /dev/null +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -0,0 +1,1798 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_COLUMN; +} + +namespace +{ +constexpr size_t default_min_token_size = 3; +constexpr size_t default_max_token_size = 84; +constexpr size_t default_ngram_token_size = 2; +// Local MATCH protocol version. TiFlash rejects unknown versions. TiDB and +// TiFlash must use the same Unicode classification when evaluating a query. +// Version 2 returns zero for NULL searches and uses MySQL's multibyte NGRAM +// document scanner. STANDARD and the NGRAM Boolean lexer use BMP word +// characters; filtered phrases also verify the unfiltered document stream. +constexpr UInt32 local_match_against_protocol_version = 2; + +struct FullTextToken +{ + // Query terms and the no-collator lowercase path own their spelling. + // Document views refer only to the current input column, which outlives + // both analyzed streams and matching. Never retain them across blocks. + String text; + std::string_view source; + size_t position = 0; + size_t code_points = 0; + + std::string_view value() const { return source.data() ? source : std::string_view{text}; } +}; + +class FullTextColumn +{ +public: + using Container = std::vector; + using iterator = Container::iterator; + using const_iterator = Container::const_iterator; + + void clear() { used = 0; } + + void reserve(size_t size) { tokens.reserve(size); } + + // This is per-token work. Keep it inline even when the surrounding + // analyzer grows: outlining it penalizes the unfiltered/OFF stream too. + ALWAYS_INLINE void append(std::string_view text, size_t position, size_t code_points, bool borrow = false) + { + if (used == tokens.size()) + tokens.emplace_back(); + auto & token = tokens[used++]; + if (borrow) + token.source = text; + else + { + token.source = {}; + token.text.assign(text.data(), text.size()); + } + token.position = position; + token.code_points = code_points; + } + + template + void filterInPlace(Predicate predicate) + { + size_t write = 0; + for (size_t read = 0; read < used; ++read) + { + if (!predicate(tokens[read])) + continue; + if (write != read) + std::swap(tokens[write], tokens[read]); + ++write; + } + used = write; + } + + bool empty() const { return used == 0; } + size_t size() const { return used; } + FullTextToken & front() { return tokens.front(); } + const FullTextToken & front() const { return tokens.front(); } + FullTextToken & operator[](size_t index) { return tokens[index]; } + const FullTextToken & operator[](size_t index) const { return tokens[index]; } + iterator begin() { return tokens.begin(); } + iterator end() { return tokens.begin() + used; } + const_iterator begin() const { return tokens.begin(); } + const_iterator end() const { return tokens.begin() + used; } + +private: + Container tokens; + size_t used = 0; +}; + +struct BooleanClause +{ + enum class Modifier + { + Should, + Must, + MustNot, + }; + + Modifier modifier = Modifier::Should; + bool phrase = false; + bool prefix = false; + std::vector terms; + std::vector offsets; + std::unique_ptr verification; + std::vector> prefix_matchers; +}; + +struct AnalyzerScratch +{ + std::vector char_boundaries; + String lowercase_run; +}; + +// Short first-character misses and long UCA expansions can make key creation +// costlier than a small linear scan, even above the document token threshold. +// Reserve automatic routing for large queries; forced routing stays test-only. +constexpr size_t exact_route_min_terms = 100; +constexpr size_t exact_route_min_terms_large_block = 64; +constexpr size_t exact_route_min_tokens = 32; +constexpr size_t exact_route_min_tokens_large_block = 16; +constexpr size_t exact_route_large_block_min_rows = 128; +constexpr size_t exact_route_budget = 4 * 1024 * 1024; +constexpr size_t exact_route_max_token_bytes = 4096; + +// sortKey's reusable std::string is not allocated through DB::Allocator. +// Account its heap capacity separately; all persistent route storage below +// uses the existing tracked Arena/HashMap/PODArray allocators. +class MatchKeyBuffer +{ +public: + MatchKeyBuffer() = default; + MatchKeyBuffer(const MatchKeyBuffer &) = delete; + MatchKeyBuffer & operator=(const MatchKeyBuffer &) = delete; + ~MatchKeyBuffer() { CurrentMemoryTracker::free(charged); } + StringRef key(std::string_view text, const TiDB::TiDBCollatorPtr & collator) + { + const auto result = collator->sortKeyFastPath(text.data(), text.size(), buffer); + const size_t bytes = buffer.capacity() > String{}.capacity() ? buffer.capacity() + 1 : 0; + if (bytes > charged) + { + CurrentMemoryTracker::alloc(bytes - charged); + charged = bytes; + } + return result; + } + +private: + String buffer; + size_t charged = 0; +}; + +struct ExactTermRoute +{ + struct Reference + { + size_t bucket; + size_t clause; + size_t term; + }; + struct Bucket + { + size_t offset; + size_t count; + }; + // Keys own their bytes, including empty keys and embedded zero weights. + Arena keys; + HashMap dictionary; + PODArray buckets; + PODArray references; +}; + +struct ExactMatchScratch +{ + PODArray seen; + UInt32 generation; + MatchKeyBuffer keys; + + ExactMatchScratch(size_t buckets, UInt32 initial_generation) + : generation(initial_generation) + { + seen.resize_fill(buckets, 0); + } + void nextRow() + { + if (++generation == 0) + { + std::fill(seen.begin(), seen.end(), 0); + generation = 1; + } + } +}; + +struct CompiledBooleanQuery +{ + std::vector clauses; + size_t must_clause_count = 0; + bool has_must_not = false; + bool matches_nothing = false; + std::unique_ptr exact_route; +}; + +// Normalize the already validated wire configuration once per block. Keep +// the tokenizer implementations and their filtered/unfiltered rules intact. +struct LocalMatchAnalyzerConfig +{ + bool use_ngram; + size_t ngram_token_size; + size_t min_token_size; + size_t max_token_size; + bool enable_stopword; + + explicit LocalMatchAnalyzerConfig(const tipb::LocalMatchAgainstBooleanQuery & query) + : use_ngram(query.parser() == tipb::LocalMatchAgainstParserNgram) + , ngram_token_size( + !use_ngram || query.ngram_token_size() == 0 ? default_ngram_token_size : query.ngram_token_size()) + , min_token_size( + query.innodb_ft_min_token_size() != 0 || query.innodb_ft_max_token_size() != 0 + ? query.innodb_ft_min_token_size() + : default_min_token_size) + , max_token_size( + query.innodb_ft_min_token_size() != 0 || query.innodb_ft_max_token_size() != 0 + ? query.innodb_ft_max_token_size() + : default_max_token_size) + , enable_stopword(query.stopword_mode() == tipb::LocalMatchAgainstStopwordModeBuiltin) + {} +}; + +// Query tokens own their text. This cold-path helper only collects the same +// terms and source-relative offsets; document views never enter it. +size_t collectClauseTerms(BooleanClause & clause, const FullTextColumn & tokens) +{ + const size_t first_position = tokens.empty() ? 0 : tokens.front().position; + for (const auto & token : tokens) + { + clause.terms.push_back(token.text); + if (clause.phrase) + clause.offsets.push_back(token.position - first_position); + } + return first_position; +} + +void lowercaseTokenInPlace(String & token) +{ + const bool is_ascii = std::all_of(token.begin(), token.end(), [](unsigned char c) { return c < 0x80; }); + if (is_ascii) + { + for (char & c : token) + if (c >= 'A' && c <= 'Z') + c = static_cast(c + ('a' - 'A')); + return; + } + Poco::UTF8::toLowerInPlace(token); +} + +bool isFullTextToken(UInt32 code_point) +{ + return code_point <= 0xFFFF && LocalMatchAgainst::isTokenChar(code_point); +} + +std::pair decodeCodePoint(std::string_view text, size_t offset) +{ + // Most document bytes are ASCII. Keep the multibyte/invalid-byte + // semantics below, but avoid calling the generic decoder for ASCII. + const auto byte = static_cast(text[offset]); + if (byte < 0x80) + return {byte, 1}; + const auto decoded = UTF8::utf8Decode(text.data() + offset, text.size() - offset); + if (decoded.second == 0 || decoded.first == UTF8::UTF8_Error || decoded.second > text.size() - offset) + return {0xFFFD, 1}; // Go's utf8.DecodeRuneInString also consumes one invalid byte as RuneError. + return {decoded.first, decoded.second}; +} + +template +void scanStandardTokens(std::string_view text, Consumer && consume) +{ + size_t position = 0; + for (size_t i = 0; i < text.size();) + { + const auto [code_point, length] = decodeCodePoint(text, i); + if (!isFullTextToken(code_point)) + { + i += length; + continue; + } + + const size_t token_start = i; + // The first character has already been decoded and classified. + size_t code_points = 1; + i += length; + while (i < text.size()) + { + const auto [code_point, length] = decodeCodePoint(text, i); + if (!isFullTextToken(code_point)) + break; + ++code_points; + i += length; + } + + // Positions include source tokens subsequently removed by filtering. + consume(text.substr(token_start, i - token_start), position++, code_points); + } +} + +void tokenizeTextInto(std::string_view text, FullTextColumn & result, const TiDB::TiDBCollatorPtr & collator = nullptr) +{ + result.clear(); + scanStandardTokens(text, [&](std::string_view token, size_t position, size_t code_points) { + result.append(token, position, code_points); + if (!collator) + lowercaseTokenInPlace(result[result.size() - 1].text); + }); +} + +FullTextColumn tokenizeText(std::string_view text, const TiDB::TiDBCollatorPtr & collator = nullptr) +{ + FullTextColumn result; + tokenizeTextInto(text, result, collator); + return result; +} + +constexpr auto builtin_stopwords = std::to_array( + {"a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", + "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", + "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}); + +// The built-in list is ASCII, so byte and code-point lengths agree. +constexpr size_t max_builtin_stopword_length = [] { + size_t result = 0; + for (const auto word : builtin_stopwords) + result = std::max(result, word.size()); + return result; +}(); +static_assert(max_builtin_stopword_length < std::numeric_limits::digits); + +const std::unordered_set & defaultStopwords() +{ + static const auto stopwords = [] { + std::unordered_set result; + result.reserve(builtin_stopwords.size()); + for (const auto word : builtin_stopwords) + result.emplace(word); + return result; + }(); + return stopwords; +} + +// Collation keys own their bytes and are reused across the block. Use the +// stopword (server) collation independently of the MATCH column collation. +class BuiltinStopwordLookup +{ +public: + explicit BuiltinStopwordLookup(TiDB::TiDBCollatorPtr collator_) + : collator(collator_) + , standard_length_limit(standardLengthLimit(collator_->getCollatorType())) + { + for (const auto stopword : builtin_stopwords) + { + const auto key = collator->sortKeyFastPath(stopword.data(), stopword.size(), key_buffer); + max_key_size = std::max(max_key_size, key.size); + auto * entry = keys.find(key); + if (!entry) + { + // The sort key may reference the reusable buffer. + const StringRef owned(key_storage.insert(key.data, key.size), key.size); + bool inserted; + keys.emplace(owned, entry, inserted); + entry->getMapped() = 0; + } + entry->getMapped() |= lengthBit(stopword.size()); + } + } + + bool containsStandard(std::string_view token, size_t code_points) const + { + const size_t length = standard_length_limit.count_code_points ? code_points : token.size(); + return length <= standard_length_limit.maximum && matchingLengths(token) != 0; + } + + bool containsNgram(std::string_view token, size_t code_points) const + { + // NGRAM also requires the original code-point count, even when a + // collation makes words of different lengths share a key. + return code_points <= max_builtin_stopword_length && (matchingLengths(token) & lengthBit(code_points)) != 0; + } + +private: + struct StandardLengthLimit + { + size_t maximum; + bool count_code_points; + }; + + static StandardLengthLimit standardLengthLimit(TiDB::ITiDBCollator::CollatorType type) + { + using Type = TiDB::ITiDBCollator::CollatorType; + switch (type) + { + case Type::UTF8MB4_BIN: + case Type::UTF8_BIN: + case Type::LATIN1_BIN: + case Type::ASCII_BIN: + case Type::BINARY: + case Type::UTF8MB4_0900_BIN: + // STANDARD tokens have no trailing spaces, so PAD SPACE cannot shorten them. + return {max_builtin_stopword_length, false}; + case Type::UTF8_GENERAL_CI: + case Type::UTF8MB4_GENERAL_CI: + // General CI emits exactly one weight per code point. + return {max_builtin_stopword_length, true}; + default: + // Collations with ignorable weights can match longer source tokens. + return {std::numeric_limits::max(), false}; + } + } + + static UInt8 lengthBit(size_t length) { return static_cast(1U << length); } + + struct CacheEntry + { + UInt64 bytes = 0; + UInt8 size = 0; + UInt8 lengths = 0; + }; + + UInt8 matchingLengths(std::string_view token) const + { + const size_t size = token.size(); + if (size == 0 || size > cache_token_bytes) + return lookupMatchingLengths(token); + + // The first and last loads cover the complete token. The size + // disambiguates equal packed values from tokens of different lengths. + const char * data = token.data(); + UInt64 bytes; + if (size >= 4) + bytes = (static_cast(unalignedLoad(data)) << 32) | unalignedLoad(data + size - 4); + else if (size >= 2) + bytes = (static_cast(unalignedLoad(data)) << 16) | unalignedLoad(data + size - 2); + else + bytes = static_cast(*data); + + auto & entry = cache[intHashCRC32(bytes) & (cache.size() - 1)]; + const bool hit = entry.size == size && entry.bytes == bytes; + if (unlikely(cache_samples_remaining != 0)) + { + cache_sample_hits += hit; + if (--cache_samples_remaining == 0 && cache_sample_hits < cache.size() / 4) + cache_token_bytes = 0; + } + if (!hit) + entry = {bytes, static_cast(size), lookupMatchingLengths(token)}; + return entry.lengths; + } + + UInt8 lookupMatchingLengths(std::string_view token) const + { + const auto key = collator->sortKeyFastPath(token.data(), token.size(), key_buffer); + if (key.size > max_key_size) + return 0; + const auto * entry = keys.find(key); + return entry ? entry->getMapped() : 0; + } + + TiDB::TiDBCollatorPtr collator; + const StandardLengthLimit standard_length_limit; + Arena key_storage; + HashMap> keys; + size_t max_key_size = 0; + mutable String key_buffer; + // A lookup and its collation are local to one block. Cache exact source + // bytes, including misses, without keeping references to document data. + mutable std::array cache; + // Sample the first cache-sized batch of short tokens. Avoid the cost of + // probing the cache for the rest of the block when most probes miss. + mutable size_t cache_samples_remaining = cache.size(); + mutable size_t cache_sample_hits = 0; + mutable size_t cache_token_bytes = sizeof(UInt64); +}; + +bool isDefaultStopword(std::string_view token, size_t code_points, const BuiltinStopwordLookup * stopwords) +{ + return stopwords ? stopwords->containsStandard(token, code_points) + : defaultStopwords().contains(Poco::UTF8::toLower(String(token))); +} + +bool containsNgramStopword( + std::string_view token, + const std::vector & char_boundaries, + size_t ngram_start, + size_t ngram_token_size, + const BuiltinStopwordLookup * stopwords) +{ + const size_t ngram_end = ngram_start + ngram_token_size; + for (size_t start = ngram_start; start < ngram_end; ++start) + { + for (size_t end = start + 1; end <= ngram_end; ++end) + { + const auto candidate = token.substr(char_boundaries[start], char_boundaries[end] - char_boundaries[start]); + if (!stopwords && defaultStopwords().contains(Poco::UTF8::toLower(String(candidate)))) + return true; + if (stopwords && stopwords->containsNgram(candidate, end - start)) + return true; + } + } + return false; +} + +void analyzeTextInto( + std::string_view text, + FullTextColumn & result, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true, + bool borrow_document = false) +{ + result.clear(); + scanStandardTokens(text, [&](std::string_view token, size_t position, size_t code_points) { + // Filter before materializing. Stopwords use the source spelling and + // server collation, independently of the MATCH column collation. + if (code_points < min_token_size || code_points > max_token_size + || (enable_stopword && isDefaultStopword(token, code_points, stopwords))) + return; + result.append(token, position, code_points, borrow_document && collator); + if (!collator) + lowercaseTokenInPlace(result[result.size() - 1].text); + }); +} + +FullTextColumn analyzeText( + std::string_view text, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true) +{ + FullTextColumn result; + analyzeTextInto(text, result, collator, stopwords, min_token_size, max_token_size, enable_stopword); + return result; +} + +void analyzeNgramTextInto( + std::string_view text, + FullTextColumn & result, + AnalyzerScratch & scratch, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + bool enable_stopword = true, + bool borrow_document = false) +{ + result.clear(); + if (ngram_token_size == 0) + return; + + const bool preserve_case = enable_stopword && stopwords; + const size_t max_stopword_length = enable_stopword ? std::min(ngram_token_size, max_builtin_stopword_length) : 0; + size_t next_position_base = 0; + auto & char_boundaries = scratch.char_boundaries; + for (size_t offset = 0; offset < text.size();) + { + const auto [first_code_point, first_length] = decodeCodePoint(text, offset); + // Stop the document on malformed UTF-8, retain all valid multibyte + // characters, and split runs on ASCII nonwords, as MySQL does. + if (first_length == 1 && first_code_point >= 0x80) + break; + if (first_length == 1 && !isFullTextToken(first_code_point)) + { + offset += first_length; + continue; + } + const size_t run_start = offset; + char_boundaries.clear(); + char_boundaries.push_back(0); + offset += first_length; + char_boundaries.push_back(offset - run_start); + while (offset < text.size()) + { + const auto [code_point, length] = decodeCodePoint(text, offset); + if (length == 1 && (code_point >= 0x80 || !isFullTextToken(code_point))) + break; + offset += length; + char_boundaries.push_back(offset - run_start); + } + + std::string_view run = text.substr(run_start, offset - run_start); + const size_t char_count = char_boundaries.size() - 1; + const size_t base_position = next_position_base; + if (char_count < ngram_token_size) + { + // A short document run emits no grams and contributes no position. + continue; + } + + if (!collator && !preserve_case) + { + // The no-collator path lowercases a whole run before splitting. + // Unicode case mapping can change byte widths, so only that path + // needs a copy and another boundary scan. Never borrow scratch. + scratch.lowercase_run.assign(run.data(), run.size()); + lowercaseTokenInPlace(scratch.lowercase_run); + run = scratch.lowercase_run; + char_boundaries.clear(); + char_boundaries.push_back(0); + for (size_t i = 0; i < run.size();) + { + i += decodeCodePoint(run, i).second; + char_boundaries.push_back(i); + } + } + + const auto append_gram = [&](size_t start) ALWAYS_INLINE { + const size_t begin = char_boundaries[start]; + const size_t end = char_boundaries[start + ngram_token_size]; + result.append( + run.substr(begin, end - begin), + base_position + start, + ngram_token_size, + borrow_document && collator); + if (!collator && preserve_case) + lowercaseTokenInPlace(result[result.size() - 1].text); + }; + + // Keep the unfiltered stream and stopwords-OFF path free of per-gram + // stopword branches and sliding-state dependencies. + if (!enable_stopword) + { + for (size_t start = 0; start + ngram_token_size <= char_count; ++start) + append_gram(start); + next_position_base = base_position + char_count - ngram_token_size + 1; + continue; + } + + // Advance candidate endings once per run, instead of enumerating all + // substrings again for every overlapping gram. Keep one past the + // greatest start of a matched stopword ending at/before the gram end. + // The gram [start, end) is filtered iff that value is greater than + // start: such a stopword lies wholly inside the current gram. Once + // start advances beyond it, it expires without a queue or cache. + size_t stopword_end = 0; + size_t filtered_until_start = 0; + for (size_t start = 0; start + ngram_token_size <= char_count; ++start) + { + while (stopword_end < start + ngram_token_size) + { + ++stopword_end; + const size_t max_length = std::min(stopword_end, max_stopword_length); + for (size_t length = 1; length <= max_length; ++length) + { + const size_t candidate_start = stopword_end - length; + // Longer candidates at this ending have earlier starts + // and cannot improve a start already known to match. + if (candidate_start < filtered_until_start) + break; + const auto candidate = run.substr( + char_boundaries[candidate_start], + char_boundaries[stopword_end] - char_boundaries[candidate_start]); + const bool matched = stopwords + ? stopwords->containsNgram(candidate, length) + : defaultStopwords().contains(Poco::UTF8::toLower(String(candidate))); + if (matched) + { + filtered_until_start = candidate_start + 1; + break; + } + } + } + if (filtered_until_start <= start) + append_gram(start); + } + next_position_base = base_position + char_count - ngram_token_size + 1; + } +} + +FullTextColumn analyzeNgramText( + std::string_view text, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + bool enable_stopword = true) +{ + FullTextColumn result; + AnalyzerScratch scratch; + analyzeNgramTextInto(text, result, scratch, ngram_token_size, collator, stopwords, enable_stopword); + return result; +} + +bool textEquals(std::string_view lhs, std::string_view rhs, const TiDB::TiDBCollatorPtr & collator) +{ + if (!collator) + return lhs == rhs; + return collator->compare(lhs.data(), lhs.size(), rhs.data(), rhs.size()) == 0; +} + +String makePrefixPattern(std::string_view prefix) +{ + // Reuse the same collation-aware pattern implementation as LIKE. Escape + // token characters that have LIKE meaning before appending the wildcard. + String pattern; + pattern.reserve(prefix.size() + 1); + for (const char c : prefix) + { + if (c == '\\' || c == '%' || c == '_') + pattern.push_back('\\'); + pattern.push_back(c); + } + pattern.push_back('%'); + return pattern; +} + +std::unique_ptr compilePrefixMatcher( + std::string_view prefix, + const TiDB::TiDBCollatorPtr & collator) +{ + auto matcher = collator->pattern(); + matcher->compile(makePrefixPattern(prefix), '\\'); + return matcher; +} + +bool textStartsWith(std::string_view value, std::string_view prefix, const TiDB::TiDBCollatorPtr & collator) +{ + if (!collator) + return value.starts_with(prefix); + const auto matcher = compilePrefixMatcher(prefix, collator); + return matcher->match(value.data(), value.size()); +} + +// This call is per unmatched term/token, including small-query fallbacks. +ALWAYS_INLINE bool matchesTermAt( + const BooleanClause & clause, + size_t term_index, + const FullTextToken & token, + const TiDB::TiDBCollatorPtr & collator) +{ + const auto value = token.value(); + if (!clause.prefix) + return textEquals(value, clause.terms[term_index], collator); + if (collator && term_index < clause.prefix_matchers.size()) + return clause.prefix_matchers[term_index]->match(value.data(), value.size()); + return textStartsWith(value, clause.terms[term_index], collator); +} + +struct ClauseMatchState +{ + std::vector matched_terms; + size_t remaining_terms = 0; + bool matched = false; + bool phrase_verification_failed = false; +}; + +void initializeClauseMatchStates(const std::vector & clauses, std::vector & states) +{ + states.resize(clauses.size()); + for (size_t i = 0; i < clauses.size(); ++i) + if (!clauses[i].phrase) + { + states[i].matched_terms.resize(clauses[i].terms.size()); + states[i].remaining_terms = clauses[i].terms.size(); + } +} + +// Preserve the baseline's inlining after adding indexed/test call sites. +ALWAYS_INLINE bool matchesPhraseEndingAt( + const BooleanClause & clause, + const FullTextColumn & column, + const FullTextToken & end, + const TiDB::TiDBCollatorPtr & collator) +{ + if (clause.terms.empty() || clause.terms.size() != clause.offsets.size()) + return false; + + const size_t last_term = clause.terms.size() - 1; + const size_t last_offset = clause.offsets[last_term]; + if (end.position < last_offset || !matchesTermAt(clause, last_term, end, collator)) + return false; + + const size_t start_position = end.position - last_offset; + for (size_t i = 0; i < last_term; ++i) + { + const size_t expected_position = start_position + clause.offsets[i]; + const auto it = std::lower_bound( + column.begin(), + column.end(), + expected_position, + [](const FullTextToken & token, size_t position) { return token.position < position; }); + if (it == column.end() || it->position != expected_position + || !textEquals(it->value(), clause.terms[i], collator)) + return false; + } + return true; +} + +void analyzeColumnInto( + std::string_view document, + FullTextColumn & output, + AnalyzerScratch & scratch, + bool use_ngram, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true) +{ + if (use_ngram) + analyzeNgramTextInto(document, output, scratch, ngram_token_size, collator, stopwords, enable_stopword, true); + else + analyzeTextInto(document, output, collator, stopwords, min_token_size, max_token_size, enable_stopword, true); +} + +void resetClauseMatchStates(std::vector & states) +{ + for (auto & state : states) + { + state.matched = false; + state.remaining_terms = state.matched_terms.size(); + std::fill(state.matched_terms.begin(), state.matched_terms.end(), 0); + } +} + +struct BooleanMatchProgress +{ + size_t matched_must_clauses = 0; + bool matched_should_clause = false; +}; + +enum class ColumnMatchResult +{ + Continue, + Accepted, + Rejected, +}; + +ALWAYS_INLINE ColumnMatchResult markClauseMatched( + const CompiledBooleanQuery & query, + size_t index, + std::vector & states, + BooleanMatchProgress & progress) +{ + states[index].matched = true; + switch (query.clauses[index].modifier) + { + case BooleanClause::Modifier::Must: + ++progress.matched_must_clauses; + break; + case BooleanClause::Modifier::Should: + progress.matched_should_clause = true; + break; + case BooleanClause::Modifier::MustNot: + return ColumnMatchResult::Rejected; + } + if (!query.has_must_not + && (query.must_clause_count != 0 ? progress.matched_must_clauses == query.must_clause_count + : progress.matched_should_clause)) + return ColumnMatchResult::Accepted; + return ColumnMatchResult::Continue; +} + +ALWAYS_INLINE ColumnMatchResult markTermMatched( + const CompiledBooleanQuery & query, + size_t clause, + size_t term, + std::vector & states, + BooleanMatchProgress & progress) +{ + auto & state = states[clause]; + if (state.matched || state.matched_terms[term]) + return ColumnMatchResult::Continue; + state.matched_terms[term] = 1; + --state.remaining_terms; + // A STANDARD SQL word may split into several terms: MUST intersects them, + // while SHOULD/MUST NOT use their union. Both matchers share this rule. + if (query.clauses[clause].modifier != BooleanClause::Modifier::Must || state.remaining_terms == 0) + return markClauseMatched(query, clause, states, progress); + return ColumnMatchResult::Continue; +} + +template +ALWAYS_INLINE ColumnMatchResult updateBooleanMatchStatesForColumn( + const CompiledBooleanQuery & query, + const FullTextColumn & column, + GetUnfilteredColumn && get_unfiltered_column, + const TiDB::TiDBCollatorPtr & collator, + std::vector & states, + BooleanMatchProgress & progress) +{ + const auto & clauses = query.clauses; + if (query.matches_nothing || clauses.empty()) + return ColumnMatchResult::Continue; + + // Raw phrase verification searches the entire column, so a failed result + // needs no recheck at later anchors. Unlike positive clause matches, this + // negative cache belongs only to this column, not other MATCH columns. + for (auto & state : states) + state.phrase_verification_failed = false; + + // Walk this MATCH column's analyzed tokens once and update every clause's + // state. Phrase clauses are checked only when their final term is + // encountered; positional lookups preserve stopword gaps and phrases do + // not cross MATCH columns. + for (const auto & token : column) + { + for (size_t clause_index = 0; clause_index < clauses.size(); ++clause_index) + { + const auto & clause = clauses[clause_index]; + auto & state = states[clause_index]; + if (state.matched || state.phrase_verification_failed || clause.terms.empty()) + continue; + + if (clause.phrase) + { + if (matchesPhraseEndingAt(clause, column, token, collator)) + { + if (clause.verification) + { + const auto & unfiltered_column = get_unfiltered_column(); + if (!std::any_of(unfiltered_column.begin(), unfiltered_column.end(), [&](const auto & end) { + return matchesPhraseEndingAt(*clause.verification, unfiltered_column, end, collator); + })) + { + state.phrase_verification_failed = true; + continue; + } + } + const auto result = markClauseMatched(query, clause_index, states, progress); + if (result != ColumnMatchResult::Continue) + return result; + } + continue; + } + + for (size_t term_index = 0; term_index < clause.terms.size(); ++term_index) + if (!state.matched_terms[term_index] && matchesTermAt(clause, term_index, token, collator)) + { + state.matched_terms[term_index] = 1; + --state.remaining_terms; + } + state.matched = clause.modifier == BooleanClause::Modifier::Must + ? state.remaining_terms == 0 + : state.remaining_terms < clause.terms.size(); + if (state.matched) + { + const auto result = markClauseMatched(query, clause_index, states, progress); + if (result != ColumnMatchResult::Continue) + return result; + } + } + } + return ColumnMatchResult::Continue; +} + +template +ColumnMatchResult updateExactMatchStatesForColumn( + const CompiledBooleanQuery & query, + const FullTextColumn & column, + const TiDB::TiDBCollatorPtr & collator, + std::vector & states, + BooleanMatchProgress & progress, + ExactMatchScratch & scratch, + LocalMatchAgainstTest::Work * work) +{ + const auto & route = *query.exact_route; + for (const auto & token : column) + { + if constexpr (count_work) + ++work->key_lookups; + const auto * const it = route.dictionary.find(scratch.keys.key(token.value(), collator)); + if (!it) + continue; + const auto bucket_id = it->getMapped(); + if (scratch.seen[bucket_id] == scratch.generation) + continue; + scratch.seen[bucket_id] = scratch.generation; + const auto & bucket = route.buckets[bucket_id]; + for (size_t i = bucket.offset; i < bucket.offset + bucket.count; ++i) + { + if constexpr (count_work) + ++work->term_references; + const auto & ref = route.references[i]; + const auto result = markTermMatched(query, ref.clause, ref.term, states, progress); + // has_must_not prevents accepting before a later prohibited + // association in this same bucket (e.g. +apple -APPLE). + if (result != ColumnMatchResult::Continue) + return result; + } + } + return ColumnMatchResult::Continue; +} + +bool evaluateBooleanMatchResult(const CompiledBooleanQuery & query, const std::vector & states) +{ + if (query.matches_nothing || query.clauses.empty()) + return false; + bool has_positive = false; + bool has_must = false; + bool positive_match = false; + for (size_t i = 0; i < query.clauses.size(); ++i) + { + const auto & clause = query.clauses[i]; + const bool matched = states[i].matched; + switch (clause.modifier) + { + case BooleanClause::Modifier::Must: + has_must = true; + if (!matched) + return false; + break; + case BooleanClause::Modifier::MustNot: + if (matched) + return false; + break; + case BooleanClause::Modifier::Should: + has_positive = true; + positive_match = positive_match || matched; + break; + } + } + + return has_must || !has_positive || positive_match; +} + +CompiledBooleanQuery compileBooleanQuery( + const tipb::LocalMatchAgainstBooleanQuery & query, + const LocalMatchAnalyzerConfig & config, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr) +{ + CompiledBooleanQuery compiled; + if (!config.use_ngram && (config.max_token_size == 0 || config.min_token_size > config.max_token_size)) + { + compiled.matches_nothing = true; + return compiled; + } + auto analyze_query = [&](std::string_view text) { + return config.use_ngram + ? analyzeNgramText(text, config.ngram_token_size, collator, stopwords, config.enable_stopword) + : analyzeText( + text, + collator, + stopwords, + config.min_token_size, + config.max_token_size, + config.enable_stopword); + }; + for (const auto & node : query.nodes()) + { + BooleanClause clause; + size_t first_query_position = 0; + switch (node.occur()) + { + case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurMust: + clause.modifier = BooleanClause::Modifier::Must; + break; + case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurMustNot: + clause.modifier = BooleanClause::Modifier::MustNot; + break; + case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurShould: + clause.modifier = BooleanClause::Modifier::Should; + break; + default: + throw Exception("Invalid Local MATCH Boolean node occurrence", ErrorCodes::ILLEGAL_COLUMN); + } + + switch (node.term_type()) + { + case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermWord: + { + const auto terms = analyze_query(node.text()); + clause.phrase = config.use_ngram; + first_query_position = collectClauseTerms(clause, terms); + break; + } + case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermPrefix: + { + const auto terms = config.use_ngram + ? analyzeNgramText(node.text(), config.ngram_token_size, collator, stopwords, config.enable_stopword) + : tokenizeText(node.text(), collator); + if (config.use_ngram) + { + if (!terms.empty()) + { + // TiDB expands an NGRAM prefix at least as long as one + // gram into a positional phrase of query ngrams. Matching + // only one analyzed term here makes prefixes such as + // "caf*" ("ca", "af" at token size 2) never match. + clause.phrase = true; + first_query_position = collectClauseTerms(clause, terms); + break; + } + + // A query shorter than the configured gram size is retained + // by TiDB as a prefix over document ngrams instead of being + // discarded by the analyzer. + const auto source_terms = tokenizeText(node.text(), collator); + if (source_terms.size() == 1) + { + const auto code_points = UTF8::countCodePoints( + reinterpret_cast(source_terms.front().text.data()), + source_terms.front().text.size()); + if (code_points < config.ngram_token_size) + { + clause.prefix = true; + clause.terms.push_back(source_terms.front().text); + break; + } + } + + if (clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + continue; + } + + clause.prefix = true; + if (terms.size() != 1) + { + if (clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + continue; + } + const auto code_points = UTF8::countCodePoints( + reinterpret_cast(terms.front().text.data()), + terms.front().text.size()); + if (code_points > config.max_token_size) + { + if (clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + continue; + } + clause.terms.push_back(terms.front().text); + break; + } + case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermPhrase: + { + clause.phrase = true; + const auto terms = analyze_query(node.text()); + if (config.use_ngram) + { + const auto words = tokenizeText(node.text(), collator); + bool seen_indexed_word = false; + if (std::any_of(words.begin(), words.end(), [&](const auto & word) { + if (word.code_points >= config.ngram_token_size) + { + seen_indexed_word = seen_indexed_word || !analyze_query(word.text).empty(); + return false; + } + if (!config.enable_stopword || seen_indexed_word) + return true; + std::vector boundaries{0}; + for (size_t offset = 0; offset < word.text.size();) + { + offset += decodeCodePoint(word.text, offset).second; + boundaries.push_back(offset); + } + return !containsNgramStopword(word.text, boundaries, 0, word.code_points, stopwords); + })) + break; // A short query unigram cannot occur in the document's fixed-size ngrams. + } + first_query_position = collectClauseTerms(clause, terms); + break; + } + default: + throw Exception("Invalid Local MATCH Boolean node term type", ErrorCodes::ILLEGAL_COLUMN); + } + + if (clause.phrase && !clause.terms.empty()) + { + auto raw_terms = config.use_ngram + ? analyzeNgramText(node.text(), config.ngram_token_size, collator, nullptr, false) + : analyzeText(node.text(), collator, nullptr, 0, std::numeric_limits::max(), false); + // InnoDB discards leading filtered tokens, but retains every + // following word in the original phrase used for verification. + raw_terms.filterInPlace([&](const auto & token) { return token.position >= first_query_position; }); + if (raw_terms.size() != clause.terms.size()) + { + // InnoDB verifies phrases against the original document, not + // just the index tokens: removing "the" must not turn it into + // an arbitrary positional gap. Ordinary terms/prefixes do not + // pay for this additional document stream. + clause.verification = std::make_unique(); + clause.verification->phrase = true; + collectClauseTerms(*clause.verification, raw_terms); + } + } + + if (clause.prefix && collator) + { + clause.prefix_matchers.reserve(clause.terms.size()); + for (const auto & term : clause.terms) + clause.prefix_matchers.push_back(compilePrefixMatcher(term, collator)); + } + + if (!clause.terms.empty() || clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + } + + for (const auto & clause : compiled.clauses) + { + if (clause.modifier == BooleanClause::Modifier::Must) + { + // An analyzer-filtered required clause cannot match any document. + // Do this after compiling ALL nodes so malformed later nodes + // retain their existing no-match handling, rather than being + // hidden by an earlier block-level shortcut. + if (clause.terms.empty()) + { + compiled.matches_nothing = true; + return compiled; + } + ++compiled.must_clause_count; + } + else if (clause.modifier == BooleanClause::Modifier::MustNot) + compiled.has_must_not = true; + } + + // A BOOLEAN MODE query containing only prohibited terms has no positive + // branch. TiDB's local evaluator treats it as matching no rows. + if (std::none_of(compiled.clauses.begin(), compiled.clauses.end(), [](const BooleanClause & clause) { + return clause.modifier != BooleanClause::Modifier::MustNot; + })) + compiled.matches_nothing = true; + + return compiled; +} + +size_t eligibleExactRouteTermCount( + const CompiledBooleanQuery & query, + const LocalMatchAnalyzerConfig & config, + const TiDB::TiDBCollatorPtr & collator, + LocalMatchAgainstTest::Matcher matcher, + size_t min_terms, + size_t budget) +{ + if (matcher == LocalMatchAgainstTest::Matcher::Linear || config.use_ngram || !collator) + return {}; + // Compare and sortKey in these implementations encode the same weight + // stream and use identical PAD/NO PAD preprocessing. GeneralCI emits one + // 16-bit weight per rune; UCA emits the same expansion/ignorable-weight + // stream consumed by compare. Binary keys preserve byte equality. + using Type = TiDB::ITiDBCollator::CollatorType; + switch (collator->getCollatorType()) + { + case Type::UTF8_GENERAL_CI: + case Type::UTF8MB4_GENERAL_CI: + case Type::UTF8_UNICODE_CI: + case Type::UTF8MB4_UNICODE_CI: + case Type::UTF8MB4_0900_AI_CI: + case Type::UTF8MB4_BIN: + case Type::UTF8_BIN: + case Type::UTF8MB4_0900_BIN: + case Type::BINARY: + case Type::LATIN1_BIN: + case Type::ASCII_BIN: + break; + default: + return {}; + } + + size_t terms = 0; + // UCA may resize a document key buffer to 16 bytes per input byte; + // allow string capacity growth independently of the query term sizes. + size_t estimate = 32 * 1024 + 2 * 16 * exact_route_max_token_bytes; + if (estimate > budget) + return {}; + for (const auto & clause : query.clauses) + { + if (clause.phrase || clause.prefix || clause.verification) + return {}; + for (const auto & term : clause.terms) + { + if (term.size() > exact_route_max_token_bytes) + return {}; + // Conservative bound for power-of-two hash/POD capacities, + // Arena growth and up to 16 key bytes per input byte. Charge + // scratch growth too. Check before constructing any route. + const size_t cost = 256 + 96 * term.size(); + if (cost > budget - estimate) + return {}; + estimate += cost; + ++terms; + } + } + if (terms == 0 || (matcher == LocalMatchAgainstTest::Matcher::Automatic && terms < min_terms)) + return {}; + + return terms; +} + +std::unique_ptr buildExactTermRoute( + const CompiledBooleanQuery & query, + const TiDB::TiDBCollatorPtr & collator, + size_t terms) +{ + auto route = std::make_unique(); + route->references.reserve(terms); + route->buckets.reserve(terms); + MatchKeyBuffer keys; + for (size_t clause = 0; clause < query.clauses.size(); ++clause) + for (size_t term = 0; term < query.clauses[clause].terms.size(); ++term) + { + const auto key = keys.key(query.clauses[clause].terms[term], collator); + auto * it = route->dictionary.find(key); + size_t bucket; + if (it) + bucket = it->getMapped(); + else + { + bucket = route->buckets.size(); + const StringRef owned(route->keys.insert(key.data, key.size), key.size); + bool inserted; + route->dictionary.emplace(owned, it, inserted); + it->getMapped() = bucket; + route->buckets.push_back(ExactTermRoute::Bucket{0, 0}); + } + route->references.push_back(ExactTermRoute::Reference{bucket, clause, term}); + } + std::sort(route->references.begin(), route->references.end(), [](const auto & a, const auto & b) { + return a.bucket < b.bucket; + }); + for (size_t i = 0; i < route->references.size(); ++i) + { + auto & bucket = route->buckets[route->references[i].bucket]; + if (bucket.count == 0) + bucket.offset = i; + ++bucket.count; + } + return route; +} + +Float64 matchBooleanValue(const CompiledBooleanQuery & query, const std::vector & states) +{ + // Local MATCH returns a Boolean 0/1 value, not a relevance score. + return evaluateBooleanMatchResult(query, states) ? 1 : 0; +} + +class FullTextColumnAccessor +{ +public: + explicit FullTextColumnAccessor(const IColumn & column) { initialize(column, false); } + + bool isNull(size_t row) const + { + return nullable_column != nullptr && nullable_column->getNullMapData()[nullable_is_constant ? 0 : row] != 0; + } + + std::string_view getString(size_t row) const + { + const auto & chars = string_column->getChars(); + const auto & offsets = string_column->getOffsets(); + const size_t string_row = string_is_constant ? 0 : row; + const size_t begin = string_row == 0 ? 0 : offsets[string_row - 1]; + const size_t end = offsets[string_row]; + return {reinterpret_cast(&chars[begin]), end - begin - 1}; + } + +private: + void initialize(const IColumn & column, bool row_is_constant) + { + if (const auto * column_const = typeid_cast(&column)) + { + string_is_constant = true; + initialize(column_const->getDataColumn(), true); + return; + } + if (const auto * column_nullable = typeid_cast(&column)) + { + nullable_column = column_nullable; + nullable_is_constant = row_is_constant; + initialize(column_nullable->getNestedColumn(), row_is_constant); + return; + } + string_column = checkAndGetColumn(&column); + if (string_column == nullptr) + throw Exception("Full-text arguments must be string columns", ErrorCodes::ILLEGAL_COLUMN); + } + + const ColumnString * string_column = nullptr; + const ColumnNullable * nullable_column = nullptr; + bool string_is_constant = false; + bool nullable_is_constant = false; +}; + +bool decodeLocalMatchAgainstBooleanQuery(const IColumn & column, tipb::LocalMatchAgainstBooleanQuery & query) +{ + const auto * constant = typeid_cast(&column); + if (constant == nullptr) + return false; + const auto encoded = constant->getValue(); + // Expr.val directly contains the versioned Local MATCH query. Validate the + // semantic version before interpreting parser or stopword fields; never + // guess a default for a version this TiFlash binary does not understand. + if (!query.ParseFromString(encoded) || query.version() != local_match_against_protocol_version) + return false; + if (query.parser() != tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard + && query.parser() != tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram) + return false; + if (query.stopword_mode() != tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled + && query.stopword_mode() != tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin) + return false; + // Validate every node before NULL-search or analyzer no-match shortcuts. + // Missing/unknown proto2 enum values read as Invalid, not as valid clauses. + for (const auto & node : query.nodes()) + { + if (node.occur() != tipb::LocalMatchAgainstBooleanOccurMust + && node.occur() != tipb::LocalMatchAgainstBooleanOccurMustNot + && node.occur() != tipb::LocalMatchAgainstBooleanOccurShould) + return false; + if (node.term_type() != tipb::LocalMatchAgainstBooleanTermWord + && node.term_type() != tipb::LocalMatchAgainstBooleanTermPrefix + && node.term_type() != tipb::LocalMatchAgainstBooleanTermPhrase) + return false; + } + return true; +} + +class FunctionLocalMatchAgainstBoolean final : public IFunction +{ +public: + explicit FunctionLocalMatchAgainstBoolean( + LocalMatchAgainstTest::Matcher matcher_ = LocalMatchAgainstTest::Matcher::Automatic, + LocalMatchAgainstTest::Work * work_ = nullptr, + size_t budget_ = exact_route_budget, + UInt32 generation_ = 0) + : matcher(matcher_) + , work(work_) + , budget(budget_) + , generation(generation_) + {} + static constexpr auto name = "local_match_against_boolean"; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + bool useDefaultImplementationForConstants() const override { return false; } + // NULL MATCH columns are empty documents, not NULL MATCH results. The + // implementation below handles nullable columns per row, so the generic + // IFunction NULL wrapper must not short-circuit the whole expression. + bool useDefaultImplementationForNulls() const override { return false; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + void setCollator(const TiDB::TiDBCollatorPtr & collator_) override { collator = collator_; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() < 3) + throw Exception("local_match_against_boolean requires a query, at least one column, and metadata"); + bool nullable = false; + for (size_t i = 0; i < arguments.size(); ++i) + { + const auto & argument = arguments[i]; + const auto nested_type = removeNullable(argument); + // An untyped SQL NULL search arrives as Nullable(Nothing). + // Documents and metadata still require string types. + const bool null_query = i == 0 && nested_type->getTypeId() == TypeIndex::Nothing; + if (!nested_type->isString() && !null_query) + throw Exception( + "Illegal type " + argument->getName() + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + nullable = nullable || argument->isNullable(); + } + DataTypePtr result_type = std::make_shared(); + return nullable ? std::make_shared(result_type) : result_type; + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override + { + if (work) + executeInternal(block, arguments, result); + else + executeInternal(block, arguments, result); + } + + template + void executeInternal(Block & block, const ColumnNumbers & arguments, size_t result) const + { + const auto * query_column = typeid_cast(&*block.getByPosition(arguments[0]).column); + if (query_column == nullptr) + throw Exception( + "The query argument of local_match_against_boolean must be constant", + ErrorCodes::ILLEGAL_COLUMN); + + const size_t rows = block.getByPosition(arguments[1]).column->size(); + auto output = ColumnFloat64::create(rows, 0); + auto & output_data = output->getData(); + const bool nullable = block.getByPosition(result).type->isNullable(); + auto null_map = nullable ? ColumnUInt8::create(rows, 0) : nullptr; + const auto finish_result = [&] { + ColumnPtr result_column = std::move(output); + if (null_map) + result_column = ColumnNullable::create(std::move(result_column), std::move(null_map)); + block.getByPosition(result).column = std::move(result_column); + }; + + tipb::LocalMatchAgainstBooleanQuery protocol_boolean_query; + if (arguments.size() <= 2 + || !decodeLocalMatchAgainstBooleanQuery( + *block.getByPosition(arguments.back()).column, + protocol_boolean_query)) + throw Exception( + "local_match_against_boolean requires valid Boolean query metadata", + ErrorCodes::ILLEGAL_COLUMN); + if (query_column->isNullAt(0)) + { + // InnoDB interprets NULL AGAINST as an empty search: non-NULL zero. + finish_result(); + return; + } + const size_t document_argument_end = arguments.size() - 1; + const LocalMatchAnalyzerConfig config(protocol_boolean_query); + // Keep scalar values in the row hot path. Passing the config by + // reference changes Release code generation, notably with stopwords + // disabled; the config object belongs to query compilation only. + const bool use_ngram = config.use_ngram; + const size_t ngram_token_size = config.ngram_token_size; + const size_t min_token_size = config.min_token_size; + const size_t max_token_size = config.max_token_size; + const bool enable_stopword = config.enable_stopword; + const auto * const stopword_collator + = enable_stopword ? TiDB::ITiDBCollator::getCollator(protocol_boolean_query.stopword_collation()) : nullptr; + if (enable_stopword && !stopword_collator) + throw Exception( + "local_match_against_boolean requires a supported stopword collation when stopwords are enabled", + ErrorCodes::ILLEGAL_COLUMN); + std::optional stopword_lookup; + if (enable_stopword) + stopword_lookup.emplace(stopword_collator); + const auto * stopwords = stopword_lookup ? &*stopword_lookup : nullptr; + CompiledBooleanQuery compiled_query = compileBooleanQuery(protocol_boolean_query, config, collator, stopwords); + if (compiled_query.matches_nothing) + { + finish_result(); + return; + } + std::vector document_columns; + document_columns.reserve(document_argument_end - 1); + for (size_t arg = 1; arg < document_argument_end; ++arg) + document_columns.emplace_back(*block.getByPosition(arguments[arg]).column); + const bool large_block = rows >= exact_route_large_block_min_rows; + const size_t min_route_terms = large_block ? exact_route_min_terms_large_block : exact_route_min_terms; + const size_t route_terms + = eligibleExactRouteTermCount(compiled_query, config, collator, matcher, min_route_terms, budget); + const size_t min_route_tokens = large_block ? exact_route_min_tokens_large_block : exact_route_min_tokens; + bool may_use_index = matcher == LocalMatchAgainstTest::Matcher::Indexed; + if (route_terms && !may_use_index) + { + // A STANDARD token consumes at least one raw byte, and consecutive + // tokens need a separator. This cheap lower bound bypasses the + // routing row loop for wholly short blocks without tokenizing twice. + // Use one byte even for unusual analyzer bounds to avoid overflow. + const size_t min_document_bytes = min_route_tokens * 2 - 1; + for (const auto & column : document_columns) + { + for (size_t row = 0; row < rows; ++row) + if (!column.isNull(row) && column.getString(row).size() >= min_document_bytes) + { + may_use_index = true; + break; + } + if (may_use_index) + break; + } + } + // Compile separate row paths so ineligible queries discard optional + // scratch and routing branches from their analyzer/matcher hot loop. + const auto execute_rows = [&, + use_ngram, + ngram_token_size, + min_token_size, + max_token_size, + enable_stopword]() ALWAYS_INLINE { + std::vector clause_states; + initializeClauseMatchStates(compiled_query.clauses, clause_states); + FullTextColumn analyzed_column; + FullTextColumn unfiltered_column; + AnalyzerScratch analyzer_scratch; + + std::optional exact_scratch; + bool route_attempted = false; + for (size_t row = 0; row < rows; ++row) + { + if constexpr (enable_index) + if (exact_scratch) + exact_scratch->nextRow(); + resetClauseMatchStates(clause_states); + BooleanMatchProgress match_progress; + bool result_known = false; + bool row_matches = false; + for (const auto & document_column : document_columns) + { + // A NULL MATCH column contributes no tokens. This is + // intentional: #70485 relies on a row with a NULL body + // still matching a required term from another MATCH column. + if (document_column.isNull(row)) + { + analyzed_column.clear(); + continue; + } + const auto document = document_column.getString(row); + analyzeColumnInto( + document, + analyzed_column, + analyzer_scratch, + use_ngram, + ngram_token_size, + collator, + stopwords, + min_token_size, + max_token_size, + enable_stopword); + if constexpr (enable_index) + { + // Do not allocate a query dictionary for blocks containing + // only short documents. Build at most once, after metadata + // validation and complete clause compilation. Earlier short + // columns may already have contributed to this row's state. + if (!route_attempted + && (matcher == LocalMatchAgainstTest::Matcher::Indexed + || analyzed_column.size() >= min_route_tokens)) + { + route_attempted = true; + compiled_query.exact_route = buildExactTermRoute(compiled_query, collator, route_terms); + if (compiled_query.exact_route) + { + exact_scratch.emplace(compiled_query.exact_route->buckets.size(), generation); + exact_scratch->nextRow(); + if constexpr (count_work) + ++work->route_builds; + } + } + } + // A filtered phrase hit is the first point that needs the + // raw tokens. Share the stream across clauses in this column. + bool unfiltered_ready = false; + const auto get_unfiltered_column = [&]() -> const FullTextColumn & { + if (!unfiltered_ready) + { + analyzeColumnInto( + document, + unfiltered_column, + analyzer_scratch, + use_ngram, + ngram_token_size, + collator, + nullptr, + 0, + std::numeric_limits::max(), + false); + unfiltered_ready = true; + if constexpr (count_work) + ++work->unfiltered_column_analyses; + } + return unfiltered_column; + }; + ColumnMatchResult match_result; + if constexpr (enable_index) + { + // STANDARD only emits valid UTF-8 BMP word tokens: malformed + // bytes become RuneError separators. Do not re-interpret the + // raw document. An oversized analyzed token falls back for + // this whole column before any state is modified. + const bool indexed = exact_scratch + && (matcher == LocalMatchAgainstTest::Matcher::Indexed + || analyzed_column.size() >= min_route_tokens) + // Valid BMP tokens use at most three bytes per code + // point. The usual analyzer bound proves this without + // another column pass, including early-hit OR queries. + && (max_token_size <= exact_route_max_token_bytes / 3 + || std::all_of(analyzed_column.begin(), analyzed_column.end(), [](const auto & token) { + return token.value().size() <= exact_route_max_token_bytes; + })); + if constexpr (count_work) + { + if (indexed) + ++work->indexed_columns; + else + ++work->linear_columns; + } + match_result = indexed ? updateExactMatchStatesForColumn( + compiled_query, + analyzed_column, + collator, + clause_states, + match_progress, + *exact_scratch, + work) + : updateBooleanMatchStatesForColumn( + compiled_query, + analyzed_column, + get_unfiltered_column, + collator, + clause_states, + match_progress); + } + else + { + if constexpr (count_work) + ++work->linear_columns; + match_result = updateBooleanMatchStatesForColumn( + compiled_query, + analyzed_column, + get_unfiltered_column, + collator, + clause_states, + match_progress); + } + if (match_result == ColumnMatchResult::Rejected) + { + result_known = true; + break; + } + if (match_result == ColumnMatchResult::Accepted) + { + result_known = true; + row_matches = true; + break; + } + } + output_data[row] + = result_known ? (row_matches ? 1 : 0) : matchBooleanValue(compiled_query, clause_states); + } + }; + if (route_terms && may_use_index) + execute_rows.template operator()(); + else + execute_rows.template operator()(); + finish_result(); + } + +private: + TiDB::TiDBCollatorPtr collator = nullptr; + LocalMatchAgainstTest::Matcher matcher; + LocalMatchAgainstTest::Work * work; + size_t budget; + UInt32 generation; +}; +} // namespace + +FunctionPtr LocalMatchAgainstTest::create(Matcher matcher, Work & work, size_t budget, UInt32 generation) +{ + return std::make_shared(matcher, &work, budget, generation); +} + +void registerFunctionsLocalMatchAgainst(FunctionFactory & factory) +{ + factory.registerFunction(FunctionFactory::CaseInsensitive); +} +} // namespace DB diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.h b/dbms/src/Functions/FunctionsLocalMatchAgainst.h new file mode 100644 index 00000000000..cd39439e5fb --- /dev/null +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.h @@ -0,0 +1,43 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace DB +{ +void registerFunctionsLocalMatchAgainst(FunctionFactory & factory); + +// Internal test seam: no SQL setting, global state, or request-shared scratch. +namespace LocalMatchAgainstTest +{ +enum class Matcher +{ + Automatic, + Linear, + Indexed +}; +struct Work +{ + size_t route_builds = 0; + size_t indexed_columns = 0; + size_t linear_columns = 0; + size_t key_lookups = 0; + size_t term_references = 0; + size_t unfiltered_column_analyses = 0; +}; +FunctionPtr create(Matcher matcher, Work & work, size_t budget = 4 * 1024 * 1024, UInt32 generation = 0); +} // namespace LocalMatchAgainstTest +} // namespace DB diff --git a/dbms/src/Functions/LocalMatchAgainstTokenChars.h b/dbms/src/Functions/LocalMatchAgainstTokenChars.h new file mode 100644 index 00000000000..abb9e63d045 --- /dev/null +++ b/dbms/src/Functions/LocalMatchAgainstTokenChars.h @@ -0,0 +1,217 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated by TiDB pkg/expression/matchagainst/generator; DO NOT EDIT. +#pragma once + +#include +#include +#include + +namespace DB::LocalMatchAgainst +{ +// Generated from Go's Unicode 17.0.0 categories L and N. TiDB uses its standard +// library and strictly tests that Unicode version and the Unicode 17.0.0 checksum. +// Equivalent Go predicate: +// word := r <= unicode.MaxRune && +// (unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_') +// This helper covers all Unicode code points; the Boolean lexer separately +// restricts word characters to the BMP (r <= 0xFFFF). +// Regenerate/check this header using the paired TiDB generator. TiDB and +// TiFlash must use the same character classification for Local MATCH. +// ASCII and '_' use the fast path below. +struct TokenCharRange +{ + std::uint32_t first; + std::uint32_t last; +}; +inline constexpr std::array token_char_ranges{{ + {0xAA, 0xAA}, {0xB2, 0xB3}, {0xB5, 0xB5}, {0xB9, 0xBA}, {0xBC, 0xBE}, + {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x2C1}, {0x2C6, 0x2D1}, {0x2E0, 0x2E4}, + {0x2EC, 0x2EC}, {0x2EE, 0x2EE}, {0x370, 0x374}, {0x376, 0x377}, {0x37A, 0x37D}, + {0x37F, 0x37F}, {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x3A1}, + {0x3A3, 0x3F5}, {0x3F7, 0x481}, {0x48A, 0x52F}, {0x531, 0x556}, {0x559, 0x559}, + {0x560, 0x588}, {0x5D0, 0x5EA}, {0x5EF, 0x5F2}, {0x620, 0x64A}, {0x660, 0x669}, + {0x66E, 0x66F}, {0x671, 0x6D3}, {0x6D5, 0x6D5}, {0x6E5, 0x6E6}, {0x6EE, 0x6FC}, + {0x6FF, 0x6FF}, {0x710, 0x710}, {0x712, 0x72F}, {0x74D, 0x7A5}, {0x7B1, 0x7B1}, + {0x7C0, 0x7EA}, {0x7F4, 0x7F5}, {0x7FA, 0x7FA}, {0x800, 0x815}, {0x81A, 0x81A}, + {0x824, 0x824}, {0x828, 0x828}, {0x840, 0x858}, {0x860, 0x86A}, {0x870, 0x887}, + {0x889, 0x88F}, {0x8A0, 0x8C9}, {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, + {0x958, 0x961}, {0x966, 0x96F}, {0x971, 0x980}, {0x985, 0x98C}, {0x98F, 0x990}, + {0x993, 0x9A8}, {0x9AA, 0x9B0}, {0x9B2, 0x9B2}, {0x9B6, 0x9B9}, {0x9BD, 0x9BD}, + {0x9CE, 0x9CE}, {0x9DC, 0x9DD}, {0x9DF, 0x9E1}, {0x9E6, 0x9F1}, {0x9F4, 0x9F9}, + {0x9FC, 0x9FC}, {0xA05, 0xA0A}, {0xA0F, 0xA10}, {0xA13, 0xA28}, {0xA2A, 0xA30}, + {0xA32, 0xA33}, {0xA35, 0xA36}, {0xA38, 0xA39}, {0xA59, 0xA5C}, {0xA5E, 0xA5E}, + {0xA66, 0xA6F}, {0xA72, 0xA74}, {0xA85, 0xA8D}, {0xA8F, 0xA91}, {0xA93, 0xAA8}, + {0xAAA, 0xAB0}, {0xAB2, 0xAB3}, {0xAB5, 0xAB9}, {0xABD, 0xABD}, {0xAD0, 0xAD0}, + {0xAE0, 0xAE1}, {0xAE6, 0xAEF}, {0xAF9, 0xAF9}, {0xB05, 0xB0C}, {0xB0F, 0xB10}, + {0xB13, 0xB28}, {0xB2A, 0xB30}, {0xB32, 0xB33}, {0xB35, 0xB39}, {0xB3D, 0xB3D}, + {0xB5C, 0xB5D}, {0xB5F, 0xB61}, {0xB66, 0xB6F}, {0xB71, 0xB77}, {0xB83, 0xB83}, + {0xB85, 0xB8A}, {0xB8E, 0xB90}, {0xB92, 0xB95}, {0xB99, 0xB9A}, {0xB9C, 0xB9C}, + {0xB9E, 0xB9F}, {0xBA3, 0xBA4}, {0xBA8, 0xBAA}, {0xBAE, 0xBB9}, {0xBD0, 0xBD0}, + {0xBE6, 0xBF2}, {0xC05, 0xC0C}, {0xC0E, 0xC10}, {0xC12, 0xC28}, {0xC2A, 0xC39}, + {0xC3D, 0xC3D}, {0xC58, 0xC5A}, {0xC5C, 0xC5D}, {0xC60, 0xC61}, {0xC66, 0xC6F}, + {0xC78, 0xC7E}, {0xC80, 0xC80}, {0xC85, 0xC8C}, {0xC8E, 0xC90}, {0xC92, 0xCA8}, + {0xCAA, 0xCB3}, {0xCB5, 0xCB9}, {0xCBD, 0xCBD}, {0xCDC, 0xCDE}, {0xCE0, 0xCE1}, + {0xCE6, 0xCEF}, {0xCF1, 0xCF2}, {0xD04, 0xD0C}, {0xD0E, 0xD10}, {0xD12, 0xD3A}, + {0xD3D, 0xD3D}, {0xD4E, 0xD4E}, {0xD54, 0xD56}, {0xD58, 0xD61}, {0xD66, 0xD78}, + {0xD7A, 0xD7F}, {0xD85, 0xD96}, {0xD9A, 0xDB1}, {0xDB3, 0xDBB}, {0xDBD, 0xDBD}, + {0xDC0, 0xDC6}, {0xDE6, 0xDEF}, {0xE01, 0xE30}, {0xE32, 0xE33}, {0xE40, 0xE46}, + {0xE50, 0xE59}, {0xE81, 0xE82}, {0xE84, 0xE84}, {0xE86, 0xE8A}, {0xE8C, 0xEA3}, + {0xEA5, 0xEA5}, {0xEA7, 0xEB0}, {0xEB2, 0xEB3}, {0xEBD, 0xEBD}, {0xEC0, 0xEC4}, + {0xEC6, 0xEC6}, {0xED0, 0xED9}, {0xEDC, 0xEDF}, {0xF00, 0xF00}, {0xF20, 0xF33}, + {0xF40, 0xF47}, {0xF49, 0xF6C}, {0xF88, 0xF8C}, {0x1000, 0x102A}, {0x103F, 0x1049}, + {0x1050, 0x1055}, {0x105A, 0x105D}, {0x1061, 0x1061}, {0x1065, 0x1066}, {0x106E, 0x1070}, + {0x1075, 0x1081}, {0x108E, 0x108E}, {0x1090, 0x1099}, {0x10A0, 0x10C5}, {0x10C7, 0x10C7}, + {0x10CD, 0x10CD}, {0x10D0, 0x10FA}, {0x10FC, 0x1248}, {0x124A, 0x124D}, {0x1250, 0x1256}, + {0x1258, 0x1258}, {0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0}, + {0x12B2, 0x12B5}, {0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, + {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, {0x1369, 0x137C}, {0x1380, 0x138F}, + {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1401, 0x166C}, {0x166F, 0x167F}, {0x1681, 0x169A}, + {0x16A0, 0x16EA}, {0x16EE, 0x16F8}, {0x1700, 0x1711}, {0x171F, 0x1731}, {0x1740, 0x1751}, + {0x1760, 0x176C}, {0x176E, 0x1770}, {0x1780, 0x17B3}, {0x17D7, 0x17D7}, {0x17DC, 0x17DC}, + {0x17E0, 0x17E9}, {0x17F0, 0x17F9}, {0x1810, 0x1819}, {0x1820, 0x1878}, {0x1880, 0x1884}, + {0x1887, 0x18A8}, {0x18AA, 0x18AA}, {0x18B0, 0x18F5}, {0x1900, 0x191E}, {0x1946, 0x196D}, + {0x1970, 0x1974}, {0x1980, 0x19AB}, {0x19B0, 0x19C9}, {0x19D0, 0x19DA}, {0x1A00, 0x1A16}, + {0x1A20, 0x1A54}, {0x1A80, 0x1A89}, {0x1A90, 0x1A99}, {0x1AA7, 0x1AA7}, {0x1B05, 0x1B33}, + {0x1B45, 0x1B4C}, {0x1B50, 0x1B59}, {0x1B83, 0x1BA0}, {0x1BAE, 0x1BE5}, {0x1C00, 0x1C23}, + {0x1C40, 0x1C49}, {0x1C4D, 0x1C7D}, {0x1C80, 0x1C8A}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, + {0x1CE9, 0x1CEC}, {0x1CEE, 0x1CF3}, {0x1CF5, 0x1CF6}, {0x1CFA, 0x1CFA}, {0x1D00, 0x1DBF}, + {0x1E00, 0x1F15}, {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, + {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, + {0x1FB6, 0x1FBC}, {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4}, {0x1FC6, 0x1FCC}, {0x1FD0, 0x1FD3}, + {0x1FD6, 0x1FDB}, {0x1FE0, 0x1FEC}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, {0x2070, 0x2071}, + {0x2074, 0x2079}, {0x207F, 0x2089}, {0x2090, 0x209C}, {0x2102, 0x2102}, {0x2107, 0x2107}, + {0x210A, 0x2113}, {0x2115, 0x2115}, {0x2119, 0x211D}, {0x2124, 0x2124}, {0x2126, 0x2126}, + {0x2128, 0x2128}, {0x212A, 0x212D}, {0x212F, 0x2139}, {0x213C, 0x213F}, {0x2145, 0x2149}, + {0x214E, 0x214E}, {0x2150, 0x2189}, {0x2460, 0x249B}, {0x24EA, 0x24FF}, {0x2776, 0x2793}, + {0x2C00, 0x2CE4}, {0x2CEB, 0x2CEE}, {0x2CF2, 0x2CF3}, {0x2CFD, 0x2CFD}, {0x2D00, 0x2D25}, + {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D6F}, {0x2D80, 0x2D96}, + {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, + {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2E2F, 0x2E2F}, {0x3005, 0x3007}, + {0x3021, 0x3029}, {0x3031, 0x3035}, {0x3038, 0x303C}, {0x3041, 0x3096}, {0x309D, 0x309F}, + {0x30A1, 0x30FA}, {0x30FC, 0x30FF}, {0x3105, 0x312F}, {0x3131, 0x318E}, {0x3192, 0x3195}, + {0x31A0, 0x31BF}, {0x31F0, 0x31FF}, {0x3220, 0x3229}, {0x3248, 0x324F}, {0x3251, 0x325F}, + {0x3280, 0x3289}, {0x32B1, 0x32BF}, {0x3400, 0x4DBF}, {0x4E00, 0xA48C}, {0xA4D0, 0xA4FD}, + {0xA500, 0xA60C}, {0xA610, 0xA62B}, {0xA640, 0xA66E}, {0xA67F, 0xA69D}, {0xA6A0, 0xA6EF}, + {0xA717, 0xA71F}, {0xA722, 0xA788}, {0xA78B, 0xA7DC}, {0xA7F1, 0xA801}, {0xA803, 0xA805}, + {0xA807, 0xA80A}, {0xA80C, 0xA822}, {0xA830, 0xA835}, {0xA840, 0xA873}, {0xA882, 0xA8B3}, + {0xA8D0, 0xA8D9}, {0xA8F2, 0xA8F7}, {0xA8FB, 0xA8FB}, {0xA8FD, 0xA8FE}, {0xA900, 0xA925}, + {0xA930, 0xA946}, {0xA960, 0xA97C}, {0xA984, 0xA9B2}, {0xA9CF, 0xA9D9}, {0xA9E0, 0xA9E4}, + {0xA9E6, 0xA9FE}, {0xAA00, 0xAA28}, {0xAA40, 0xAA42}, {0xAA44, 0xAA4B}, {0xAA50, 0xAA59}, + {0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, {0xAA7E, 0xAAAF}, {0xAAB1, 0xAAB1}, {0xAAB5, 0xAAB6}, + {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, {0xAAC2, 0xAAC2}, {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, + {0xAAF2, 0xAAF4}, {0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, + {0xAB28, 0xAB2E}, {0xAB30, 0xAB5A}, {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, {0xABF0, 0xABF9}, + {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xF900, 0xFA6D}, {0xFA70, 0xFAD9}, + {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB1D}, {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, + {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFBB1}, + {0xFBD3, 0xFD3D}, {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, + {0xFE76, 0xFEFC}, {0xFF10, 0xFF19}, {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, {0xFF66, 0xFFBE}, + {0xFFC2, 0xFFC7}, {0xFFCA, 0xFFCF}, {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, {0x10000, 0x1000B}, + {0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, + {0x10080, 0x100FA}, {0x10107, 0x10133}, {0x10140, 0x10178}, {0x1018A, 0x1018B}, {0x10280, 0x1029C}, + {0x102A0, 0x102D0}, {0x102E1, 0x102FB}, {0x10300, 0x10323}, {0x1032D, 0x1034A}, {0x10350, 0x10375}, + {0x10380, 0x1039D}, {0x103A0, 0x103C3}, {0x103C8, 0x103CF}, {0x103D1, 0x103D5}, {0x10400, 0x1049D}, + {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, + {0x10570, 0x1057A}, {0x1057C, 0x1058A}, {0x1058C, 0x10592}, {0x10594, 0x10595}, {0x10597, 0x105A1}, + {0x105A3, 0x105B1}, {0x105B3, 0x105B9}, {0x105BB, 0x105BC}, {0x105C0, 0x105F3}, {0x10600, 0x10736}, + {0x10740, 0x10755}, {0x10760, 0x10767}, {0x10780, 0x10785}, {0x10787, 0x107B0}, {0x107B2, 0x107BA}, + {0x10800, 0x10805}, {0x10808, 0x10808}, {0x1080A, 0x10835}, {0x10837, 0x10838}, {0x1083C, 0x1083C}, + {0x1083F, 0x10855}, {0x10858, 0x10876}, {0x10879, 0x1089E}, {0x108A7, 0x108AF}, {0x108E0, 0x108F2}, + {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, {0x10920, 0x10939}, {0x10940, 0x10959}, {0x10980, 0x109B7}, + {0x109BC, 0x109CF}, {0x109D2, 0x10A00}, {0x10A10, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A35}, + {0x10A40, 0x10A48}, {0x10A60, 0x10A7E}, {0x10A80, 0x10A9F}, {0x10AC0, 0x10AC7}, {0x10AC9, 0x10AE4}, + {0x10AEB, 0x10AEF}, {0x10B00, 0x10B35}, {0x10B40, 0x10B55}, {0x10B58, 0x10B72}, {0x10B78, 0x10B91}, + {0x10BA9, 0x10BAF}, {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, {0x10CFA, 0x10D23}, + {0x10D30, 0x10D39}, {0x10D40, 0x10D65}, {0x10D6F, 0x10D85}, {0x10E60, 0x10E7E}, {0x10E80, 0x10EA9}, + {0x10EB0, 0x10EB1}, {0x10EC2, 0x10EC7}, {0x10F00, 0x10F27}, {0x10F30, 0x10F45}, {0x10F51, 0x10F54}, + {0x10F70, 0x10F81}, {0x10FB0, 0x10FCB}, {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11052, 0x1106F}, + {0x11071, 0x11072}, {0x11075, 0x11075}, {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x110F0, 0x110F9}, + {0x11103, 0x11126}, {0x11136, 0x1113F}, {0x11144, 0x11144}, {0x11147, 0x11147}, {0x11150, 0x11172}, + {0x11176, 0x11176}, {0x11183, 0x111B2}, {0x111C1, 0x111C4}, {0x111D0, 0x111DA}, {0x111DC, 0x111DC}, + {0x111E1, 0x111F4}, {0x11200, 0x11211}, {0x11213, 0x1122B}, {0x1123F, 0x11240}, {0x11280, 0x11286}, + {0x11288, 0x11288}, {0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, + {0x112F0, 0x112F9}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330}, + {0x11332, 0x11333}, {0x11335, 0x11339}, {0x1133D, 0x1133D}, {0x11350, 0x11350}, {0x1135D, 0x11361}, + {0x11380, 0x11389}, {0x1138B, 0x1138B}, {0x1138E, 0x1138E}, {0x11390, 0x113B5}, {0x113B7, 0x113B7}, + {0x113D1, 0x113D1}, {0x113D3, 0x113D3}, {0x11400, 0x11434}, {0x11447, 0x1144A}, {0x11450, 0x11459}, + {0x1145F, 0x11461}, {0x11480, 0x114AF}, {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, {0x114D0, 0x114D9}, + {0x11580, 0x115AE}, {0x115D8, 0x115DB}, {0x11600, 0x1162F}, {0x11644, 0x11644}, {0x11650, 0x11659}, + {0x11680, 0x116AA}, {0x116B8, 0x116B8}, {0x116C0, 0x116C9}, {0x116D0, 0x116E3}, {0x11700, 0x1171A}, + {0x11730, 0x1173B}, {0x11740, 0x11746}, {0x11800, 0x1182B}, {0x118A0, 0x118F2}, {0x118FF, 0x11906}, + {0x11909, 0x11909}, {0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x1192F}, {0x1193F, 0x1193F}, + {0x11941, 0x11941}, {0x11950, 0x11959}, {0x119A0, 0x119A7}, {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, + {0x119E3, 0x119E3}, {0x11A00, 0x11A00}, {0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, + {0x11A5C, 0x11A89}, {0x11A9D, 0x11A9D}, {0x11AB0, 0x11AF8}, {0x11BC0, 0x11BE0}, {0x11BF0, 0x11BF9}, + {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, {0x11C40, 0x11C40}, {0x11C50, 0x11C6C}, {0x11C72, 0x11C8F}, + {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, {0x11D0B, 0x11D30}, {0x11D46, 0x11D46}, {0x11D50, 0x11D59}, + {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11DA0, 0x11DA9}, + {0x11DB0, 0x11DDB}, {0x11DE0, 0x11DE9}, {0x11EE0, 0x11EF2}, {0x11F02, 0x11F02}, {0x11F04, 0x11F10}, + {0x11F12, 0x11F33}, {0x11F50, 0x11F59}, {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FD4}, {0x12000, 0x12399}, + {0x12400, 0x1246E}, {0x12480, 0x12543}, {0x12F90, 0x12FF0}, {0x13000, 0x1342F}, {0x13441, 0x13446}, + {0x13460, 0x143FA}, {0x14400, 0x14646}, {0x16100, 0x1611D}, {0x16130, 0x16139}, {0x16800, 0x16A38}, + {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, {0x16A70, 0x16ABE}, {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, + {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, {0x16B63, 0x16B77}, + {0x16B7D, 0x16B8F}, {0x16D40, 0x16D6C}, {0x16D70, 0x16D79}, {0x16E40, 0x16E96}, {0x16EA0, 0x16EB8}, + {0x16EBB, 0x16ED3}, {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, + {0x16FE3, 0x16FE3}, {0x16FF2, 0x16FF6}, {0x17000, 0x18CD5}, {0x18CFF, 0x18D1E}, {0x18D80, 0x18DF2}, + {0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122}, {0x1B132, 0x1B132}, + {0x1B150, 0x1B152}, {0x1B155, 0x1B155}, {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, + {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1CCF0, 0x1CCF9}, {0x1D2C0, 0x1D2D3}, + {0x1D2E0, 0x1D2F3}, {0x1D360, 0x1D378}, {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, + {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, + {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, + {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, + {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA}, {0x1D6DC, 0x1D6FA}, {0x1D6FC, 0x1D714}, + {0x1D716, 0x1D734}, {0x1D736, 0x1D74E}, {0x1D750, 0x1D76E}, {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, + {0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB}, {0x1D7CE, 0x1D7FF}, {0x1DF00, 0x1DF1E}, {0x1DF25, 0x1DF2A}, + {0x1E030, 0x1E06D}, {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, {0x1E140, 0x1E149}, {0x1E14E, 0x1E14E}, + {0x1E290, 0x1E2AD}, {0x1E2C0, 0x1E2EB}, {0x1E2F0, 0x1E2F9}, {0x1E4D0, 0x1E4EB}, {0x1E4F0, 0x1E4F9}, + {0x1E5D0, 0x1E5ED}, {0x1E5F0, 0x1E5FA}, {0x1E6C0, 0x1E6DE}, {0x1E6E0, 0x1E6E2}, {0x1E6E4, 0x1E6E5}, + {0x1E6E7, 0x1E6ED}, {0x1E6F0, 0x1E6F4}, {0x1E6FE, 0x1E6FF}, {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, + {0x1E7ED, 0x1E7EE}, {0x1E7F0, 0x1E7FE}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8CF}, {0x1E900, 0x1E943}, + {0x1E94B, 0x1E94B}, {0x1E950, 0x1E959}, {0x1EC71, 0x1ECAB}, {0x1ECAD, 0x1ECAF}, {0x1ECB1, 0x1ECB4}, + {0x1ED01, 0x1ED2D}, {0x1ED2F, 0x1ED3D}, {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, + {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, + {0x1EE3B, 0x1EE3B}, {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, + {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, + {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, + {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, + {0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, + {0x1F100, 0x1F10C}, {0x1FBF0, 0x1FBF9}, {0x20000, 0x2A6DF}, {0x2A700, 0x2B81D}, {0x2B820, 0x2CEAD}, + {0x2CEB0, 0x2EBE0}, {0x2EBF0, 0x2EE5D}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, {0x31350, 0x33479}, +}}; + +inline bool isTokenChar(std::uint32_t code_point) +{ + if (code_point < 128) + return (code_point >= 'a' && code_point <= 'z') || (code_point >= 'A' && code_point <= 'Z') + || (code_point >= '0' && code_point <= '9') || code_point == '_'; + std::size_t lo = 0; + std::size_t hi = token_char_ranges.size(); + while (lo < hi) + { + const std::size_t mid = lo + (hi - lo) / 2; + const auto & span = token_char_ranges[mid]; + if (code_point < span.first) + hi = mid; + else if (code_point > span.last) + lo = mid + 1; + else + return true; + } + return false; +} +} // namespace DB::LocalMatchAgainst diff --git a/dbms/src/Functions/registerFunctions.cpp b/dbms/src/Functions/registerFunctions.cpp index cb43ae7760d..7a9193c392c 100644 --- a/dbms/src/Functions/registerFunctions.cpp +++ b/dbms/src/Functions/registerFunctions.cpp @@ -51,6 +51,7 @@ void registerFunctionsRegexpSubstr(FunctionFactory &); void registerFunctionsRegexpReplace(FunctionFactory &); void registerFunctionsGrouping(FunctionFactory &); void registerFunctionsVector(FunctionFactory &); +void registerFunctionsLocalMatchAgainst(FunctionFactory &); void registerFunctions() { @@ -85,6 +86,7 @@ void registerFunctions() registerFunctionsIsIPAddr(factory); registerFunctionsGrouping(factory); registerFunctionsVector(factory); + registerFunctionsLocalMatchAgainst(factory); } } // namespace DB diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp new file mode 100644 index 00000000000..80e02ddbc16 --- /dev/null +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -0,0 +1,2521 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace DB::ErrorCodes +{ +extern const int ILLEGAL_COLUMN; +} + +namespace DB::tests +{ +class TestLocalMatchAgainst : public DB::tests::FunctionTest +{ +}; + +namespace +{ +constexpr std::array exact_route_collations{ + TiDB::ITiDBCollator::UTF8_GENERAL_CI, + TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI, + TiDB::ITiDBCollator::UTF8_UNICODE_CI, + TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI, + TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI, + TiDB::ITiDBCollator::UTF8MB4_BIN, + TiDB::ITiDBCollator::UTF8_BIN, + TiDB::ITiDBCollator::UTF8MB4_0900_BIN, + TiDB::ITiDBCollator::BINARY, + TiDB::ITiDBCollator::LATIN1_BIN, + TiDB::ITiDBCollator::ASCII_BIN}; + +constexpr std::array builtin_stopwords{ + "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", + "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", + "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; + +// Count token comparisons rather than wall time, so a repeated full-column +// verification regression is deterministic even on a loaded CI host. +class CountingMatchCollator final : public TiDB::ITiDBCollator +{ +public: + explicit CountingMatchCollator(bool supported = true) + : ITiDBCollator(UTF8MB4_BIN) + , inner(getCollator(UTF8MB4_BIN)) + { + collator_type = supported ? inner->getCollatorType() : CollatorType::MAX_; + } + + int compare(const char * lhs, size_t lhs_size, const char * rhs, size_t rhs_size) const override + { + ++comparisons; + return inner->compare(lhs, lhs_size, rhs, rhs_size); + } + StringRef convert(const char * s, size_t size, String & buffer, std::vector * lens) const override + { + return inner->convert(s, size, buffer, lens); + } + StringRef sortKeyNoTrim(const char * s, size_t size, String & buffer) const override + { + return inner->sortKeyNoTrim(s, size, buffer); + } + StringRef sortKey(const char * s, size_t size, String & buffer) const override + { + return inner->sortKey(s, size, buffer); + } + std::unique_ptr pattern() const override { return inner->pattern(); } + + mutable size_t comparisons = 0; + +private: + TiDB::TiDBCollatorPtr inner; +}; + +String serializeLocalMatchAgainstBooleanQuery(const tipb::LocalMatchAgainstBooleanQuery & query) +{ + auto encoded_query = query; + if (encoded_query.version() == 0) + encoded_query.set_version(2); // The tests exercise the currently supported Local MATCH semantic protocol. + if (encoded_query.parser() == tipb::LocalMatchAgainstParser::LocalMatchAgainstParserInvalid) + encoded_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + if (encoded_query.stopword_mode() == tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeInvalid) + encoded_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + if (encoded_query.stopword_collation().empty()) + encoded_query.set_stopword_collation("utf8mb4_bin"); + return encoded_query.SerializeAsString(); +} + +tipb::LocalMatchAgainstBooleanQuery makeRequiredLocalMatchQuery( + const String & text, + tipb::LocalMatchAgainstBooleanTermType term_type = tipb::LocalMatchAgainstBooleanTermWord) +{ + tipb::LocalMatchAgainstBooleanQuery query; + query.set_innodb_ft_min_token_size(3); + query.set_innodb_ft_max_token_size(84); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(term_type); + node->set_text(text); + return query; +} + +// Construct the wire expression directly: metadata belongs to Expr.val, not +// SQL children. ColumnRef offsets and signed collations use the real DAG path. +tipb::Expr makeLocalMatchDAGExpr( + const String & search, + const tipb::LocalMatchAgainstBooleanQuery & query, + const NamesAndTypes & inputs, + Int32 collation, + bool null_search = false) +{ + tipb::Expr expr; + expr.set_tp(tipb::ExprType::ScalarFunc); + expr.set_sig(tipb::ScalarFuncSig::LocalMatchAgainstBoolean); + expr.set_val(serializeLocalMatchAgainstBooleanQuery(query)); + auto * result_type = expr.mutable_field_type(); + result_type->set_tp(TiDB::TypeDouble); + result_type->set_collate(-collation); + const bool nullable = null_search + || std::any_of(inputs.begin(), inputs.end(), [](const auto & input) { return input.type->isNullable(); }); + result_type->set_flag(nullable ? 0 : TiDB::ColumnFlagNotNull); + auto search_expr = null_search ? constructNULLLiteralTiExpr() : constructStringLiteralTiExpr(search); + search_expr.mutable_field_type()->set_collate(-collation); + *expr.add_children() = std::move(search_expr); + for (size_t i = 0; i < inputs.size(); ++i) + { + auto column = constructInt64LiteralTiExpr(static_cast(i)); + column.set_tp(tipb::ExprType::ColumnRef); + column.mutable_field_type()->set_tp(TiDB::TypeString); + column.mutable_field_type()->set_flag(inputs[i].type->isNullable() ? 0 : TiDB::ColumnFlagNotNull); + column.mutable_field_type()->set_collate(-collation); + *expr.add_children() = std::move(column); + } + return expr; +} + +std::pair buildLocalMatchDAGActions( + Context & context, + const NamesAndTypes & inputs, + const std::vector & expressions) +{ + DAGExpressionAnalyzer analyzer(inputs, context); + ExpressionActionsChain chain; + auto & step = analyzer.initAndGetLastStep(chain); + Names results; + for (const auto & expr : expressions) + { + results.push_back(analyzer.getActions(expr, step.actions)); + step.required_output.push_back(results.back()); + } + chain.finalize(); + return {step.actions, results}; +} +} // namespace + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGCollationAndMultiColumn) +try +{ + const ColumnsWithTypeAndName columns{ + createColumn>({"CAFE", {}, "cafe", "cafeteria", {}, "cafe"}, "title"), + createColumn>({"", "café", "mysql", "", {}, "MYSQL"}, "body")}; + const NamesAndTypes inputs{{"title", columns[0].type}, {"body", columns[1].type}}; + for (const auto * collation : + {"utf8mb4_bin", "utf8mb4_0900_bin", "utf8mb4_general_ci", "utf8mb4_unicode_ci", "utf8mb4_0900_ai_ci"}) + { + const auto collator = TiDB::ITiDBCollator::getCollator(collation); + ASSERT_NE(collator, nullptr); + const bool binary = std::string_view(collation).ends_with("_bin"); + for (const bool prefix : {false, true}) + { + SCOPED_TRACE(fmt::format("collation={} prefix={}", collation, prefix)); + auto query = makeRequiredLocalMatchQuery( + prefix ? "caf" : "cafe", + prefix ? tipb::LocalMatchAgainstBooleanTermPrefix : tipb::LocalMatchAgainstBooleanTermWord); + auto * prohibited = query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("mysql"); + const auto expr = makeLocalMatchDAGExpr( + prefix ? "+caf* -mysql" : "+cafe -mysql", + query, + inputs, + collator->getCollatorId()); + ASSERT_EQ(expr.children_size(), 3); // Query + two MATCH columns; no metadata SQL child. + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); + Block block(columns); + actions->execute(block); + const auto expected = prefix + ? (binary ? createColumn>({0, 1, 0, 1, 0, 1}) + : createColumn>({1, 1, 0, 1, 0, 0})) + : (binary ? createColumn>({0, 0, 0, 0, 0, 1}) + : createColumn>({1, 1, 0, 0, 0, 0})); + ASSERT_COLUMN_EQ(expected, block.getByName(names[0])); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGMetadataIsolation) +try +{ + const auto documents + = createColumn({"the", "the 中文", "中文", "中文词", "中文测试", "", "中 文", "foo", "thé"}, "body"); + const NamesAndTypes inputs{{"body", documents.type}}; + const auto binary = TiDB::ITiDBCollator::UTF8MB4_BIN; + std::vector expressions; + ColumnsWithTypeAndName expected; + const auto append = [&](const String & search, + const tipb::LocalMatchAgainstBooleanQuery & query, + Int32 collation, + const ColumnWithTypeAndName & result) { + expressions.push_back(makeLocalMatchDAGExpr(search, query, inputs, collation)); + expected.push_back(result); + }; + auto query = makeRequiredLocalMatchQuery("the"); + append("+the", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + append("+the", query, binary, createColumn({1, 1, 0, 0, 0, 0, 0, 0, 0})); + + query = makeRequiredLocalMatchQuery("中文"); + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + query.set_ngram_token_size(2); + append("+中文", query, binary, createColumn({0, 1, 1, 1, 1, 0, 0, 0, 0})); + query.set_ngram_token_size(3); + append("+中文", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + + query = makeRequiredLocalMatchQuery("foo"); + append("+foo", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 1, 0})); + query.set_innodb_ft_min_token_size(4); + append("+foo", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + + query = makeRequiredLocalMatchQuery("thé"); + query.set_stopword_collation("utf8mb4_bin"); + append("+thé", query, TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 1})); + query.set_stopword_collation("utf8mb4_general_ci"); + append("+thé", query, TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, expressions); + ASSERT_EQ(names.size(), expected.size()); + for (size_t i = 0; i < names.size(); i += 2) + ASSERT_NE(names[i], names[i + 1]) << "Different metadata must not alias the same action"; + Block block({documents}); + actions->execute(block); + for (size_t i = 0; i < names.size(); ++i) + { + SCOPED_TRACE(i); + ASSERT_COLUMN_EQ(expected[i], block.getByName(names[i])); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGReusesActionsAcrossBlocks) +try +{ + struct BlockCase + { + ColumnsWithTypeAndName columns; + ColumnWithTypeAndName expected; + }; + for (const bool ngram : {false, true}) + { + SCOPED_TRACE(ngram); + auto query = makeRequiredLocalMatchQuery( + ngram ? "中文测试" : "quick the fox", + tipb::LocalMatchAgainstBooleanTermPhrase); + if (ngram) + { + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(2); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + } + auto * prohibited = query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("mysql"); + const auto first = ngram + ? BlockCase{{createColumn>({"中文测试", "中文", "中文测试", "", {}}, "title"), createColumn>({{}, "测试", "mysql", "中文测试", {}}, "body")}, createColumn>({1, 0, 0, 1, 0})} + : BlockCase{ + {createColumn>( + {"QUICK the fox", "quick the fox", "quick", {}, "quick the fox"}, + "title"), + createColumn>({{}, "mysql", "the fox", {}, "MYSQL"}, "body")}, + createColumn>({1, 0, 0, 0, 0})}; + const auto second = ngram + ? BlockCase{{createColumn>({"中文 test", {}, "中文测试", "中 文测试"}, "title"), createColumn>({"", {}, "", "中文测试"}, "body")}, createColumn>({0, 0, 1, 1})} + : BlockCase{ + {createColumn>( + {"slow the fox", "quick the fox", {}, "quick fox", "quick the fox"}, + "title"), + createColumn>({"", {}, "quick the fox", "", {}}, "body")}, + createColumn>({0, 1, 1, 0, 1})}; + const BlockCase nulls{ + {createColumn>({{}, {}}, "title"), createColumn>({{}, {}}, "body")}, + createColumn>({0, 0})}; + const BlockCase empty{ + {createColumn>({}, "title"), createColumn>({}, "body")}, + createColumn>({})}; + const NamesAndTypes inputs{{"title", first.columns[0].type}, {"body", first.columns[1].type}}; + const auto expr = makeLocalMatchDAGExpr( + ngram ? "+\"中文测试\" -mysql" : "+\"quick the fox\" -mysql", + query, + inputs, + TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); + // The same compiled actions/function sees changing row counts, empty + // input, and old input again. No previous result or NULL map may leak. + for (size_t pass = 0; pass < 2; ++pass) + for (const auto * batch : {&first, &second, &nulls, &empty, &first}) + { + SCOPED_TRACE(fmt::format("pass={} rows={}", pass, batch->expected.column->size())); + Block block(batch->columns); + actions->execute(block); + ASSERT_COLUMN_EQ(batch->expected, block.getByName(names[0])); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGColumnWrappersAndEmptyBlocks) +try +{ + struct ColumnCase + { + ColumnWithTypeAndName document; + ColumnWithTypeAndName expected; + }; + const std::vector cases{ + {createColumn({"quick", "", "quick fox", "slow"}, "body"), createColumn({1, 0, 1, 0})}, + {createColumn>({"quick", {}, "", "slow"}, "body"), + createColumn>({1, 0, 0, 0})}, + {createColumn>({{}, {}, {}, {}}, "body"), createColumn>({0, 0, 0, 0})}, + {createConstColumn(4, "quick", "body"), createColumn({1, 1, 1, 1})}, + {createConstColumn>(4, "quick", "body"), createColumn>({1, 1, 1, 1})}, + {createConstColumn>(4, {}, "body"), createColumn>({0, 0, 0, 0})}, + {createConstColumn(4, "", "body"), createColumn({0, 0, 0, 0})}}; + const auto query = makeRequiredLocalMatchQuery("quick"); + for (size_t i = 0; i < cases.size(); ++i) + for (const bool null_search : {false, true}) + { + SCOPED_TRACE(fmt::format("case={} null_search={}", i, null_search)); + const auto & column_case = cases[i]; + const NamesAndTypes inputs{{"body", column_case.document.type}}; + const auto expr + = makeLocalMatchDAGExpr("+quick", query, inputs, TiDB::ITiDBCollator::UTF8MB4_BIN, null_search); + ASSERT_EQ(expr.children(0).field_type().tp(), null_search ? TiDB::TypeNull : TiDB::TypeString); + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); + const auto expected = null_search ? createColumn>({0, 0, 0, 0}) : column_case.expected; + for (const size_t rows : {4U, 0U, 4U}) + { + SCOPED_TRACE(rows); + auto document = column_case.document; + document.column = document.column->cloneResized(rows); + auto expected_rows = expected; + expected_rows.column = expected_rows.column->cloneResized(rows); + Block block({document}); + actions->execute(block); + ASSERT_COLUMN_EQ(expected_rows, block.getByName(names[0])); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanMultiColumnNullable) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+quick"), + createColumn>({{}, "quick fox", {}, "slow fox"}), + createColumn>({"quick fox", {}, "slow fox", {}}), + createConstColumn(4, metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, FilteredPhraseVerificationComparisonGrowth) +try +{ + tipb::LocalMatchAgainstBooleanQuery query; + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + node->set_text("foo a bar"); + size_t small_comparisons = 0; + for (const size_t repeats : {32, 256}) + { + String document; + for (size_t i = 0; i < repeats; ++i) + document += "foo x bar "; + CountingMatchCollator collator; + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+\"foo a bar\""), + createColumn({document}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}, + &collator, + true)); + if (repeats == 32) + small_comparisons = collator.comparisons; + else + { + EXPECT_GT(small_comparisons, 0); + // Eight times the input may add proportional token work, not + // 64 times the work from rechecking every failed anchor. + EXPECT_LE(collator.comparisons, small_comparisons * 12); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, FilteredPhraseVerificationCacheScope) +try +{ + String miss; + for (size_t i = 0; i < 64; ++i) + miss += "foo x bar "; + for (const auto occur : + {tipb::LocalMatchAgainstBooleanOccurMust, + tipb::LocalMatchAgainstBooleanOccurShould, + tipb::LocalMatchAgainstBooleanOccurMustNot}) + { + tipb::LocalMatchAgainstBooleanQuery query; + if (occur != tipb::LocalMatchAgainstBooleanOccurShould) + { + auto * anchor = query.add_nodes(); + anchor->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + anchor->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + anchor->set_text("foo"); + } + auto * phrase = query.add_nodes(); + phrase->set_occur(occur); + phrase->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + phrase->set_text("foo a bar"); + const Float64 hit = occur == tipb::LocalMatchAgainstBooleanOccurMustNot ? 0 : 1; + const Float64 no_phrase = occur == tipb::LocalMatchAgainstBooleanOccurMustNot ? 1 : 0; + ASSERT_COLUMN_EQ( + createColumn>({hit, hit, no_phrase, hit, no_phrase}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(5, "foo \"foo a bar\""), + createColumn>({miss, "foo a bar", miss, {}, miss}), + createColumn>({"foo a bar", miss, {}, "foo a bar", miss}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } + + tipb::LocalMatchAgainstBooleanQuery query; + auto * required = query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + required->set_text("foo x bar"); + auto * excluded = query.add_nodes(); + excluded->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + excluded->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + excluded->set_text("foo a bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+\"foo x bar\" -\"foo a bar\""), + createColumn({miss, "foo x bar foo a bar", miss}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + + // NGRAM size 3 removes "the" but retains "foo" and "zoo". + // STANDARD exercises the same cache with a filtered stopword. + for (const bool ngram : {false, true}) + { + tipb::LocalMatchAgainstBooleanQuery filtered; + filtered.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + filtered.set_ngram_token_size(3); + auto * node = filtered.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + node->set_text("foo the zoo"); + String repetitive_miss; + for (size_t i = 0; i < 64; ++i) + repetitive_miss += "foo xyz zoo "; + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+\"foo the zoo\""), + createColumn>({repetitive_miss, "foo the zoo", repetitive_miss, {}}), + createColumn>({"foo the zoo", repetitive_miss, {}, "foo the zoo"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(filtered))}, + nullptr, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanMySQLCharacterBoundaries) +try +{ + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(2); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + node->set_text("a"); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1, 1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(6, "+a*"), + createColumn({"a,b", "a,b", "a🙃b", "a𞤀b", "a𝟙b", String("ab") + char(0xff) + "cd"}), + createConstColumn(6, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + + query.set_parser(tipb::LocalMatchAgainstParserStandard); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("foo"); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+foo"), + createColumn({"foo𞤀bar", "foo𝟙bar", "foo🙃bar", "foobar"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanFilteredPhraseVerification) +try +{ + for (const bool ngram : {false, true}) + for (const bool stopwords : {false, true}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + query.set_ngram_token_size(2); + query.set_stopword_mode( + stopwords ? tipb::LocalMatchAgainstStopwordModeBuiltin : tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + node->set_text("quick x fox"); + ASSERT_COLUMN_EQ( + createColumn({ngram ? 0.0 : 1.0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+\"quick x fox\""), + createColumn({"quick x fox", "quick the fox", "quick xx fox"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + node->set_text("foo bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 1, ngram ? 0.0 : 1.0, ngram ? 0.0 : 1.0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(5, "+\"foo bar\""), + createColumn({"foo bar", "foo,bar", "foo,bar", "foo🙃bar", "foobar"}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } + + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(3); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeBuiltin); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + for (const String & text : {String("quick the fox"), String("quick a fox")}) + { + node->set_text(text); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, text), + createColumn({"quick fox", "quick x fox", "fox quick"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } + for (const String & text : {String("foo a zoo"), String("foo zoo")}) + { + node->set_text(text); + const Float64 expected = text == "foo zoo" ? 1.0 : 0.0; + ASSERT_COLUMN_EQ( + createColumn({expected, expected, expected, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, text), + createColumn({"foo zoo", "foo a zoo", "foo x zoo", "zoo foo"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanFilteredRequiredClause) +try +{ + for (const bool ngram : {false, true}) + for (const String & term : {String("the"), String("x")}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + query.set_ngram_token_size(3); + auto * required = query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text(term); + auto * optional = query.add_nodes(); + optional->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + optional->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + optional->set_text("quick"); + const auto columns = ColumnsWithTypeAndName{ + createConstColumn(3, "+" + term + " quick"), + createColumn>({"the x quick", String(4096, 'x'), {}}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}; + ASSERT_COLUMN_EQ( + createColumn>({0, 0, 0}), + executeFunction("local_match_against_boolean", columns, nullptr, true)); + + // NULL AGAINST is an empty search even for an impossible query. + auto null_search = columns; + null_search[0] = createConstColumn>(3, {}); + ASSERT_COLUMN_EQ( + createColumn>({0, 0, 0}), + executeFunction("local_match_against_boolean", null_search, nullptr, true)); + + // Envelope validation must still reject an unknown version. + query.set_version(999); + EXPECT_THROW( + executeFunction( + "local_match_against_boolean", + {columns[0], + columns[1], + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true), + Exception); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanProtocolQuery) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + auto * prohibited = boolean_query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("slow"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+quick -slow"), + createColumn({"quick brown", "slow fox", "brown fox"}), + createConstColumn(3, metadata)}, + nullptr, + true)); + + tipb::LocalMatchAgainstBooleanQuery prohibited_query; + auto * prohibited_only = prohibited_query.add_nodes(); + prohibited_only->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited_only->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited_only->set_text("slow"); + const String prohibited_metadata = serializeLocalMatchAgainstBooleanQuery(prohibited_query); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "-slow"), + createColumn({"quick brown", "slow fox", "brown fox"}), + createConstColumn(3, prohibited_metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanPhrasePositionsWithStopwordGaps) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + required->set_text("quick the fox"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+\"quick the fox\""), + createColumn({"quick the fox", "quick fox", "quick slow the fox", "quick the fox quick the fox"}), + createConstColumn(4, metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanConstantDocumentColumn) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+quick"), + createConstColumn(3, "quick fox"), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(boolean_query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanMultiColumnStateAndPhraseBoundary) +try +{ + const auto make_query = [](tipb::LocalMatchAgainstBooleanTermType term_type) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(term_type); + required->set_text("quick fox"); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+quick +fox"), + createColumn({"quick", "quick"}), + createColumn({"fox", "slow"}), + createConstColumn(2, make_query(tipb::LocalMatchAgainstBooleanTermWord))}, + nullptr, + true)); + + ASSERT_COLUMN_EQ( + createColumn({0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+\"quick fox\""), + createColumn({"quick", "quick fox"}), + createColumn({"fox", "slow"}), + createConstColumn(2, make_query(tipb::LocalMatchAgainstBooleanTermPhrase))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDecidesPositiveMatchEarly) +try +{ + tipb::LocalMatchAgainstBooleanQuery required_query; + auto * required_quick = required_query.add_nodes(); + required_quick->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_quick->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_quick->set_text("quick"); + auto * required_fox = required_query.add_nodes(); + required_fox->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_fox->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_fox->set_text("fox"); + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+quick +fox"), + createColumn({"quick", "quick"}), + createColumn({"fox", "quiet"}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(required_query))}, + nullptr, + true)); + + tipb::LocalMatchAgainstBooleanQuery optional_query; + auto * optional_quick = optional_query.add_nodes(); + optional_quick->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + optional_quick->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + optional_quick->set_text("quick"); + auto * optional_fox = optional_query.add_nodes(); + optional_fox->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + optional_fox->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + optional_fox->set_text("fox"); + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "quick fox"), + createColumn({"quick", "quiet"}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(optional_query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanStillChecksProhibitedTermsAfterPositiveMatch) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required_quick = boolean_query.add_nodes(); + required_quick->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_quick->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_quick->set_text("quick"); + auto * required_fox = boolean_query.add_nodes(); + required_fox->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_fox->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_fox->set_text("fox"); + auto * prohibited_slow = boolean_query.add_nodes(); + prohibited_slow->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited_slow->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited_slow->set_text("slow"); + + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+quick +fox -slow"), + createColumn({"quick", "quick", "quick"}), + createColumn({"fox", "fox", "quiet"}), + createColumn({"calm", "slow", "slow"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(boolean_query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanRejectsInvalidProtocol) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_version(1); // Version 1 must not be silently reinterpreted as version 2. + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + boolean_query.set_stopword_collation("utf8mb4_bin"); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + + ASSERT_THROW( + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, boolean_query.SerializeAsString())}, + nullptr, + true), + Exception); + + boolean_query.set_version(2); + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserInvalid); + ASSERT_THROW( + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, boolean_query.SerializeAsString())}, + nullptr, + true), + Exception); + + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeInvalid); + ASSERT_THROW( + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, boolean_query.SerializeAsString())}, + nullptr, + true), + Exception); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanRejectsInvalidNodeEnums) +try +{ + enum class InvalidEnumEncoding + { + Missing, + Invalid, + Unknown, + }; + for (const bool invalid_occur : {false, true}) + for (const auto encoding : + {InvalidEnumEncoding::Missing, InvalidEnumEncoding::Invalid, InvalidEnumEncoding::Unknown}) + { + auto query = makeRequiredLocalMatchQuery("quick"); + query.set_version(2); + query.set_parser(tipb::LocalMatchAgainstParserStandard); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + // Put the malformed node after a valid one: all nodes must be checked. + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("fox"); + if (invalid_occur) + { + node->clear_occur(); + if (encoding == InvalidEnumEncoding::Invalid) + node->set_occur(tipb::LocalMatchAgainstBooleanOccurInvalid); + } + else + { + node->clear_term_type(); + if (encoding == InvalidEnumEncoding::Invalid) + node->set_term_type(tipb::LocalMatchAgainstBooleanTermInvalid); + } + // Encode unknown values on the wire; generated enum setters may + // assert rather than allow an out-of-range value in debug builds. + if (encoding == InvalidEnumEncoding::Unknown) + ASSERT_TRUE(node->MergeFromString(invalid_occur ? "\x08\x7f" : "\x10\x7f")); + + for (const bool empty_analyzer : {false, true}) + for (const bool null_search : {false, true}) + { + SCOPED_TRACE(fmt::format( + "invalid_occur={} encoding={} empty_analyzer={} null_search={}", + invalid_occur, + static_cast(encoding), + empty_analyzer, + null_search)); + query.set_innodb_ft_max_token_size(empty_analyzer ? 0 : 84); + auto search = createConstColumn(1, "+quick"); + if (null_search) + search = createOnlyNullColumnConst(1); + try + { + executeFunction( + "local_match_against_boolean", + {search, + createColumn({"quick fox"}), + createConstColumn(1, query.SerializeAsString())}, + nullptr, + true); + FAIL() << "Invalid Local MATCH node metadata must be rejected"; + } + catch (const Exception & error) + { + ASSERT_EQ(ErrorCodes::ILLEGAL_COLUMN, error.code()); + } + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanStandardAnalyzerProtocolSettings) +try +{ + const auto make_metadata = [](const String & term, + UInt32 min_token_size, + UInt32 max_token_size, + bool enable_stopword, + const String & stopword_collation) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + boolean_query.set_stopword_collation(stopword_collation); + boolean_query.set_innodb_ft_min_token_size(min_token_size); + boolean_query.set_innodb_ft_max_token_size(max_token_size); + boolean_query.set_stopword_mode( + enable_stopword ? tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin + : tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text(term); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + const auto short_term_metadata = make_metadata("go", 1, 84, true, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+go"), + createColumn({"go", "good"}), + createConstColumn(2, short_term_metadata)}, + nullptr, + true)); + + const auto stopword_metadata = make_metadata("the", 1, 84, false, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+the"), + createColumn({"the", "there"}), + createConstColumn(2, stopword_metadata)}, + nullptr, + true)); + + const auto collated_stopword_metadata = make_metadata("thé", 3, 84, true, "utf8mb4_general_ci"); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+thé"), + createColumn({"thé"}), + createConstColumn(1, collated_stopword_metadata)}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); + + const auto binary_server_metadata = make_metadata("The", 3, 84, true, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+The"), + createColumn({"The"}), + createConstColumn(1, binary_server_metadata)}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); + + const auto max_size_metadata = make_metadata("extraordinary", 1, 10, false, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+extraordinary"), + createColumn({"extraordinary"}), + createConstColumn(1, max_size_metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, AnalyzerDefaultsAndResultFinalization) +try +{ + struct ConfigCase + { + UInt32 min_size; + UInt32 max_size; + InferredDataVector expected; + }; + for (const auto & config : + {ConfigCase{0, 0, {1, 0, 0, 0}}, ConfigCase{0, 4, {0, 1, 0, 0}}, ConfigCase{5, 4, {0, 0, 0, 0}}}) + { + SCOPED_TRACE(fmt::format("min={} max={}", config.min_size, config.max_size)); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_innodb_ft_min_token_size(config.min_size); + query.set_innodb_ft_max_token_size(config.max_size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + for (const auto * term : {"quick", "a"}) + { + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(term); + } + const auto metadata = createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query)); + ASSERT_COLUMN_EQ( + createColumn(config.expected), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "quick a"), createColumn({"quick", "a", "", "none"}), metadata}, + nullptr, + true)); + InferredDataVector> nullable_expected; + for (auto value : config.expected) + nullable_expected.emplace_back(value); + ASSERT_COLUMN_EQ( + createColumn>(nullable_expected), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "quick a"), + createColumn>({"quick", "a", {}, ""}), + metadata}, + nullptr, + true)); + // NULL search must still bypass analyzer setup and return non-NULL + // zeros, even when that setup would reject the stopword collation. + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeBuiltin); + query.set_stopword_collation("unsupported_stopword_collation"); + ASSERT_COLUMN_EQ( + createColumn>({0, 0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn>(4, {}), + createColumn({"quick", "a", "", "none"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanNgramProtocolQuery) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + boolean_query.set_ngram_token_size(2); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("数据库"); + auto * prohibited = boolean_query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("mysql"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+数据库 -mysql"), + createColumn({"数据库系统", "MySQL 数据库", "数据库", "数据科学"}), + createConstColumn(4, metadata)}, + nullptr, + true)); + + const auto ci_collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + tipb::LocalMatchAgainstBooleanQuery case_query; + case_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + case_query.set_ngram_token_size(2); + auto * case_term = case_query.add_nodes(); + case_term->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + case_term->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + case_term->set_text("mysql"); + const String case_metadata = serializeLocalMatchAgainstBooleanQuery(case_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+mysql"), + createColumn({"MySQL", "PostgreSQL"}), + createConstColumn(2, case_metadata)}, + ci_collator, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanNgramStopwordSettings) +try +{ + const auto make_metadata = [](UInt32 token_size, bool enable_stopword) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + boolean_query.set_ngram_token_size(token_size); + boolean_query.set_stopword_mode( + enable_stopword ? tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin + : tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("the"); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + const auto query = createConstColumn(3, "+the"); + const auto documents = createColumn({"the", "other", "there"}); + + // A stopword longer than ngram_token_size is ignored by the NGRAM parser. + const auto size_two_metadata = createConstColumn(3, make_metadata(2, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, size_two_metadata}, nullptr, true)); + + // At size 3, the gram "the" is removed from both query and document. + const auto enabled_metadata = createConstColumn(3, make_metadata(3, true)); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 0}), + executeFunction("local_match_against_boolean", {query, documents, enabled_metadata}, nullptr, true)); + + tipb::LocalMatchAgainstBooleanQuery filtered_prefix; + filtered_prefix.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + filtered_prefix.set_ngram_token_size(2); + filtered_prefix.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + auto * prefix_node = filtered_prefix.add_nodes(); + prefix_node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + prefix_node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + prefix_node->set_text("caf"); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+caf*"), + createColumn({"cafe"}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(filtered_prefix))}, + nullptr, + true)); + + // Explicit OFF keeps that gram and restores the matches. + const auto disabled_metadata = createConstColumn(3, make_metadata(3, false)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, disabled_metadata}, nullptr, true)); + + // Stopword matching follows collation_server independently from the + // MATCH column collation. + tipb::LocalMatchAgainstBooleanQuery collated_query; + collated_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + collated_query.set_ngram_token_size(2); + collated_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + collated_query.set_stopword_collation("utf8mb4_general_ci"); + auto * collated_required = collated_query.add_nodes(); + collated_required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + collated_required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + collated_required->set_text("xá"); + const auto collated_metadata = createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(collated_query)); + const auto document = createColumn({"xá"}); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+xá"), document, collated_metadata}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); + + // The MATCH column collation remains accent-insensitive, but the binary + // server collation does not consider "á" to contain stopword "a". + collated_query.set_stopword_collation("utf8mb4_bin"); + const auto binary_stopword_metadata + = createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(collated_query)); + ASSERT_COLUMN_EQ( + createColumn({1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+xá"), document, binary_stopword_metadata}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanNgramPrefix) +try +{ + const auto make_metadata = [](UInt32 token_size, const String & prefix) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + boolean_query.set_ngram_token_size(token_size); + boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled); + auto * node = boolean_query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + node->set_text(prefix); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + const auto query = createConstColumn(4, "+caf*"); + const auto documents = createColumn({"CAFE", "café", "cafe", "decaf"}); + const auto metadata = createConstColumn(4, make_metadata(2, "caf")); + const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, metadata}, utf8mb4_bin, true)); + + const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, metadata}, utf8mb4_general_ci, true)); + + const auto short_query = createConstColumn(4, "+c*"); + const auto short_metadata = createConstColumn(4, make_metadata(2, "c")); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {short_query, documents, short_metadata}, + utf8mb4_general_ci, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanCollationMatrix) +try +{ + const auto query = createConstColumn(3, "+cafe"); + const auto documents = createColumn({"CAFE", "café", "cafe"}); + const auto prefix_query = createConstColumn(3, "+caf*"); + const auto make_metadata = [](tipb::LocalMatchAgainstBooleanTermType term_type, const String & term) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + auto * node = boolean_query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(term_type); + node->set_text(term); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + const auto word_metadata + = createConstColumn(3, make_metadata(tipb::LocalMatchAgainstBooleanTermWord, "cafe")); + const auto prefix_metadata + = createConstColumn(3, make_metadata(tipb::LocalMatchAgainstBooleanTermPrefix, "caf")); + + const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_bin, true)); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1}), + executeFunction("local_match_against_boolean", {prefix_query, documents, prefix_metadata}, utf8mb4_bin, true)); + + const auto utf8mb4_0900_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_0900_bin, true)); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1}), + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_0900_bin, + true)); + + const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_general_ci, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_general_ci, + true)); + + const auto utf8mb4_unicode_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_unicode_ci, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_unicode_ci, + true)); + + const auto utf8mb4_0900_ai_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_0900_ai_ci, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_0900_ai_ci, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanSplitWordModifiers) +try +{ + const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + const auto documents = createColumn({"foo only", "bar only", "foo bar", "baz foo", "baz bar", "baz qux"}); + for (const auto occur : + {tipb::LocalMatchAgainstBooleanOccurShould, + tipb::LocalMatchAgainstBooleanOccurMust, + tipb::LocalMatchAgainstBooleanOccurMustNot}) + { + tipb::LocalMatchAgainstBooleanQuery query; + auto * node = query.add_nodes(); + node->set_occur(occur); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("foo.bar"); + if (occur == tipb::LocalMatchAgainstBooleanOccurMustNot) + { + auto * positive = query.add_nodes(); + positive->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + positive->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + positive->set_text("baz"); + } + const auto expected = occur == tipb::LocalMatchAgainstBooleanOccurShould + ? createColumn({1, 1, 1, 1, 1, 0}) + : occur == tipb::LocalMatchAgainstBooleanOccurMust ? createColumn({0, 0, 1, 0, 0, 0}) + : createColumn({0, 0, 0, 0, 0, 1}); + const auto metadata = createConstColumn(6, serializeLocalMatchAgainstBooleanQuery(query)); + ASSERT_COLUMN_EQ( + expected, + executeFunction( + "local_match_against_boolean", + {createConstColumn(6, "foo.bar"), documents, metadata}, + collator, + true)); + } + // The intersection for a required split word may span MATCH columns. + tipb::LocalMatchAgainstBooleanQuery query; + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("foo.bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+foo.bar"), + createColumn({"foo", "foo"}), + createColumn({"bar", "qux"}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, BuiltinStopwordLookupMatchesComparisons) +try +{ + // Independent reference: retain the old compare-based stopword rule, + // including NGRAM's substring and source-code-point-length restriction. + const auto & stopwords = builtin_stopwords; + std::vector candidates; + for (const auto word : stopwords) + candidates.emplace_back(word); + for (const String & word : + {"ABOUT", "The", "THE", "WWW", "WITH", "ábout", "thé", "tĥe", "with", + "www", "aß", "ß", "xá", "áx", "xA", "Ax", "xáy", "aaa", + "AAA", "xy", "xyz", "foo", "数据库", "aboutx", "about", "ab\u0640out", "ab\uFC5Eout"}) + candidates.push_back(word); + for (const auto collation : + {"binary", + "ascii_bin", + "latin1_bin", + "utf8_bin", + "utf8mb4_bin", + "utf8mb4_0900_bin", + "utf8_general_ci", + "utf8mb4_general_ci", + "utf8_unicode_ci", + "utf8mb4_unicode_ci", + "utf8mb4_0900_ai_ci"}) + { + const auto stopword_collator = TiDB::ITiDBCollator::getCollator(collation); + ASSERT_NE(stopword_collator, nullptr); + // Check the key-equality premise directly, including padding and + // combining marks which the STANDARD tokenizer may split away. + String key_buffer; + auto key_copy = [&](const String & text) { + const auto key = stopword_collator->sortKeyFastPath(text.data(), text.size(), key_buffer); + return String(key.data, key.size); + }; + for (const auto word : stopwords) + { + const String source(word); + const auto source_key = key_copy(source); + for (const auto & candidate : candidates) + for (const String & suffix : {"", " ", " ", "\u0301"}) + { + const String text = candidate + suffix; + EXPECT_EQ( + source_key == key_copy(text), + stopword_collator->compare(source.data(), source.size(), text.data(), text.size()) == 0) + << collation << " source=" << source << " candidate=" << text; + } + } + auto equals_stopword = [&](std::string_view token, size_t length, bool restrict_length) { + return std::any_of(stopwords.begin(), stopwords.end(), [&](const auto word) { + return (!restrict_length || length == word.size()) + && stopword_collator->compare(token.data(), token.size(), word.data(), word.size()) == 0; + }); + }; + for (const auto size : {0U, 1U, 2U, 3U}) + for (const auto & candidate : candidates) + { + SCOPED_TRACE(fmt::format("stopword_collation={} size={} candidate={}", collation, size, candidate)); + std::vector boundaries{0}; + for (size_t offset = 0; offset < candidate.size();) + { + offset += UTF8::utf8Decode(candidate.data() + offset, candidate.size() - offset).second; + boundaries.push_back(offset); + } + bool retained = size == 0 && !equals_stopword(candidate, boundaries.size() - 1, false); + if (size != 0) + for (size_t start = 0; start + size < boundaries.size(); ++start) + { + bool filtered = false; + for (size_t begin = start; begin < start + size; ++begin) + for (size_t end = begin + 1; end <= start + size; ++end) + filtered = filtered + || equals_stopword( + std::string_view(candidate).substr( + boundaries[begin], + boundaries[end] - boundaries[begin]), + end - begin, + true); + retained = retained || !filtered; + } + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser( + size == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_innodb_ft_min_token_size(1); + query.set_innodb_ft_max_token_size(84); + query.set_stopword_collation(collation); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(candidate); + // Repeated rows exercise key-buffer reuse. Column and server + // collations are deliberately independent. + for (const auto match_collation : {"utf8mb4_bin", "utf8mb4_general_ci"}) + ASSERT_COLUMN_EQ( + createColumn({retained ? 1.0 : 0.0, retained ? 1.0 : 0.0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, candidate), + createColumn({candidate, candidate}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(query))}, + TiDB::ITiDBCollator::getCollator(match_collation), + true)); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, NgramSlidingStopwordWindows) +try +{ + InferredDataVector> documents{ + "", + "z", + "zzaz", + "zzazzzzzzzzzzz", + "zthe", + "zzthezz", + "zzthezzzzzzzzzz", + "zaboutzz", + "zaboutzaboutzzzzzz", + "zzazzizzz", + "za zzzzzzzzzzz", + "zzz!zzz", + "zThezz", + "zTHEzz", + "zthézz", + "ztĥezz", + "zwithzz", + "zwwwzz", + "zázz", + "zAzz", + "zßzz", + "z\u0301azz", + "z🙃a🙃zz", + "z数据azzz", + "ZTHEZZ", + String("za\0zzzzzzzzzz", 13), + String("zthe\xffzzzzzzzzzz"), + String("\xffzzzzzzzzzz"), + "zzazzazzzzzzzzzzzzz", + "zaboutzzzzzzzzzz", + "z", + {}, + "zzzzzzzzzzzz"}; + // Deterministic mixed runs, ending candidates and repeated rows. The + // reference below never uses the optimized lookup or sliding state. + UInt32 seed = 70485; + const std::array pieces{"z", "a", "i", "the", "about", "thé", "🙃", "!"}; + for (size_t row = 0; row < 24; ++row) + { + String text; + for (size_t i = 0; i < 12; ++i) + { + seed = seed * 1664525 + 1013904223; + text += pieces[(seed >> 16) % pieces.size()]; + } + documents.emplace_back(std::move(text)); + } + + for (const auto stopword_collation : + {"binary", + "ascii_bin", + "latin1_bin", + "utf8_bin", + "utf8mb4_bin", + "utf8mb4_0900_bin", + "utf8_general_ci", + "utf8mb4_general_ci", + "utf8_unicode_ci", + "utf8mb4_unicode_ci", + "utf8mb4_0900_ai_ci"}) + for (const auto match_collation : {"utf8mb4_bin", "utf8mb4_general_ci"}) + for (size_t size = 1; size <= 10; ++size) + { + SCOPED_TRACE(fmt::format("stopwords={} match={} size={}", stopword_collation, match_collation, size)); + const auto stopword_collator = TiDB::ITiDBCollator::getCollator(stopword_collation); + const auto match_collator = TiDB::ITiDBCollator::getCollator(match_collation); + auto prefix = match_collator->pattern(); + prefix->compile("z%", '\\'); + InferredDataVector> expected; + for (const auto & document : documents) + { + bool retained = false; + if (document) + { + const String & text = *document; + size_t run_start = 0; + std::vector boundaries{0}; + auto check_run = [&] { + for (size_t start = 0; start + size < boundaries.size() && !retained; ++start) + { + bool filtered = false; + for (size_t begin = start; begin < start + size && !filtered; ++begin) + for (size_t end = begin + 1; end <= start + size && !filtered; ++end) + { + const auto candidate = std::string_view(text).substr( + run_start + boundaries[begin], + boundaries[end] - boundaries[begin]); + filtered = std::any_of( + builtin_stopwords.begin(), + builtin_stopwords.end(), + [&](auto word) { + return word.size() == end - begin + && stopword_collator->compare( + candidate.data(), + candidate.size(), + word.data(), + word.size()) + == 0; + }); + } + const auto gram = std::string_view(text).substr( + run_start + boundaries[start], + boundaries[start + size] - boundaries[start]); + retained = !filtered && prefix->match(gram.data(), gram.size()); + } + }; + for (size_t offset = 0; offset < text.size();) + { + const auto byte = static_cast(text[offset]); + const auto decoded = UTF8::utf8Decode(text.data() + offset, text.size() - offset); + if (decoded.second == 0 || decoded.first == UTF8::UTF8_Error + || decoded.second > text.size() - offset || (byte >= 0x80 && decoded.second == 1)) + break; + if (byte < 0x80 && !LocalMatchAgainst::isTokenChar(byte)) + { + check_run(); + run_start = ++offset; + boundaries.assign(1, 0); + } + else + { + offset += decoded.second; + boundaries.push_back(offset - run_start); + } + } + check_run(); + } + expected.emplace_back(retained ? 1.0 : 0.0); + } + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_collation(stopword_collation); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + node->set_text("z"); + ASSERT_COLUMN_EQ( + createColumn>(expected), + executeFunction( + "local_match_against_boolean", + {createConstColumn(documents.size(), "+z*"), + createColumn>(documents), + createConstColumn(documents.size(), serializeLocalMatchAgainstBooleanQuery(query))}, + match_collator, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanASCIIAndMalformedUTF8) +try +{ + const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + const auto documents = createColumn( + {"bar", + "foo bar", + "foobar", + "bar!", + String("bar\0foo", 7), + "bar\xff" + "foo", + "foo\xff" + "bar", + "foo中bar", + ""}); + for (const auto size : {0U, 2U, 3U}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(size == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("bar"); + ASSERT_COLUMN_EQ( + size == 0 ? createColumn({1, 1, 0, 1, 1, 1, 1, 0, 0}) + : createColumn({1, 1, 1, 1, 1, 1, 0, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(9, "+bar"), + documents, + createConstColumn(9, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, DocumentViewsPreserveFilteredPhrase) +try +{ + String miss; + for (size_t i = 0; i < 4096; ++i) + miss += "foo xyz zoo "; + for (const bool ngram : {false, true}) + for (const bool stopwords : {false, true}) + for (const auto collation : {"utf8mb4_bin", "utf8mb4_general_ci"}) + { + SCOPED_TRACE(fmt::format("ngram={} stopwords={} collation={}", ngram, stopwords, collation)); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + query.set_ngram_token_size(3); + query.set_stopword_mode( + stopwords ? tipb::LocalMatchAgainstStopwordModeBuiltin + : tipb::LocalMatchAgainstStopwordModeDisabled); + auto * phrase = query.add_nodes(); + phrase->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + phrase->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + phrase->set_text("foo the zoo"); + auto * excluded = query.add_nodes(); + excluded->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + excluded->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + excluded->set_text("blocked"); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(5, "+\"foo the zoo\" -blocked"), + createColumn>({miss, "foo the zoo", {}, "", miss}), + createColumn>( + {"foo the zoo", miss, "foo the zoo", "foo the zoo blocked", miss}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}, + TiDB::ITiDBCollator::getCollator(collation), + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, NgramDocumentViewsAndPositions) +try +{ + // Overlapping grams in separate runs still have consecutive positions; + // short runs contribute none. Exercise vector growth, scratch reuse and + // successive large/small/NULL documents without borrowing scratch memory. + for (const auto size : {1U, 2U, 3U, 4U, 5U, 6U, 7U, 8U, 9U, 10U}) + for (const bool chinese : {false, true}) + for (const auto collation : {"", "utf8mb4_bin", "utf8mb4_general_ci"}) + { + SCOPED_TRACE(fmt::format("size={} chinese={} collation={}", size, chinese, collation)); + const String alphabet = chinese ? "甲乙丙丁戊己庚辛壬癸子丑" : "ABCDEFGHIJKL"; + const size_t width = chinese ? 3 : 1; + const String query_text = alphabet.substr(0, (size + 1) * width); + const String first = query_text.substr(0, size * width); + const String last = query_text.substr(width); + const String short_run(size - 1, 'z'); + const String padding(4096, 'z'); + const auto documents = createColumn>( + {padding + " " + query_text, + query_text, + first + "!" + last, + first + "!" + String(size, 'z') + "!" + last, + first + "!" + short_run + "!" + last, + query_text + "\xff" + padding, + first + "\xff" + last, + {}, + "", + query_text + "!" + padding}); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(query_text); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 1, 0, 1, 1, 0, 0, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(10, query_text), + documents, + createConstColumn(10, serializeLocalMatchAgainstBooleanQuery(query))}, + String(collation).empty() ? nullptr : TiDB::ITiDBCollator::getCollator(collation), + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanUnicodeTokenBoundaries) +try +{ + const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + const auto documents = createColumn( + {"foo🙃bar", + "foo👁bar", + "foo bar", + "foobar", + "foo𞤀bar", + "foo𝟙bar", + "foo\U0002EBF0bar", + "foo\xff" + "bar", + "foo\u088Fbar"}); + for (const auto size : {0U, 2U, 3U}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(size == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(size == 0 ? "foo" : "foob"); + ASSERT_COLUMN_EQ( + size == 0 ? createColumn({1, 1, 1, 0, 1, 1, 1, 1, 0}) + : createColumn({0, 0, 0, 1, 0, 0, 0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(documents.column->size(), node->text()), + documents, + createConstColumn(documents.column->size(), serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); + // TiDB's version-2 NGRAM Boolean lexer emits "bar" for this search. + node->set_text(size == 0 ? "𞤀bar" : "bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+𞤀bar"), + createColumn({"foo𞤀bar", "𞤀bar", "foo bar"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); + } +} +CATCH + +TEST(LocalMatchAgainstTokenChars, Unicode17Classification) +{ + for (const UInt32 code_point : {0x1F643U, 0x1F441U, 0x301U, 0xFFFDU, 0x110000U}) + EXPECT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + for (const UInt32 code_point : {0x5FU, 0x88FU, 0x4E2DU, 0xE9U, 0x1D7D9U, 0x1E900U, 0x2EBF0U}) + EXPECT_TRUE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + UInt32 previous = 127; + for (const auto & span : LocalMatchAgainst::token_char_ranges) + { + ASSERT_GT(span.first, previous); + ASSERT_LE(span.first, span.last); + for (UInt32 code_point = previous + 1; code_point < span.first; ++code_point) + ASSERT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + for (UInt32 code_point = span.first; code_point <= span.last; ++code_point) + ASSERT_TRUE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + previous = span.last; + } + for (UInt32 code_point = previous + 1; code_point <= 0x10FFFFU; ++code_point) + ASSERT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + + // FNV-1a over one 0/1 byte per code point, U+0000..U+10FFFF. The paired + // TiDB tests hash Go's Unicode 17.0.0 IsLetter/IsNumber/'_' classification + // identically. The fingerprint catches changes to the shared classification. + std::uint64_t fingerprint = 14695981039346656037ULL; + for (UInt32 code_point = 0; code_point <= 0x10FFFFU; ++code_point) + { + fingerprint ^= static_cast(LocalMatchAgainst::isTokenChar(code_point)); + fingerprint *= 1099511628211ULL; + } + EXPECT_EQ(0x0b93995fa724afa0ULL, fingerprint); +} + +namespace +{ +ColumnWithTypeAndName runMatcher( + const ColumnsWithTypeAndName & columns, + TiDB::TiDBCollatorPtr collator, + LocalMatchAgainstTest::Matcher strategy, + LocalMatchAgainstTest::Work & work, + size_t budget = 4 * 1024 * 1024, + UInt32 generation = 0, + FunctionPtr reuse = {}) +{ + auto function = reuse ? reuse : LocalMatchAgainstTest::create(strategy, work, budget, generation); + function->setCollator(collator); + Block block; + DataTypes types; + ColumnNumbers arguments; + for (size_t i = 0; i < columns.size(); ++i) + { + auto column = columns[i]; + column.name = "argument" + std::to_string(i); + block.insert(column); + types.push_back(column.type); + arguments.push_back(i); + } + block.insert({nullptr, function->getReturnTypeImpl(types), "result"}); + function->executeImpl(block, arguments, columns.size()); + return block.getByPosition(columns.size()); +} + +tipb::LocalMatchAgainstBooleanQuery exactQuery(size_t terms, bool duplicate = false) +{ + tipb::LocalMatchAgainstBooleanQuery query; + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + for (size_t i = 0; i < terms; ++i) + { + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(duplicate ? "apple" : "query" + std::to_string(i)); + } + return query; +} +} // namespace + +TEST_F(TestLocalMatchAgainst, FilteredPhraseAnalyzesRawColumnOnDemand) +try +{ + const auto * collator = TiDB::ITiDBCollator::getCollator("utf8mb4_bin"); + for (const auto parser : {tipb::LocalMatchAgainstParserStandard, tipb::LocalMatchAgainstParserNgram}) + { + SCOPED_TRACE(parser); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(parser); + query.set_ngram_token_size(3); + for (const auto * text : {"foo the zoo", "red the qux"}) + { + auto * phrase = query.add_nodes(); + phrase->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + phrase->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + phrase->set_text(text); + } + const ColumnsWithTypeAndName columns{ + createConstColumn(6, "+\"foo the zoo\" +\"red the qux\""), + createColumn>( + {"unrelated", "foo the zoo red the qux", "foo xyz zoo red the qux", "foo xyz zoo", "foo the zoo", {}}), + createColumn>( + {"filler", "filler", "foo the zoo", "filler", "red the qux", "foo the zoo red the qux"}), + createConstColumn(6, serializeLocalMatchAgainstBooleanQuery(query))}; + LocalMatchAgainstTest::Work work; + ASSERT_COLUMN_EQ( + createColumn>({0, 1, 1, 0, 1, 1}), + runMatcher(columns, collator, LocalMatchAgainstTest::Matcher::Automatic, work)); + // Raw scans by row: 0, 1 (two clauses share a scan), 2, 1, 2, 1. + EXPECT_EQ(work.unfiltered_column_analyses, 7); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, ExactRouteWorkAndBucketDeduplication) +{ + using namespace LocalMatchAgainstTest; + CountingMatchCollator collator; + auto query = exactQuery(100); + String document; + for (size_t i = 0; i < 1000; ++i) + document += "document "; + const auto columns = ColumnsWithTypeAndName{ + createConstColumn(1, "query"), + createColumn({document}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}; + Work linear, indexed; + const auto expected = runMatcher(columns, &collator, Matcher::Linear, linear); + EXPECT_EQ(100000, collator.comparisons); + collator.comparisons = 0; + ASSERT_COLUMN_EQ(expected, runMatcher(columns, &collator, Matcher::Automatic, indexed)); + EXPECT_EQ(0, collator.comparisons); + EXPECT_EQ(1, indexed.indexed_columns); + EXPECT_EQ(1000, indexed.key_lookups); + EXPECT_EQ(0, indexed.term_references); + + query = exactQuery(100, true); + auto * prohibited = query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("absent"); + document.clear(); + for (size_t i = 0; i < 500; ++i) + document += "apple "; + const auto multi = ColumnsWithTypeAndName{ + createConstColumn(2, "query"), + createColumn({document, "absent"}), + createColumn({document, document}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(query))}; + Work dedup; + ASSERT_COLUMN_EQ(createColumn({1, 0}), runMatcher(multi, &collator, Matcher::Indexed, dedup)); + EXPECT_EQ(101, dedup.term_references); // 100 positive references, then one rejection in row 2. + EXPECT_EQ(1001, dedup.key_lookups); +} + +TEST_F(TestLocalMatchAgainst, ExactRouteDifferentialAndFallbacks) +{ + using namespace LocalMatchAgainstTest; + std::mt19937 random(20261030); + const std::array words{ + "apple", + "APPLE", + "banana", + "cafe", + "café", + "Straße", + "strasse", + "ᅠᅠᅠ", + "中文文本", + "foo.bar", + "abc", + "ABC"}; + for (const auto id : + {TiDB::ITiDBCollator::UTF8MB4_BIN, + TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI, + TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI, + TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI}) + { + const auto * const collator = TiDB::ITiDBCollator::getCollator(id); + for (size_t iteration = 0; iteration < 64; ++iteration) + { + auto query = exactQuery(0); + for (size_t i = 0; i < 25; ++i) + { + auto * node = query.add_nodes(); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + const auto mode = iteration % 4; + node->set_occur( + mode == 0 ? tipb::LocalMatchAgainstBooleanOccurShould + : mode == 1 ? tipb::LocalMatchAgainstBooleanOccurMust + : static_cast(1 + random() % 3)); + node->set_text(words[random() % words.size()]); + if (mode == 3 && node->occur() == tipb::LocalMatchAgainstBooleanOccurMustNot) + node->set_text("absent"); + } + std::vector left(8), right(8); + for (size_t row = 0; row < left.size(); ++row) + for (size_t i = 0; i < 60; ++i) + { + left[row] += words[random() % words.size()] + ' '; + right[row] += words[random() % words.size()] + ' '; + } + const ColumnsWithTypeAndName columns{ + createConstColumn(8, "query"), + createColumn(left), + createColumn(right), + createConstColumn(8, serializeLocalMatchAgainstBooleanQuery(query))}; + Work old_work, new_work; + ASSERT_COLUMN_EQ( + runMatcher(columns, collator, Matcher::Linear, old_work), + runMatcher( + columns, + collator, + Matcher::Indexed, + new_work, + 4 * 1024 * 1024, + std::numeric_limits::max() - 1)); + } + } + auto query = exactQuery(100, true); + String document; + for (size_t i = 0; i < 100; ++i) + document += "apple "; + const auto check = [&](const auto & q, TiDB::TiDBCollatorPtr collator, size_t budget) { + const ColumnsWithTypeAndName columns{ + createConstColumn(1, "query"), + createColumn({document}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(q))}; + Work old_work, new_work; + ASSERT_COLUMN_EQ( + runMatcher(columns, collator, Matcher::Linear, old_work), + runMatcher(columns, collator, Matcher::Indexed, new_work, budget)); + EXPECT_EQ(0, new_work.indexed_columns); + }; + const auto * collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + check(query, collator, 1); + check(query, nullptr, 4 * 1024 * 1024); + CountingMatchCollator unsupported(false); + check(query, &unsupported, 4 * 1024 * 1024); + query.mutable_nodes(0)->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + check(query, collator, 4 * 1024 * 1024); + query.mutable_nodes(0)->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + check(query, collator, 4 * 1024 * 1024); + query.set_parser(tipb::LocalMatchAgainstParserNgram); + check(query, collator, 4 * 1024 * 1024); +} + +TEST_F(TestLocalMatchAgainst, ExactRouteEquivalentProhibitedAndColumnFallback) +{ + using namespace LocalMatchAgainstTest; + const auto * const collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); + auto query = exactQuery(100, true); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("APPLE"); + Work work; + ASSERT_COLUMN_EQ( + createColumn({0}), + runMatcher( + {createConstColumn(1, "query"), + createColumn({"apple"}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + Matcher::Indexed, + work)); + + query = exactQuery(100, true); + query.set_innodb_ft_max_token_size(10000); + node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("banana"); + String oversized(5000, 'x'); + Work fallback; + const auto result = runMatcher( + {createConstColumn(2, "query"), + createColumn({"apple", "apple"}), + createColumn({oversized + " banana", oversized}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + Matcher::Indexed, + fallback); + ASSERT_COLUMN_EQ(createColumn({1, 0}), result); + EXPECT_EQ(2, fallback.indexed_columns); + EXPECT_EQ(2, fallback.linear_columns); +} + +TEST_F(TestLocalMatchAgainst, ExactRouteTokenThresholdForLargeBlocks) +{ + using namespace LocalMatchAgainstTest; + const auto * const collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + for (size_t rows : {127, 128}) + { + for (size_t tokens : {15, 16, 31, 32}) + { + String document = "a"; + for (size_t i = 1; i < tokens; ++i) + document += " a"; + for (size_t terms : {63, 64, 99, 100}) + { + SCOPED_TRACE(::testing::Message() << "rows=" << rows << " tokens=" << tokens << " terms=" << terms); + auto query = exactQuery(terms, true); + query.set_innodb_ft_min_token_size(1); + query.set_innodb_ft_max_token_size(84); + const ColumnsWithTypeAndName columns{ + createConstColumn(rows, "query"), + createColumn(std::vector(rows, document)), + createConstColumn(rows, serializeLocalMatchAgainstBooleanQuery(query))}; + Work linear, automatic; + ASSERT_COLUMN_EQ( + runMatcher(columns, collator, Matcher::Linear, linear), + runMatcher(columns, collator, Matcher::Automatic, automatic)); + const bool indexed = rows == 128 ? terms >= 64 && tokens >= 16 : terms >= 100 && tokens >= 32; + EXPECT_EQ(indexed ? 1 : 0, automatic.route_builds); + EXPECT_EQ(indexed ? rows : 0, automatic.indexed_columns); + EXPECT_EQ(indexed ? 0 : rows, automatic.linear_columns); + } + } + } +} + +TEST_F(TestLocalMatchAgainst, ExactRouteThresholdNullableAndReusedBlocks) +{ + using namespace LocalMatchAgainstTest; + const auto * const collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + String document; + for (size_t i = 0; i < 100; ++i) + document += "apple "; + for (size_t count : {1, 4, 15, 16, 17, 99, 100, 101}) + { + const auto query = exactQuery(count, true); + Work work; + ASSERT_COLUMN_EQ( + createColumn({1}), + runMatcher( + {createConstColumn(1, "query"), + createColumn({document}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + Matcher::Automatic, + work)); + EXPECT_EQ(count >= 100 ? 1 : 0, work.indexed_columns); + } + for (size_t tokens : {31, 32}) + { + auto query = exactQuery(100, true); + query.set_innodb_ft_min_token_size(1); + query.set_innodb_ft_max_token_size(84); + String short_tokens; + for (size_t i = 0; i < tokens; ++i) + short_tokens += i == 0 ? "a" : " a"; + Work linear, automatic; + const ColumnsWithTypeAndName columns{ + createConstColumn(1, "query"), + createColumn({short_tokens}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}; + ASSERT_COLUMN_EQ( + runMatcher(columns, collator, Matcher::Linear, linear), + runMatcher(columns, collator, Matcher::Automatic, automatic)); + EXPECT_EQ(tokens == 32 ? 1 : 0, automatic.indexed_columns); + } + { + const auto query = exactQuery(100, true); + Work short_work; + ASSERT_COLUMN_EQ( + createColumn({1}), + runMatcher( + {createConstColumn(1, "query"), + createColumn({"apple"}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + Matcher::Automatic, + short_work)); + EXPECT_EQ(0, short_work.route_builds); + EXPECT_EQ(1, short_work.linear_columns); + } + Work work; + auto function = LocalMatchAgainstTest::create(Matcher::Indexed, work); + for (size_t rows : {4, 0, 4}) + { + const auto query = exactQuery(100, true); + auto column = createColumn>({document, {}, "absent", document}); + column.column = column.column->cloneResized(rows); + auto expected = createColumn>({1, 0, 0, 1}); + expected.column = expected.column->cloneResized(rows); + ASSERT_COLUMN_EQ( + expected, + runMatcher( + {createConstColumn(rows, "query"), + column, + createConstColumn(rows, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + Matcher::Indexed, + work, + 4 * 1024 * 1024, + 0, + function)); + } + auto query = exactQuery(100, true); + query.mutable_nodes(0)->set_text("banana"); + ASSERT_COLUMN_EQ( + createColumn({0}), + runMatcher( + {createConstColumn(1, "query"), + createColumn({document}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + Matcher::Indexed, + work, + 4 * 1024 * 1024, + 0, + function)); +} + +TEST_F(TestLocalMatchAgainst, ExactRouteIndependentConcurrentExecutions) +{ + using namespace LocalMatchAgainstTest; + const auto * const collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); + const auto query = exactQuery(100, true); + String document; + for (size_t i = 0; i < 100; ++i) + document += "apple "; + const ColumnsWithTypeAndName columns{ + createConstColumn(4, "query"), + createColumn({document, "absent", document, "absent"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}; + std::vector> tasks; + for (size_t i = 0; i < 8; ++i) + tasks.push_back(std::async(std::launch::async, [&] { + Work work; + return runMatcher(columns, collator, Matcher::Indexed, work); + })); + for (auto & task : tasks) + ASSERT_COLUMN_EQ(createColumn({1, 0, 1, 0}), task.get()); +} + +TEST(LocalMatchAgainstTokenChars, ExactRouteKeyEquality) +{ + std::mt19937 random(20261030); + for (const auto id : exact_route_collations) + { + SCOPED_TRACE(id); + const auto * const collator = TiDB::ITiDBCollator::getCollator(id); + std::unordered_map representatives; + std::vector words{"café", "CAFE", "Straße", "strasse", "fi", "fi", "ᅠ", "", "abc", "ABC"}; + String buffer; + for (const auto & a : words) + for (const auto & b : words) + { + auto key = collator->sortKeyFastPath(a.data(), a.size(), buffer); + const String owned(key.data, key.size); + key = collator->sortKeyFastPath(b.data(), b.size(), buffer); + ASSERT_EQ( + collator->compare(a.data(), a.size(), b.data(), b.size()) == 0, + owned == String(key.data, key.size)); + } + for (UInt32 cp = 128; cp <= 0xFFFF; ++cp) + { + if (!LocalMatchAgainst::isTokenChar(cp)) + continue; + String word; + if (cp < 0x800) + { + word += static_cast(0xC0 | cp >> 6); + word += static_cast(0x80 | (cp & 63)); + } + else + { + word += static_cast(0xE0 | cp >> 12); + word += static_cast(0x80 | ((cp >> 6) & 63)); + word += static_cast(0x80 | (cp & 63)); + } + const auto key = collator->sortKeyFastPath(word.data(), word.size(), buffer); + auto [it, inserted] = representatives.emplace(String(key.data, key.size), word); + if (!inserted) + ASSERT_EQ(0, collator->compare(word.data(), word.size(), it->second.data(), it->second.size())); + if (random() % 32 == 0) + words.push_back(word); + } + for (size_t i = 0; i < 10000; ++i) + { + const String a = words[random() % words.size()] + words[random() % words.size()]; + const String b = i % 2 == 0 ? a : words[random() % words.size()] + words[random() % words.size()]; + auto key = collator->sortKeyFastPath(a.data(), a.size(), buffer); + const String owned(key.data, key.size); + key = collator->sortKeyFastPath(b.data(), b.size(), buffer); + ASSERT_EQ( + collator->compare(a.data(), a.size(), b.data(), b.size()) == 0, + owned == String(key.data, key.size)); + } + } +} + +TEST(LocalMatchAgainstTokenChars, ExactRouteKeyEqualitySpecialCases) +{ + // Include raw collation boundaries as well as valid STANDARD tokens: + // trailing spaces, embedded NUL, UCA expansions, zero weights and accents. + // The two calls deliberately reuse one buffer; retain the returned size + // and copy each key before a later call can overwrite its storage. + const std::vector words{ + "", + " ", + " ", + "abc", + "ABC", + "abc ", + "abc ", + "abc\t", + "cafe", + "CAFÉ", + "café", + "cafe\u0301", + "Straße", + "strasse", + "fi", + "fi", + "ffi", + "ffi", + "æ", + "ae", + "\uFDFA", + "\uFDFB", + "جل جلاله", + "㌖", + "キロメートル", + "x㌖y", + "xキロメートルy", + "㌖\u034F", + "\u034F", + "\u00AD", + "\u200D", + "\u1160", + "\uFE0F", + "a\u034Fb", + "ab", + "a\u00ADb", + "a\u200Db", + "a\u1160b", + "a\uFE0Fb", + "中文", + "ABC", + "𠀀", + "😀", + String(1, '\0'), + String("abc\0", 4), + String("abc\0 ", 5), + String("a\0b", 3), + String(512, 'a'), + "a"}; + for (const auto id : exact_route_collations) + { + const auto * const collator = TiDB::ITiDBCollator::getCollator(id); + ASSERT_NE(collator, nullptr); + String buffer; + const auto key = [&](const String & word) { + const auto result = collator->sortKeyFastPath(word.data(), word.size(), buffer); + return String(result.data, result.size); + }; + for (size_t i = 0; i < words.size(); ++i) + for (size_t j = 0; j < words.size(); ++j) + { + SCOPED_TRACE(fmt::format("collation={} left={} right={}", id, i, j)); + const auto left = key(words[i]); + const auto right = key(words[j]); + ASSERT_EQ( + collator->compare(words[i].data(), words[i].size(), words[j].data(), words[j].size()) == 0, + left == right); + } + + // Explicit expected semantics prevent a shared trimming error from + // making the compare/key differential pass unnoticed. + const bool padded = id != TiDB::ITiDBCollator::BINARY && id != TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI + && id != TiDB::ITiDBCollator::UTF8MB4_0900_BIN; + SCOPED_TRACE(id); + EXPECT_EQ(padded, collator->compare("abc", 3, "abc ", 4) == 0); + const auto unpadded_key = key("abc"); + EXPECT_EQ(padded, unpadded_key == key("abc ")); + } +} + +TEST_F(TestLocalMatchAgainst, ExactRouteMalformedUTF8AfterTokenization) +{ + using namespace LocalMatchAgainstTest; + const std::array malformed{ + "\xff", + "\x80", + "\xc0\xaf", + "\xe0\x80\xaf", + "\xed\xa0\x80", + "\xf4\x90\x80\x80", + "\xc2", + "\xe4\xb8", + "\xf0\x9f"}; + String padding; + for (size_t i = 0; i < 40; ++i) + padding += " padding"; + for (const auto id : exact_route_collations) + for (size_t i = 0; i < malformed.size(); ++i) + { + SCOPED_TRACE(fmt::format("collation={} malformed={}", id, i)); + const auto * const collator = TiDB::ITiDBCollator::getCollator(id); + auto query = exactQuery(100); + const auto broken = "foo" + malformed[i] + "bar"; + for (auto & node : *query.mutable_nodes()) + node.set_text(broken); + const ColumnsWithTypeAndName columns{ + createConstColumn(5, "query"), + createColumn( + {broken + padding, "foo bar" + padding, "foobar" + padding, "foo" + padding, padding}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}; + Work linear, indexed, automatic; + const auto expected = runMatcher(columns, collator, Matcher::Linear, linear); + ASSERT_COLUMN_EQ(createColumn({1, 1, 0, 0, 0}), expected); + ASSERT_COLUMN_EQ(expected, runMatcher(columns, collator, Matcher::Indexed, indexed)); + ASSERT_COLUMN_EQ(expected, runMatcher(columns, collator, Matcher::Automatic, automatic)); + EXPECT_EQ(5, indexed.indexed_columns); + EXPECT_EQ(5, automatic.indexed_columns); + } +} + +} // namespace DB::tests