From e08824afe2707c0bedfa59a14b119019afba0f5f Mon Sep 17 00:00:00 2001 From: David Gidwani Date: Thu, 8 Oct 2026 10:49:42 -0400 Subject: [PATCH] fix: compile UTF-8 parser with unsigned chars --- CMakeLists.txt | 29 +++--- MANIFEST.in | 1 + cmake/patch_vectorscan.cmake | 30 +++++++ src/hyperscan/extension.c | 168 +++++++++++++++++++++++++++++++---- tests/test_hyperscan.py | 23 +++-- 5 files changed, 211 insertions(+), 40 deletions(-) create mode 100644 cmake/patch_vectorscan.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 44b3948..5e93213 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -286,8 +286,13 @@ if(HS_BUILD_REQUIRED) set(HS_CMAKE_COMMON_FLAGS "-fPIC") set(HS_USE_SIMDE_BACKEND ON) endif() - - + + # Vectorscan's Ragel-generated parser compares regex bytes in the + # 0x80..0xff range when HS_FLAG_UTF8 is enabled. On targets where plain + # char is signed (notably x86/x86_64), those bytes can be misclassified as + # invalid UTF-8 unless downstream C/C++ code uses unsigned char semantics. + set(HS_CMAKE_COMMON_FLAGS "${HS_CMAKE_COMMON_FLAGS} -funsigned-char") + set(HS_CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${HS_CMAKE_COMMON_FLAGS}") set(HS_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${HS_CMAKE_COMMON_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=1") @@ -495,16 +500,20 @@ if(HS_BUILD_REQUIRED) # archdetect.cmake, but GCC <11 (e.g. manylinux2014 devtoolset) does # not recognize this value. Patch it to use "nehalem" which provides # the same SSE4.2 baseline and is supported by all GCC versions. - # Uses perl instead of sed to avoid BSD/GNU sed -i syntax differences. + # + # Its Ragel-generated parser also compares input bytes against UTF-8 ranges. + # Force an unsigned key expression so x86/x86_64 builds don't reject valid + # multibyte UTF-8 patterns under HS_FLAG_UTF8. + set(HS_PATCH_X86_64_V2 OFF) if(USE_VECTORSCAN AND NOT HS_USE_SIMDE_BACKEND) - set(HS_PATCH_COMMAND - perl -pi -e "s/x86-64-v2/nehalem/g" - ${hyperscan_SOURCE_DIR}/cmake/cflags-x86.cmake - ${hyperscan_SOURCE_DIR}/cmake/archdetect.cmake - ) - else() - set(HS_PATCH_COMMAND "") + set(HS_PATCH_X86_64_V2 ON) endif() + set(HS_PATCH_COMMAND + ${CMAKE_COMMAND} + -DVECTORSCAN_SOURCE_DIR=${hyperscan_SOURCE_DIR} + -DVECTORSCAN_PATCH_X86_64_V2=${HS_PATCH_X86_64_V2} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patch_vectorscan.cmake + ) ExternalProject_Add( libhs diff --git a/MANIFEST.in b/MANIFEST.in index cfaafe3..7e7330a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include *.rst +recursive-include cmake *.cmake recursive-include docs * recursive-include examples * recursive-include src/hyperscan *.c *.cpp *.h *.pyx *.pxd *.pxi py.typed diff --git a/cmake/patch_vectorscan.cmake b/cmake/patch_vectorscan.cmake new file mode 100644 index 0000000..c9aaf13 --- /dev/null +++ b/cmake/patch_vectorscan.cmake @@ -0,0 +1,30 @@ +if(NOT DEFINED VECTORSCAN_SOURCE_DIR) + message(FATAL_ERROR "VECTORSCAN_SOURCE_DIR is required") +endif() + +set(_parser_file "${VECTORSCAN_SOURCE_DIR}/src/parser/Parser.rl") +if(EXISTS "${_parser_file}") + file(READ "${_parser_file}" _parser_content) + string(REPLACE + "alphtype unsigned char;\n" + "alphtype unsigned char;\n getkey ((unsigned char)*p);\n" + _parser_content + "${_parser_content}") + file(WRITE "${_parser_file}" "${_parser_content}") +else() + message(WARNING "Vectorscan parser file not found at ${_parser_file}") +endif() + +if(DEFINED VECTORSCAN_PATCH_X86_64_V2 AND VECTORSCAN_PATCH_X86_64_V2) + foreach(_file IN ITEMS + "${VECTORSCAN_SOURCE_DIR}/cmake/cflags-x86.cmake" + "${VECTORSCAN_SOURCE_DIR}/cmake/archdetect.cmake") + if(EXISTS "${_file}") + file(READ "${_file}" _content) + string(REPLACE "x86-64-v2" "nehalem" _content "${_content}") + file(WRITE "${_file}" "${_content}") + else() + message(WARNING "Vectorscan patch target not found at ${_file}") + endif() + endforeach() +endif() diff --git a/src/hyperscan/extension.c b/src/hyperscan/extension.c index e072d10..8c654ba 100644 --- a/src/hyperscan/extension.c +++ b/src/hyperscan/extension.c @@ -344,6 +344,114 @@ static int Database_init(Database *self, PyObject *args, PyObject *kwds) return 0; } +static PyObject *escape_utf8_non_ascii(PyObject *obytes) +{ + char *bytes; + Py_ssize_t bytes_len; + if (PyBytes_AsStringAndSize(obytes, &bytes, &bytes_len) == -1) { + return NULL; + } + + int has_non_ascii = 0; + for (Py_ssize_t i = 0; i < bytes_len; i++) { + if (((unsigned char *)bytes)[i] > 0x7f) { + has_non_ascii = 1; + break; + } + } + if (!has_non_ascii) { + return Py_NewRef(obytes); + } + + PyObject *unicode = PyUnicode_DecodeUTF8(bytes, bytes_len, "strict"); + if (unicode == NULL) { + PyErr_Clear(); + return Py_NewRef(obytes); + } + + Py_ssize_t chars_len = PyUnicode_GET_LENGTH(unicode); + Py_ssize_t escaped_len = 0; + for (Py_ssize_t i = 0; i < chars_len; i++) { + Py_UCS4 ch = PyUnicode_ReadChar(unicode, i); + if (ch == (Py_UCS4)-1) { + Py_DECREF(unicode); + return NULL; + } + + Py_ssize_t needed = 1; + if (ch > 0x7f) { + Py_UCS4 value = ch; + Py_ssize_t hex_digits = 1; + while (value >>= 4) { + hex_digits++; + } + needed = hex_digits + 4; // \x{...} + } + + if (escaped_len > PY_SSIZE_T_MAX - needed) { + Py_DECREF(unicode); + PyErr_NoMemory(); + return NULL; + } + escaped_len += needed; + } + + PyObject *result = PyBytes_FromStringAndSize(NULL, escaped_len); + if (result == NULL) { + Py_DECREF(unicode); + return NULL; + } + + char *escaped = PyBytes_AS_STRING(result); + Py_ssize_t write_pos = 0; + static const char hex[] = "0123456789abcdef"; + for (Py_ssize_t i = 0; i < chars_len; i++) { + Py_UCS4 ch = PyUnicode_ReadChar(unicode, i); + if (ch == (Py_UCS4)-1) { + Py_DECREF(result); + Py_DECREF(unicode); + return NULL; + } + if (ch <= 0x7f) { + escaped[write_pos++] = (char)ch; + continue; + } + + escaped[write_pos++] = '\\'; + escaped[write_pos++] = 'x'; + escaped[write_pos++] = '{'; + int shift = 0; + for (Py_UCS4 value = ch >> 4; value != 0; value >>= 4) { + shift += 4; + } + for (; shift >= 0; shift -= 4) { + escaped[write_pos++] = hex[(ch >> shift) & 0xf]; + } + escaped[write_pos++] = '}'; + } + + Py_DECREF(unicode); + return result; +} + +static void free_compile_inputs( + const char **expressions, + uint32_t *flags, + uint32_t *ids, + PyObject **expression_refs, + uint64_t elements) +{ + if (expression_refs != NULL) { + for (uint64_t i = 0; i < elements; i++) { + Py_XDECREF(expression_refs[i]); + } + free(expression_refs); + } + free(expressions); + free(flags); + free(ids); +} + static PyObject *Database_compile( Database *self, PyObject *args, PyObject *kwds) { @@ -392,9 +500,10 @@ static PyObject *Database_compile( PyObject *oexpr = NULL; PyObject *oid = NULL; - const char **expressions; - uint32_t *flags; - uint32_t *ids; + const char **expressions = NULL; + PyObject **expression_refs = NULL; + uint32_t *flags = NULL; + uint32_t *ids = NULL; size_t *lens; uint32_t globalflag; @@ -408,6 +517,10 @@ static PyObject *Database_compile( if (expressions == NULL) goto memory_error; + expression_refs = calloc(elements, sizeof(PyObject *)); + if (expression_refs == NULL) + goto memory_error; + flags = malloc(elements * sizeof(uint32_t)); if (flags == NULL) goto memory_error; @@ -426,6 +539,10 @@ static PyObject *Database_compile( uint32_t expr_id; oexpr = PySequence_ITEM(oexpressions, i); + if (oexpr == NULL) { + break; + } + expression_refs[i] = oexpr; // Handle both bytes and unicode strings if (PyBytes_Check(oexpr)) { @@ -440,6 +557,7 @@ static PyObject *Database_compile( // Replace the original object with the encoded version Py_DECREF(oexpr); oexpr = temp_bytes; + expression_refs[i] = oexpr; } else { PyErr_SetString(PyExc_TypeError, "expressions must be bytes or str"); break; @@ -468,11 +586,22 @@ static PyObject *Database_compile( expr_flags = globalflag; } + if (expr_flags & HS_FLAG_UTF8) { + PyObject *escaped_expr = escape_utf8_non_ascii(oexpr); + if (escaped_expr == NULL) { + break; + } + Py_DECREF(oexpr); + oexpr = escaped_expr; + expression_refs[i] = oexpr; + expression = PyBytes_AsString(oexpr); + if (PyErr_Occurred()) + break; + } + expressions[i] = expression; ids[i] = expr_id; flags[i] = expr_flags; - - Py_XDECREF(oexpr); } if (oflag != Py_None) @@ -515,9 +644,11 @@ static PyObject *Database_compile( &hs_compile_err); free(lens); Py_END_ALLOW_THREADS; - free(expressions); - free(flags); - free(ids); + free_compile_inputs(expressions, flags, ids, expression_refs, elements); + expressions = NULL; + flags = NULL; + ids = NULL; + expression_refs = NULL; if (hs_err != HS_SUCCESS) { PyErr_Format( @@ -544,9 +675,11 @@ static PyObject *Database_compile( &self->ch_db, &ch_compile_err); Py_END_ALLOW_THREADS; - free(expressions); - free(flags); - free(ids); + free_compile_inputs(expressions, flags, ids, expression_refs, elements); + expressions = NULL; + flags = NULL; + ids = NULL; + expression_refs = NULL; HANDLE_CHIMERA_ERR(ch_err, NULL); } else { if (oext != Py_None) { @@ -589,9 +722,11 @@ static PyObject *Database_compile( &hs_compile_err); Py_END_ALLOW_THREADS; - free(expressions); - free(flags); - free(ids); + free_compile_inputs(expressions, flags, ids, expression_refs, elements); + expressions = NULL; + flags = NULL; + ids = NULL; + expression_refs = NULL; if (hs_err != HS_SUCCESS) { PyErr_SetString(HyperscanError, hs_compile_err->message); hs_free_compile_error(hs_compile_err); @@ -631,13 +766,12 @@ static PyObject *Database_compile( HS_LOCK_RETURN(Py_NewRef(Py_None)); memory_error: + free_compile_inputs(expressions, flags, ids, expression_refs, elements); PyErr_NoMemory(); HS_LOCK_RETURN_NULL(); python_error: - free(expressions); - free(flags); - free(ids); + free_compile_inputs(expressions, flags, ids, expression_refs, elements); HS_LOCK_RETURN_NULL(); } diff --git a/tests/test_hyperscan.py b/tests/test_hyperscan.py index fcfac2d..7026d9d 100644 --- a/tests/test_hyperscan.py +++ b/tests/test_hyperscan.py @@ -426,15 +426,9 @@ def test_unicode_expressions(): valid UTF-8 patterns. This was caused by PCRE being built without UTF-8 support in v0.7.9+ when the build system switched from setup.py to CMake. - Note on HS_FLAG_UTF8: - We avoid using HS_FLAG_UTF8 by default due to known Hyperscan/Vectorscan - limitations and bugs: - - intel/hyperscan#57: UTF-8 match failures with \\Q...\\E patterns - - intel/hyperscan#133: Parser bug with Ragel v7 incorrectly rejecting valid UTF-8 - - intel/hyperscan#163: Performance issues with UTF-8 + case-insensitive flags - - Unicode patterns work correctly without HS_FLAG_UTF8 when PCRE has proper - UTF-8 support, which is what our CMake fixes provide. + The HS_FLAG_UTF8 cases cover issue #274, where x86/x86_64 wheels compiled + Vectorscan with signed-char semantics and rejected valid multibyte UTF-8 + literals such as é and €. """ complex_patterns = [ r"السلام عليكم\s<\/span>", @@ -454,10 +448,13 @@ def test_unicode_expressions(): db_bytes.compile(expressions=bytes_patterns) db_utf8 = hyperscan.Database() - try: - db_utf8.compile(expressions=simple_patterns, flags=hyperscan.HS_FLAG_UTF8) - except Exception as e: - pytest.skip(f"HS_FLAG_UTF8 validation failed (known limitation): {e}") + db_utf8.compile(expressions=simple_patterns, flags=hyperscan.HS_FLAG_UTF8) + + db_utf8_latin = hyperscan.Database() + db_utf8_latin.compile( + expressions=["é".encode("utf-8"), "€".encode("utf-8")], + flags=hyperscan.HS_FLAG_UTF8, + ) test_text = 'السلام عليكم '