fix: compile UTF-8 parser with unsigned chars

CMakeLists.txt

@@ -286,8 +286,13 @@ if(HS_BUILD_REQUIRED)
       set(HS_CMAKE_COMMON_FLAGS "-fPIC")
       set(HS_USE_SIMDE_BACKEND ON)
     endif()
-
-
+
+    # Vectorscan's Ragel-generated parser compares regex bytes in the
+    # 0x80..0xff range when HS_FLAG_UTF8 is enabled. On targets where plain
+    # char is signed (notably x86/x86_64), those bytes can be misclassified as
+    # invalid UTF-8 unless downstream C/C++ code uses unsigned char semantics.
+    set(HS_CMAKE_COMMON_FLAGS "${HS_CMAKE_COMMON_FLAGS} -funsigned-char")
+
     set(HS_CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${HS_CMAKE_COMMON_FLAGS}")
     set(HS_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${HS_CMAKE_COMMON_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=1")
 
@@ -495,16 +500,20 @@ if(HS_BUILD_REQUIRED)
   # archdetect.cmake, but GCC <11 (e.g. manylinux2014 devtoolset) does
   # not recognize this value. Patch it to use "nehalem" which provides
   # the same SSE4.2 baseline and is supported by all GCC versions.
-  # Uses perl instead of sed to avoid BSD/GNU sed -i syntax differences.
+  #
+  # Its Ragel-generated parser also compares input bytes against UTF-8 ranges.
+  # Force an unsigned key expression so x86/x86_64 builds don't reject valid
+  # multibyte UTF-8 patterns under HS_FLAG_UTF8.
+  set(HS_PATCH_X86_64_V2 OFF)
   if(USE_VECTORSCAN AND NOT HS_USE_SIMDE_BACKEND)
-    set(HS_PATCH_COMMAND
-      perl -pi -e "s/x86-64-v2/nehalem/g"
-        ${hyperscan_SOURCE_DIR}/cmake/cflags-x86.cmake
-        ${hyperscan_SOURCE_DIR}/cmake/archdetect.cmake
-    )
-  else()
-    set(HS_PATCH_COMMAND "")
+    set(HS_PATCH_X86_64_V2 ON)
   endif()
+  set(HS_PATCH_COMMAND
+    ${CMAKE_COMMAND}
+      -DVECTORSCAN_SOURCE_DIR=${hyperscan_SOURCE_DIR}
+      -DVECTORSCAN_PATCH_X86_64_V2=${HS_PATCH_X86_64_V2}
+      -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patch_vectorscan.cmake
+  )
 
   ExternalProject_Add(
     libhs

MANIFEST.in

@@ -1,4 +1,5 @@
 include *.rst
+recursive-include cmake *.cmake
 recursive-include docs *
 recursive-include examples *
 recursive-include src/hyperscan *.c *.cpp *.h *.pyx *.pxd *.pxi py.typed

cmake/patch_vectorscan.cmake

@@ -0,0 +1,30 @@
+if(NOT DEFINED VECTORSCAN_SOURCE_DIR)
+  message(FATAL_ERROR "VECTORSCAN_SOURCE_DIR is required")
+endif()
+
+set(_parser_file "${VECTORSCAN_SOURCE_DIR}/src/parser/Parser.rl")
+if(EXISTS "${_parser_file}")
+  file(READ "${_parser_file}" _parser_content)
+  string(REPLACE
+    "alphtype unsigned char;\n"
+    "alphtype unsigned char;\n    getkey ((unsigned char)*p);\n"
+    _parser_content
+    "${_parser_content}")
+  file(WRITE "${_parser_file}" "${_parser_content}")
+else()
+  message(WARNING "Vectorscan parser file not found at ${_parser_file}")
+endif()
+
+if(DEFINED VECTORSCAN_PATCH_X86_64_V2 AND VECTORSCAN_PATCH_X86_64_V2)
+  foreach(_file IN ITEMS
+      "${VECTORSCAN_SOURCE_DIR}/cmake/cflags-x86.cmake"
+      "${VECTORSCAN_SOURCE_DIR}/cmake/archdetect.cmake")
+    if(EXISTS "${_file}")
+      file(READ "${_file}" _content)
+      string(REPLACE "x86-64-v2" "nehalem" _content "${_content}")
+      file(WRITE "${_file}" "${_content}")
+    else()
+      message(WARNING "Vectorscan patch target not found at ${_file}")
+    endif()
+  endforeach()
+endif()

src/hyperscan/extension.c

@@ -344,6 +344,114 @@ static int Database_init(Database *self, PyObject *args, PyObject *kwds)
   return 0;
 }
 
+static PyObject *escape_utf8_non_ascii(PyObject *obytes)
+{
+  char *bytes;
+  Py_ssize_t bytes_len;
+  if (PyBytes_AsStringAndSize(obytes, &bytes, &bytes_len) == -1) {
+    return NULL;
+  }
+
+  int has_non_ascii = 0;
+  for (Py_ssize_t i = 0; i < bytes_len; i++) {
+    if (((unsigned char *)bytes)[i] > 0x7f) {
+      has_non_ascii = 1;
+      break;
+    }
+  }
+  if (!has_non_ascii) {
+    return Py_NewRef(obytes);
+  }
+
+  PyObject *unicode = PyUnicode_DecodeUTF8(bytes, bytes_len, "strict");
+  if (unicode == NULL) {
+    PyErr_Clear();
+    return Py_NewRef(obytes);
+  }
+
+  Py_ssize_t chars_len = PyUnicode_GET_LENGTH(unicode);
+  Py_ssize_t escaped_len = 0;
+  for (Py_ssize_t i = 0; i < chars_len; i++) {
+    Py_UCS4 ch = PyUnicode_ReadChar(unicode, i);
+    if (ch == (Py_UCS4)-1) {
+      Py_DECREF(unicode);
+      return NULL;
+    }
+
+    Py_ssize_t needed = 1;
+    if (ch > 0x7f) {
+      Py_UCS4 value = ch;
+      Py_ssize_t hex_digits = 1;
+      while (value >>= 4) {
+        hex_digits++;
+      }
+      needed = hex_digits + 4;  // \x{...}
+    }
+
+    if (escaped_len > PY_SSIZE_T_MAX - needed) {
+      Py_DECREF(unicode);
+      PyErr_NoMemory();
+      return NULL;
+    }
+    escaped_len += needed;
+  }
+
+  PyObject *result = PyBytes_FromStringAndSize(NULL, escaped_len);
+  if (result == NULL) {
+    Py_DECREF(unicode);
+    return NULL;
+  }
+
+  char *escaped = PyBytes_AS_STRING(result);
+  Py_ssize_t write_pos = 0;
+  static const char hex[] = "0123456789abcdef";
+  for (Py_ssize_t i = 0; i < chars_len; i++) {
+    Py_UCS4 ch = PyUnicode_ReadChar(unicode, i);
+    if (ch == (Py_UCS4)-1) {
+      Py_DECREF(result);
+      Py_DECREF(unicode);
+      return NULL;
+    }
+    if (ch <= 0x7f) {
+      escaped[write_pos++] = (char)ch;
+      continue;
+    }
+
+    escaped[write_pos++] = '\\';
+    escaped[write_pos++] = 'x';
+    escaped[write_pos++] = '{';
+    int shift = 0;
+    for (Py_UCS4 value = ch >> 4; value != 0; value >>= 4) {
+      shift += 4;
+    }
+    for (; shift >= 0; shift -= 4) {
+      escaped[write_pos++] = hex[(ch >> shift) & 0xf];
+    }
+    escaped[write_pos++] = '}';
+  }
+
+  Py_DECREF(unicode);
+  return result;
+}
+
+static void free_compile_inputs(
+  const char **expressions,
+  uint32_t *flags,
+  uint32_t *ids,
+  PyObject **expression_refs,
+  uint64_t elements)
+{
+  if (expression_refs != NULL) {
+    for (uint64_t i = 0; i < elements; i++) {
+      Py_XDECREF(expression_refs[i]);
+    }
+    free(expression_refs);
+  }
+  free(expressions);
+  free(flags);
+  free(ids);
+}
+
 static PyObject *Database_compile(
   Database *self, PyObject *args, PyObject *kwds)
 {
@@ -392,9 +500,10 @@ static PyObject *Database_compile(
 
   PyObject *oexpr = NULL;
   PyObject *oid = NULL;
-  const char **expressions;
-  uint32_t *flags;
-  uint32_t *ids;
+  const char **expressions = NULL;
+  PyObject **expression_refs = NULL;
+  uint32_t *flags = NULL;
+  uint32_t *ids = NULL;
   size_t *lens;
   uint32_t globalflag;
 
@@ -408,6 +517,10 @@ static PyObject *Database_compile(
   if (expressions == NULL)
     goto memory_error;
 
+  expression_refs = calloc(elements, sizeof(PyObject *));
+  if (expression_refs == NULL)
+    goto memory_error;
+
   flags = malloc(elements * sizeof(uint32_t));
   if (flags == NULL)
     goto memory_error;
@@ -426,6 +539,10 @@ static PyObject *Database_compile(
     uint32_t expr_id;
 
     oexpr = PySequence_ITEM(oexpressions, i);
+    if (oexpr == NULL) {
+      break;
+    }
+    expression_refs[i] = oexpr;
 
     // Handle both bytes and unicode strings
     if (PyBytes_Check(oexpr)) {
@@ -440,6 +557,7 @@ static PyObject *Database_compile(
       // Replace the original object with the encoded version
       Py_DECREF(oexpr);
       oexpr = temp_bytes;
+      expression_refs[i] = oexpr;
     } else {
       PyErr_SetString(PyExc_TypeError, "expressions must be bytes or str");
       break;
@@ -468,11 +586,22 @@ static PyObject *Database_compile(
       expr_flags = globalflag;
     }
 
+    if (expr_flags & HS_FLAG_UTF8) {
+      PyObject *escaped_expr = escape_utf8_non_ascii(oexpr);
+      if (escaped_expr == NULL) {
+        break;
+      }
+      Py_DECREF(oexpr);
+      oexpr = escaped_expr;
+      expression_refs[i] = oexpr;
+      expression = PyBytes_AsString(oexpr);
+      if (PyErr_Occurred())
+        break;
+    }
+
     expressions[i] = expression;
     ids[i] = expr_id;
     flags[i] = expr_flags;
-
-    Py_XDECREF(oexpr);
   }
 
   if (oflag != Py_None)
@@ -515,9 +644,11 @@ static PyObject *Database_compile(
       &hs_compile_err);
     free(lens);
     Py_END_ALLOW_THREADS;
-    free(expressions);
-    free(flags);
-    free(ids);
+    free_compile_inputs(expressions, flags, ids, expression_refs, elements);
+    expressions = NULL;
+    flags = NULL;
+    ids = NULL;
+    expression_refs = NULL;
 
     if (hs_err != HS_SUCCESS) {
       PyErr_Format(
@@ -544,9 +675,11 @@ static PyObject *Database_compile(
         &self->ch_db,
         &ch_compile_err);
       Py_END_ALLOW_THREADS;
-      free(expressions);
-      free(flags);
-      free(ids);
+      free_compile_inputs(expressions, flags, ids, expression_refs, elements);
+      expressions = NULL;
+      flags = NULL;
+      ids = NULL;
+      expression_refs = NULL;
       HANDLE_CHIMERA_ERR(ch_err, NULL);
     } else {
       if (oext != Py_None) {
@@ -589,9 +722,11 @@ static PyObject *Database_compile(
         &hs_compile_err);
       Py_END_ALLOW_THREADS;
 
-      free(expressions);
-      free(flags);
-      free(ids);
+      free_compile_inputs(expressions, flags, ids, expression_refs, elements);
+      expressions = NULL;
+      flags = NULL;
+      ids = NULL;
+      expression_refs = NULL;
       if (hs_err != HS_SUCCESS) {
         PyErr_SetString(HyperscanError, hs_compile_err->message);
         hs_free_compile_error(hs_compile_err);
@@ -631,13 +766,12 @@ static PyObject *Database_compile(
   HS_LOCK_RETURN(Py_NewRef(Py_None));
 
 memory_error:
+  free_compile_inputs(expressions, flags, ids, expression_refs, elements);
   PyErr_NoMemory();
   HS_LOCK_RETURN_NULL();
 
 python_error:
-  free(expressions);
-  free(flags);
-  free(ids);
+  free_compile_inputs(expressions, flags, ids, expression_refs, elements);
   HS_LOCK_RETURN_NULL();
 }
 

tests/test_hyperscan.py

@@ -426,15 +426,9 @@ def test_unicode_expressions():
     valid UTF-8 patterns. This was caused by PCRE being built without UTF-8 support
     in v0.7.9+ when the build system switched from setup.py to CMake.
 
-    Note on HS_FLAG_UTF8:
-    We avoid using HS_FLAG_UTF8 by default due to known Hyperscan/Vectorscan
-    limitations and bugs:
-    - intel/hyperscan#57: UTF-8 match failures with \\Q...\\E patterns
-    - intel/hyperscan#133: Parser bug with Ragel v7 incorrectly rejecting valid UTF-8
-    - intel/hyperscan#163: Performance issues with UTF-8 + case-insensitive flags
-
-    Unicode patterns work correctly without HS_FLAG_UTF8 when PCRE has proper
-    UTF-8 support, which is what our CMake fixes provide.
+    The HS_FLAG_UTF8 cases cover issue #274, where x86/x86_64 wheels compiled
+    Vectorscan with signed-char semantics and rejected valid multibyte UTF-8
+    literals such as é and €.
     """
     complex_patterns = [
         r"<span\s+.*>السلام عليكم\s<\/span>",
@@ -454,10 +448,13 @@ def test_unicode_expressions():
     db_bytes.compile(expressions=bytes_patterns)
 
     db_utf8 = hyperscan.Database()
-    try:
-        db_utf8.compile(expressions=simple_patterns, flags=hyperscan.HS_FLAG_UTF8)
-    except Exception as e:
-        pytest.skip(f"HS_FLAG_UTF8 validation failed (known limitation): {e}")
+    db_utf8.compile(expressions=simple_patterns, flags=hyperscan.HS_FLAG_UTF8)
+
+    db_utf8_latin = hyperscan.Database()
+    db_utf8_latin.compile(
+        expressions=["é".encode("utf-8"), "€".encode("utf-8")],
+        flags=hyperscan.HS_FLAG_UTF8,
+    )
 
     test_text = '<span class="greeting">السلام عليكم </span>'