From 2c76f69ccf38d4335d76a1dbafeac12cea0f1fbd Mon Sep 17 00:00:00 2001 From: Cheruku Sri Charan Reddy <185475071+sricharanreddycheruku@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:25:00 +0000 Subject: [PATCH] fix: retain expression buffers through native compilation --- docs/usage.md | 4 + src/hyperscan/extension.c | 37 ++++++-- tests/test_compile_lifetimes.py | 157 ++++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 10 deletions(-) create mode 100644 tests/test_compile_lifetimes.py diff --git a/docs/usage.md b/docs/usage.md index a4ec38d..cbc0307 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -35,6 +35,10 @@ The only required parameter to ``hyperscan.Database`` is rest of the parameters, including **ids**, **elements**, and **flags** are optional. +Expression items may be ``bytes`` or ``str``. Strings are encoded as UTF-8 before +compilation. The wrapper retains each expression's backing bytes until the native +compiler returns, including items generated by a custom sequence. + ```python import hyperscan diff --git a/src/hyperscan/extension.c b/src/hyperscan/extension.c index e072d10..426ce61 100644 --- a/src/hyperscan/extension.c +++ b/src/hyperscan/extension.c @@ -352,7 +352,7 @@ static PyObject *Database_compile( PyObject *oexpressions; PyObject *oflags = Py_None; - PyObject *oflag = Py_None; + PyObject *oflag = NULL; PyObject *oids = Py_None; PyObject *oext = Py_None; uint32_t literal = 0; @@ -392,9 +392,10 @@ static PyObject *Database_compile( PyObject *oexpr = NULL; PyObject *oid = NULL; - const char **expressions; - uint32_t *flags; - uint32_t *ids; + PyObject *expression_objects = NULL; + const char **expressions = NULL; + uint32_t *flags = NULL; + uint32_t *ids = NULL; size_t *lens; uint32_t globalflag; @@ -416,6 +417,10 @@ static PyObject *Database_compile( if (ids == NULL) goto memory_error; + expression_objects = PyList_New(elements); + if (expression_objects == NULL) + goto python_error; + globalflag = (oflags == Py_None ? 0 : PyLong_AsUnsignedLong(oflags)); PyErr_Clear(); @@ -426,6 +431,8 @@ static PyObject *Database_compile( uint32_t expr_id; oexpr = PySequence_ITEM(oexpressions, i); + if (oexpr == NULL) + break; // Handle both bytes and unicode strings if (PyBytes_Check(oexpr)) { @@ -450,6 +457,8 @@ static PyObject *Database_compile( if (PyObject_IsTrue(oids)) { oid = PySequence_ITEM(oids, i); + if (oid == NULL) + break; expr_id = PyLong_AsUnsignedLong(oid); if (PyErr_Occurred()) break; @@ -472,13 +481,15 @@ static PyObject *Database_compile( ids[i] = expr_id; flags[i] = expr_flags; - Py_XDECREF(oexpr); + // Keep the backing bytes alive until the native compiler returns. + PyList_SET_ITEM(expression_objects, i, oexpr); + oexpr = NULL; + Py_CLEAR(oid); + Py_CLEAR(oflag); } - if (oflag != Py_None) - Py_XDECREF(oflag); - if (oid != Py_None) - Py_XDECREF(oid); + Py_CLEAR(oflag); + Py_CLEAR(oid); if (PyErr_Occurred()) { goto python_error; @@ -518,6 +529,7 @@ static PyObject *Database_compile( free(expressions); free(flags); free(ids); + Py_CLEAR(expression_objects); if (hs_err != HS_SUCCESS) { PyErr_Format( @@ -547,6 +559,7 @@ static PyObject *Database_compile( free(expressions); free(flags); free(ids); + Py_CLEAR(expression_objects); HANDLE_CHIMERA_ERR(ch_err, NULL); } else { if (oext != Py_None) { @@ -592,6 +605,7 @@ static PyObject *Database_compile( free(expressions); free(flags); free(ids); + Py_CLEAR(expression_objects); if (hs_err != HS_SUCCESS) { PyErr_SetString(HyperscanError, hs_compile_err->message); hs_free_compile_error(hs_compile_err); @@ -632,9 +646,12 @@ static PyObject *Database_compile( memory_error: PyErr_NoMemory(); - HS_LOCK_RETURN_NULL(); python_error: + Py_XDECREF(oexpr); + Py_XDECREF(oflag); + Py_XDECREF(oid); + Py_XDECREF(expression_objects); free(expressions); free(flags); free(ids); diff --git a/tests/test_compile_lifetimes.py b/tests/test_compile_lifetimes.py new file mode 100644 index 0000000..4a5c26f --- /dev/null +++ b/tests/test_compile_lifetimes.py @@ -0,0 +1,157 @@ +import gc +import sys + +import pytest + +import hyperscan + + +@pytest.fixture +def reference_count(): + getrefcount = getattr(sys, "getrefcount", None) + if getrefcount is None: + pytest.skip("Reference counts are specific to CPython") + return getrefcount + + +@pytest.mark.parametrize( + "chimera,literal", [(False, False), (False, True), (True, False)] +) +@pytest.mark.parametrize( + "expressions", + [ + ["alpha", "omega"], + [b"alpha", "omega"], + ["alpha", b"omega"], + [b"alpha", b"omega"], + ["caf\u00e9", "th\u00e9"], + ], +) +def test_compile_keeps_each_encoded_expression(chimera, literal, expressions): + database = hyperscan.Database( + chimera=chimera, + mode=hyperscan.CH_MODE_GROUPS if chimera else hyperscan.HS_MODE_BLOCK, + ) + database.compile(expressions=expressions, ids=[7, 11], literal=literal) + matches = [] + data = ( + b"alpha omega" + if expressions[0] != "caf\u00e9" + else "caf\u00e9 th\u00e9".encode() + ) + database.scan( + data, + match_event_handler=lambda id, start, end, *args: matches.append((id, end)), + ) + first_end = 5 + second_end = len(data) + assert sorted(matches) == [(7, first_end), (11, second_end)] + + +class GeneratedExpressions: + def __init__(self, as_bytes): + self.as_bytes = as_bytes + + def __len__(self): + return 2 + + def __getitem__(self, index): + expression = ["alpha", "omega"][index] + return expression.encode() if self.as_bytes else expression + + +@pytest.mark.parametrize( + "chimera,literal", [(False, False), (False, True), (True, False)] +) +@pytest.mark.parametrize("as_bytes", [False, True]) +def test_compile_keeps_temporary_sequence_items(chimera, literal, as_bytes): + database = hyperscan.Database( + chimera=chimera, + mode=hyperscan.CH_MODE_GROUPS if chimera else hyperscan.HS_MODE_BLOCK, + ) + database.compile(expressions=GeneratedExpressions(as_bytes), literal=literal) + matches = [] + database.scan( + b"alpha omega", + match_event_handler=lambda id, start, end, *args: matches.append((id, end)), + ) + assert sorted(matches) == [(0, 5), (1, 11)] + + +class CompileItemError(Exception): + pass + + +@pytest.mark.parametrize("argument", ["expressions", "ids", "flags"]) +def test_compile_propagates_sequence_item_errors(argument): + class BrokenSequence: + def __len__(self): + return 2 + + def __getitem__(self, index): + if index == 1: + raise CompileItemError("second item is unavailable") + return b"alpha" if argument == "expressions" else 0 + + kwargs = {"expressions": [b"alpha", b"omega"], argument: BrokenSequence()} + database = hyperscan.Database() + with pytest.raises(CompileItemError, match="second item is unavailable"): + database.compile(**kwargs) + + +@pytest.mark.parametrize("invalid_expression", [None, 42, "\ud800"]) +def test_compile_releases_expression_references_on_preparation_error( + invalid_expression, + reference_count, +): + expression = bytes(bytearray(b"alpha")) + before = reference_count(expression) + database = hyperscan.Database() + error = UnicodeEncodeError if isinstance(invalid_expression, str) else TypeError + with pytest.raises(error): + database.compile(expressions=[expression, invalid_expression]) + gc.collect() + assert reference_count(expression) == before + + +@pytest.mark.parametrize( + "chimera,literal", [(False, False), (False, True), (True, False)] +) +def test_compile_releases_expression_references_after_success( + chimera, literal, reference_count +): + expression = bytes(bytearray(b"alpha")) + before = reference_count(expression) + database = hyperscan.Database( + chimera=chimera, + mode=hyperscan.CH_MODE_GROUPS if chimera else hyperscan.HS_MODE_BLOCK, + ) + database.compile(expressions=[expression, "omega"], literal=literal) + gc.collect() + assert reference_count(expression) == before + + +@pytest.mark.parametrize("chimera", [False, True]) +def test_compile_releases_expression_references_after_native_error( + chimera, reference_count +): + expression = bytes(bytearray(b"alpha")) + before = reference_count(expression) + database = hyperscan.Database( + chimera=chimera, + mode=hyperscan.CH_MODE_GROUPS if chimera else hyperscan.HS_MODE_BLOCK, + ) + with pytest.raises(hyperscan.error): + database.compile(expressions=[expression, "("]) + gc.collect() + assert reference_count(expression) == before + + +@pytest.mark.parametrize("argument", ["ids", "flags"]) +def test_compile_releases_each_metadata_item(argument, reference_count): + first = int("1048576") if argument == "ids" else int("1024") + second = int("1048577") if argument == "ids" else int("1025") + before = (reference_count(first), reference_count(second)) + database = hyperscan.Database() + database.compile(expressions=[b"alpha", b"omega"], **{argument: [first, second]}) + assert (reference_count(first), reference_count(second)) == before