wasm: enable Fluent Typed Scripting in browser artifacts

cmake/wasm-browser.cmake

@@ -68,6 +68,7 @@ set(FLB_WASM_BROWSER_PLUGINS
   FLB_IN_STORAGE_BACKLOG
   FLB_PROCESSOR_CONTENT_MODIFIER
   FLB_PROCESSOR_CUMULATIVE_TO_DELTA
+  FLB_PROCESSOR_FTS
   FLB_PROCESSOR_LABELS
   FLB_PROCESSOR_METRICS_SELECTOR
   FLB_PROCESSOR_OPENTELEMETRY_ENVELOPE
@@ -97,7 +98,7 @@ set(FLB_WASM_BROWSER_PLUGINS
   FLB_OUT_STDOUT)
 
 if(NOT FLB_WASM_LUA)
-  list(REMOVE_ITEM FLB_WASM_BROWSER_PLUGINS FLB_FILTER_LUA)
+  list(REMOVE_ITEM FLB_WASM_BROWSER_PLUGINS FLB_FILTER_LUA FLB_PROCESSOR_FTS)
 endif()
 
 # Compile every dependency with the same threading ABI. The embedding API

plugins/CMakeLists.txt

@@ -347,7 +347,7 @@ REGISTER_IN_PLUGIN("in_random")
 # ==========
 REGISTER_PROCESSOR_PLUGIN("processor_content_modifier")
 REGISTER_PROCESSOR_PLUGIN("processor_cumulative_to_delta")
-if(FLB_LUAJIT AND FLB_YYJSON)
+if((FLB_LUAJIT OR FLB_WASM_LUA) AND FLB_YYJSON)
   REGISTER_PROCESSOR_PLUGIN("processor_fts")
 endif()
 REGISTER_PROCESSOR_PLUGIN("processor_labels")

plugins/processor_fts/README.md

@@ -7,6 +7,11 @@ attributes are exposed as properties instead of raw metadata.
 
 FTS supports logs, metrics, and traces. Its configuration name is `fts`.
 
+The experimental browser WASM profile includes FTS when `FLB_WASM_LUA=ON`
+(the default). It uses the portable Lua 5.4 interpreter; `jit: true` is rejected.
+Typed values, restricted policy libraries and instruction/memory limits remain
+available. Setting `FLB_WASM_LUA=OFF` disables both FTS and the Lua filter.
+
 | Signal  | Callback          | Called once per |
 | ------- | ----------------- | --------------- |
 | Logs    | `process(log)`    | log record      |

plugins/processor_fts/fts.h

@@ -24,6 +24,19 @@
 #include <fluent-bit/flb_luajit.h>
 #include <cfl/cfl_variant.h>
 
+/* Typed views keep their parent in a userdata environment/uservalue table. */
+#if LUA_VERSION_NUM >= 502
+#define fts_lua_rawlen lua_rawlen
+#define fts_lua_getuservalue lua_getuservalue
+#define fts_lua_setuservalue lua_setuservalue
+#define fts_lua_register(l, methods) luaL_setfuncs(l, methods, 0)
+#else
+#define fts_lua_rawlen lua_objlen
+#define fts_lua_getuservalue lua_getfenv
+#define fts_lua_setuservalue lua_setfenv
+#define fts_lua_register(l, methods) luaL_register(l, NULL, methods)
+#endif
+
 #define FTS_MAX_DEPTH 64
 
 /* Each signal owns its representation, validation and commit semantics. */

plugins/processor_fts/fts_runtime.c

@@ -17,14 +17,17 @@
  *  limitations under the License.
  */
 
+#include <fluent-bit/flb_info.h>
+#ifdef FLB_HAVE_LUAJIT
 #include <luajit.h>
+#endif
 #include <fluent-bit/flb_config.h>
 #include <fluent-bit/flb_plugin.h>
 #include <fluent-bit/flb_input.h>
 #include <fluent-bit/flb_output.h>
 #include <limits.h>
 
-#ifdef FLB_HAVE_SANITIZE_ADDRESS
+#if defined(FLB_HAVE_SANITIZE_ADDRESS) && defined(FLB_HAVE_LUAJIT)
 #include <sanitizer/asan_interface.h>
 #endif
 
@@ -41,12 +44,12 @@
  */
 #define FTS_UNWOUND_STACK_SIZE (256 * 1024)
 
-#ifdef FLB_HAVE_SANITIZE_ADDRESS
+#if defined(FLB_HAVE_SANITIZE_ADDRESS) && defined(FLB_HAVE_LUAJIT)
 __attribute__((noinline))
 #endif
 void fts_sanitizer_forget_unwound_frames(void)
 {
-#ifdef FLB_HAVE_SANITIZE_ADDRESS
+#if defined(FLB_HAVE_SANITIZE_ADDRESS) && defined(FLB_HAVE_LUAJIT)
     char *top;
 
     top = (char *) __builtin_frame_address(0);
@@ -210,7 +213,11 @@ int fts_load_helpers(lua_State *l, const char *code, size_t size, const char *na
     }
 
     lua_rawgeti(l, LUA_REGISTRYINDEX, ctx->helpers_env_ref);
+#if LUA_VERSION_NUM >= 502
+    lua_setupvalue(l, -2, 1);
+#else
     lua_setfenv(l, -2);
+#endif
     if (lua_pcall(l, 0, 0, 0) != 0) {
         fts_sanitizer_forget_unwound_frames();
         return -1;
@@ -223,13 +230,19 @@ static int initialize_policy(lua_State *l)
     struct fts_runtime *ctx;
     const char *blocked[] = {"io", "os", "debug", "jit", "ffi", "package", "coroutine",
                             "module", "load", "loadfile", "loadstring", "dofile",
-                            "newproxy", "xpcall", "getfenv", "setfenv", "print", NULL};
+                            "newproxy", "xpcall", "getfenv", "setfenv", "print", "warn", NULL};
     int i;
     int ret;
     int environment;
 
     ctx = lua_touserdata(l, 1);
+#if LUA_VERSION_NUM >= 502
+    lua_pushglobaltable(l);
+    copy_table(l, -1);
+    lua_remove(l, -2);
+#else
     copy_table(l, LUA_GLOBALSINDEX);
+#endif
     environment = lua_gettop(l);
 
     /* tables used by validation must also be independent of policy globals */
@@ -338,9 +351,13 @@ int fts_runtime_initialize(struct fts_runtime *ctx, struct flb_config *config)
     }
     l = ctx->vm->state;
 
+#ifdef FLB_HAVE_LUAJIT
     ret = luaJIT_setmode(l, 0, LUAJIT_MODE_ENGINE | (ctx->jit ? LUAJIT_MODE_ON : LUAJIT_MODE_OFF));
+#else
+    ret = 0;
+#endif
     if (ctx->jit && !ret) {
-        lua_pushliteral(l, "JIT is unavailable in this LuaJIT build");
+        lua_pushliteral(l, "JIT is unavailable in this Lua runtime");
         return FLB_PROCESSOR_FAILURE;
     }
 

plugins/processor_fts/fts_values.c

@@ -69,7 +69,7 @@ static struct fts_value *check_value(lua_State *l, int index)
     lua_getallocf(l, (void **) &ctx);
 
     value = NULL;
-    if (lua_type(l, index) == LUA_TUSERDATA && lua_objlen(l, index) == sizeof(*value)) {
+    if (lua_type(l, index) == LUA_TUSERDATA && fts_lua_rawlen(l, index) == sizeof(*value)) {
         value = lua_touserdata(l, index);
     }
 
@@ -448,7 +448,7 @@ struct cfl_variant *fts_resolve_value(lua_State *l, int index, int depth)
         luaL_error(l, "typed value nesting exceeds limit");
     }
 
-    lua_getfenv(l, index);
+    fts_lua_getuservalue(l, index);
     lua_rawgeti(l, -1, 1);
     parent = fts_resolve_value(l, -1, depth + 1);
     lua_pop(l, 1);
@@ -538,7 +538,7 @@ static void push_view(lua_State *l, int parent, int key)
     lua_rawseti(l, -2, 1);
     lua_pushvalue(l, key);
     lua_rawseti(l, -2, 2);
-    lua_setfenv(l, -2);
+    fts_lua_setuservalue(l, -2);
 }
 
 static int value_gc(lua_State *l)
@@ -1013,7 +1013,7 @@ static void detach_root(lua_State *l, int index, int depth)
             luaL_error(l, "typed value nesting exceeds limit");
         }
 
-        lua_getfenv(l, index);
+        fts_lua_getuservalue(l, index);
         lua_rawgeti(l, -1, 1);
         detach_root(l, lua_gettop(l), depth + 1);
         lua_pop(l, 2);
@@ -1621,7 +1621,7 @@ static int redact(lua_State *l)
         root_value(l, 1)->depth_changed = FLB_TRUE;
     }
 
-    count = lua_objlen(l, 2);
+    count = fts_lua_rawlen(l, 2);
     lua_newtable(l);
     keys = lua_gettop(l);
 
@@ -2079,9 +2079,9 @@ int fts_register_values(lua_State *l)
     ctx->value_metatable = lua_topointer(l, -1);
     lua_pushvalue(l, -1);
     ctx->value_metatable_ref = luaL_ref(l, LUA_REGISTRYINDEX);
-    luaL_register(l, NULL, value_methods);
+    fts_lua_register(l, value_methods);
     lua_newtable(l);
-    luaL_register(l, NULL, value_methods + 1);
+    fts_lua_register(l, value_methods + 1);
     lua_setfield(l, -2, "__index");
     lua_pushliteral(l, "typed value");
     lua_setfield(l, -2, "__metatable");

sdk/browser/README.md

@@ -184,6 +184,14 @@ unsanitized browser assets, not a production release or an npm publication.
 Artifact availability does not imply the subsequent browser tests or the ASan
 job passed; check the workflow results before choosing a build.
 
+## Typed scripting processor
+
+The default browser artifact includes the `fts` processor using portable Lua 5.4.
+Supply a policy with `writeFile('/config/policy.lua', source)` and reference that
+path from the YAML processor's `script` property. `jit: true` is unavailable in
+browser builds. See the [FTS guide](https://github.com/fluent/fluent-bit/blob/master/plugins/processor_fts/README.md) for
+policies, typed values and resource limits.
+
 ## Release gates
 
 Before promoting this experimental API: run ASan lifecycle/network/storage and

tests/internal/fts.c

@@ -1,4 +1,4 @@
-/* Fluent Bit: native log-policy regression tests, Apache-2.0. */
+/* Fluent Bit: log-policy regression tests, Apache-2.0. */
 #include <fluent-bit/flb_config.h>
 #include <fluent-bit/flb_processor.h>
 #include <fluent-bit/flb_conditionals.h>
@@ -457,6 +457,7 @@ static void garbage_collector_is_private(void)
                0, "drop", 6, 0, 0, NULL, NULL);
 }
 
+#ifdef FLB_HAVE_LUAJIT
 static void trusted_policy_jit_compiles(void)
 {
     struct flb_config *config;
@@ -494,6 +495,28 @@ static void trusted_policy_jit_compiles(void)
     flb_config_exit(config);
 }
 
+#else
+static void portable_runtime_rejects_jit(void)
+{
+    struct flb_config *config;
+    struct flb_processor *processor;
+    struct flb_processor_unit *unit;
+
+    config = flb_config_init();
+    TEST_ASSERT(config != NULL);
+    processor = flb_processor_create(config, "fts_no_jit_test", NULL, 0);
+    TEST_ASSERT(processor != NULL);
+    unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts");
+    TEST_ASSERT(unit != NULL);
+    TEST_CHECK(flb_processor_unit_set_property_str(unit, "jit", "true") == 0);
+    TEST_CHECK(flb_processor_unit_set_property_str(unit, "instruction_limit", "0") == 0);
+    TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", "function process(log) end") == 0);
+    TEST_CHECK(flb_processor_init(processor) != 0);
+    flb_processor_destroy(processor);
+    flb_config_exit(config);
+}
+#endif
+
 static void native_histogram_buckets_are_preserved(void)
 {
     struct flb_config *config;
@@ -871,7 +894,7 @@ static void memory_exhaustion_does_not_poison_later_callbacks(void)
     TEST_CHECK(fts_runtime_call(runtime, NULL, 1) == LUA_ERRMEM);
     lua_settop(l, 0);
     TEST_CHECK(fts_runtime_call(runtime, runtime, 1) == 0);
-    TEST_CHECK(lua_objlen(l, -1) == 300000);
+    TEST_CHECK(fts_lua_rawlen(l, -1) == 300000);
     lua_settop(l, 0);
     flb_processor_destroy(processor);
     flb_config_exit(config);
@@ -1482,7 +1505,11 @@ TEST_LIST = {
     {"schema_promotion_is_record_local", schema_promotion_is_record_local},
     {"reserved_timestamps_are_record_errors", reserved_timestamps_are_record_errors},
     {"small_record_batch_uses_wire_budget", small_record_batch_uses_wire_budget},
+#ifdef FLB_HAVE_LUAJIT
     {"trusted_policy_jit_compiles", trusted_policy_jit_compiles},
+#else
+    {"portable_runtime_rejects_jit", portable_runtime_rejects_jit},
+#endif
     {"native_histogram_buckets_are_preserved", native_histogram_buckets_are_preserved},
     {"native_metric_scalar_preservation", native_metric_scalar_preservation},
     {"kept_native_family_no_merge", kept_native_family_does_not_merge_into_outputs},

tests/wasm/CMakeLists.txt

@@ -37,6 +37,19 @@ if(FLB_WASM_LUA)
     "-sASSERTIONS=2 -sEXIT_RUNTIME=1")
   add_test(NAME flb-wasm-lua COMMAND flb-wasm-lua --no-exec)
   set_tests_properties(flb-wasm-lua PROPERTIES TIMEOUT 30)
+
+  if(FLB_PROCESSOR_FTS)
+    add_executable(flb-wasm-fts "${PROJECT_SOURCE_DIR}/tests/internal/fts.c")
+    target_include_directories(flb-wasm-fts PRIVATE
+      "${CMAKE_CURRENT_BINARY_DIR}" "${PROJECT_SOURCE_DIR}/tests/internal")
+    target_compile_definitions(flb-wasm-fts PRIVATE FLB_TEST_INIT_NOP FLB_TEST_FINI_NOP)
+    target_link_libraries(flb-wasm-fts fluent-bit-static)
+    set_target_properties(flb-wasm-fts PROPERTIES LINK_FLAGS
+      "-sASSERTIONS=2 -sEXIT_RUNTIME=1 -sALLOW_MEMORY_GROWTH=1 -sSTACK_SIZE=262144")
+    add_test(NAME flb-wasm-fts COMMAND flb-wasm-fts --no-exec)
+    # The large-object regression is substantially slower with WASM ASan.
+    set_tests_properties(flb-wasm-fts PROPERTIES TIMEOUT 180)
+  endif()
 endif()
 
 add_test(NAME flb-wasm-build-profile

tests/wasm/browser_smoke.py

@@ -80,7 +80,8 @@ def main():
     parser.add_argument("--build-dir", type=Path, default=DEFAULT_BUILD_DIR)
     parser.add_argument("--target", choices=("flb-wasm-dependencies", "co-wasm-test",
                                              "flb-wasm-pipeline", "flb-wasm-events", "flb-wasm-event-fiber",
-                                             "flb-wasm-storage", "flb-wasm-yaml", "flb-wasm-lua", "flb-wasm-http",
+                                             "flb-wasm-storage", "flb-wasm-yaml", "flb-wasm-lua", "flb-wasm-fts",
+                                             "flb-wasm-http",
                                              "flb-wasm-http-lifecycle", "flb-wasm-http-ng", "mk-test-wasm-events"),
                         default="flb-wasm-dependencies")
     parser.add_argument("--browser", default=shutil.which("chromium") or shutil.which("google-chrome"))
@@ -111,9 +112,9 @@ def main():
                     for phase in phases:
                         page.goto(f"http://127.0.0.1:{server.server_port}/?phase={phase}"
                                   if phase else f"http://127.0.0.1:{server.server_port}/" +
-                                  ("?arg=--no-exec" if args.target == "flb-wasm-lua" else ""))
+                                  ("?arg=--no-exec" if args.target in ("flb-wasm-lua", "flb-wasm-fts") else ""))
                         page.wait_for_function("document.body.dataset.result !== 'pending'",
-                                               timeout=45000)
+                                               timeout=180000 if args.target == "flb-wasm-fts" else 45000)
                         passed = passed and page.get_attribute("body", "data-result") == "passed"
                         print(page.locator("#output").inner_text())
                         if not passed:

tests/wasm/check_profile.cmake

@@ -31,12 +31,15 @@ function(check_profile name expected_error)
   message(STATUS "${name}: passed")
 endfunction()
 
-check_profile(defaults "")
+check_profile(defaults "" -DEXPECT_FTS=ON)
 check_profile(old_sdk "FLB_WASM_BROWSER requires Emscripten 6.0.9" -DEMSCRIPTEN_VERSION=5.0.7)
 check_profile(unvalidated_sdk "FLB_WASM_BROWSER requires Emscripten 6.0.9" -DEMSCRIPTEN_VERSION=6.0.10)
-check_profile(minimal "" -DFLB_MINIMAL=ON)
+check_profile(minimal "" -DFLB_MINIMAL=ON -DEXPECT_FTS=OFF)
 check_profile(disable_filter "" -DFLB_FILTER_GREP=OFF)
-check_profile(disable_lua "" -DFLB_WASM_LUA=OFF -DFLB_FILTER_LUA=OFF)
+check_profile(disable_lua "" -DFLB_WASM_LUA=OFF -DFLB_FILTER_LUA=OFF -DFLB_PROCESSOR_FTS=OFF -DEXPECT_FTS=OFF)
+check_profile(disable_fts "" -DFLB_PROCESSOR_FTS=OFF -DEXPECT_FTS=OFF)
+check_profile(reject_fts_without_backend "FLB_PROCESSOR_FTS is not supported by the browser build profile"
+  -DFLB_WASM_LUA=OFF -DFLB_PROCESSOR_FTS=ON)
 check_profile(reject_lua_without_backend "FLB_FILTER_LUA is not supported by the browser build profile"
   -DFLB_WASM_LUA=OFF -DFLB_FILTER_LUA=ON)
 check_profile(native_defaults "" -DPROFILE_TEST_MODE=native)

tests/wasm/profile/CMakeLists.txt

@@ -38,7 +38,15 @@ else()
   if(NOT FLB_MINIMAL AND (NOT FLB_IN_LIB OR NOT FLB_OUT_LIB OR NOT FLB_FILTER_MODIFY))
     message(FATAL_ERROR "Local processing plugin defaults were lost")
   endif()
-  if(FLB_MINIMAL AND (FLB_IN_LIB OR FLB_OUT_LIB OR FLB_FILTER_MODIFY))
+  if(DEFINED EXPECT_FTS)
+    if((EXPECT_FTS AND NOT FLB_PROCESSOR_FTS) OR (NOT EXPECT_FTS AND FLB_PROCESSOR_FTS))
+      message(FATAL_ERROR "FTS browser default or override was lost")
+    endif()
+  endif()
+  if(NOT FLB_WASM_LUA AND FLB_PROCESSOR_FTS)
+    message(FATAL_ERROR "FTS requires a Lua backend")
+  endif()
+  if(FLB_MINIMAL AND (FLB_IN_LIB OR FLB_OUT_LIB OR FLB_FILTER_MODIFY OR FLB_PROCESSOR_FTS))
     message(FATAL_ERROR "Minimal mode was ignored")
   endif()
 endif()