diff --git a/tests/nemo_text_processing/test_native_runtime.py b/tests/nemo_text_processing/test_native_runtime.py new file mode 100644 index 000000000..cb5f112bd --- /dev/null +++ b/tests/nemo_text_processing/test_native_runtime.py @@ -0,0 +1,12 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Expose native runtime regressions through the repository's pytest entry point.""" + +import pytest + +from tools.text_processing_deployment.native_runtime.test_byte_transducer import ByteTransducerTest +from tools.text_processing_deployment.native_runtime.test_text_processing import TextProcessingTest + +pytestmark = pytest.mark.run_only_on("CPU") + +__all__ = ["ByteTransducerTest", "TextProcessingTest"] diff --git a/tools/text_processing_deployment/native_runtime/.clang-format b/tools/text_processing_deployment/native_runtime/.clang-format new file mode 100644 index 000000000..f5e3ec5b1 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/.clang-format @@ -0,0 +1,2 @@ +BasedOnStyle: LLVM +ReflowComments: false diff --git a/tools/text_processing_deployment/native_runtime/LICENSE.sacremoses b/tools/text_processing_deployment/native_runtime/LICENSE.sacremoses new file mode 100644 index 000000000..ccf820205 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/LICENSE.sacremoses @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 alvations + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/text_processing_deployment/native_runtime/README.md b/tools/text_processing_deployment/native_runtime/README.md new file mode 100644 index 000000000..8b9c4f773 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/README.md @@ -0,0 +1,108 @@ +# Experimental native transducer runtime + +Keep Pynini/OpenFst for grammar construction and export byte-transition tables for native execution. +The C++17 evaluator has no OpenFst dependency and does not construct per-input FSTs. +Its complete English entry point includes Unicode whitespace collapse, bracket preprocessing, classification, tagged-text parsing, field permutations, verbalization, the postprocessing FST, Moses detokenization and punctuation restoration. + +This is a development prototype, not a supported backend. +The punctuation options enabled in the reference pipeline are implemented natively. +Compatibility targets the pinned English cached-lookahead reference. +Corpus differential tests and synthetic weighted-graph regressions check exact output; this does not establish compatibility with arbitrary grammars or library versions. +Queue selection includes unweighted LIFO and the reference's cached acyclicity. +The former unweighted counterexample is a passing regression, including cached-property and unused-weighted-branch variants. +Passing finite tests does not prove equivalence on every input. +The current evaluator supports standard tropical weights with byte labels on little-endian hosts; generated symbol labels are rejected. +Resource limits raise errors rather than silently pruning paths. +A runtime handle is mutable and must not be called concurrently. + +## Build and test + +Run from this directory: + +```sh +g++ -std=c++17 -O3 -Wall -Wextra -shared -fPIC byte_transducer.cc -o libtn_byte_transducer.so +g++ -std=c++17 -O3 -Wall -Wextra byte_transducer.cc normalize_stream.cc -o normalize_stream +``` + +Run the tests from the repository root: + +```sh +python -m unittest tools.text_processing_deployment.native_runtime.test_byte_transducer tools.text_processing_deployment.native_runtime.test_text_processing +``` + +The test needs Pynini for its independent OpenFst reference. +Tests check minimum-cost membership and exact outputs separately, including negative arcs, float rounding at merges, lookahead phase merges, projected-output traversal, unsupported cycles, UTF-8, literal brackets, failed verbalization and repeated calls. + +Export a normalizer and benchmark independent written/spoken JSONL references: + +```sh +python export_grammars.py --cache-dir CACHE --output-dir GRAMMARS +python benchmark.py --library ./libtn_byte_transducer.so --grammars GRAMMARS --input REFERENCE.jsonl --report REPORT.json --mismatches DIFFERENCES.jsonl +``` + +Each reference row contains written and spoken string fields. +The JSONL benchmark calls the complete native normalizer but retains Python input/output instrumentation. +The standalone normalize_stream executable reads and writes length-framed UTF-8 records with no Python processing or callbacks. +Invoke it as ./normalize_stream GRAMMARS INPUT.frames OUTPUT.frames; each frame is a four-byte little-endian byte count followed by that many UTF-8 bytes. +The driver fails on malformed input, unsupported text, no accepting path or I/O errors; output before an error is partial. +Source-token decoding, corpus selection and provenance/checkpoint integration are separate from this driver. +Use export_fst from byte_transducer.py to export existing tagger, verbalizer and postprocessor FSTs directly. +NativePipeline loads tagger.bin, verbalizer.bin and postprocessor.bin from a supplied directory. +Its normalize method performs complete English processing; normalize_core exposes the transducer stages for diagnostics. +normalize_batch uses one native call for a list of records, with no Python callbacks during normalization. +Batches use the same framing as the standalone driver, allow at most 64 MiB of input or output, and return no partial result on failure. +Embedded NUL characters follow Pynini's epsilon interpretation at transducer input. +They remain available in the original text for punctuation handling and failed-verbalization fallback; containing NUL does not discard a record. +The evaluator retains the original input's emptiness and trailing-epsilon phase when choosing tied paths. +The low-level direct-transducer mode still rejects NUL because its composition-filter tie ordering is not implemented; the complete English pipeline uses prepared lookahead. +A record failure identifies the zero-based batch index, publishes no partial output and leaves the handle usable for the next call. +ByteTransducer evaluates individual exported graphs and returns both output and path cost. +The C ABI also supports native callers without Python. + +## Execution and compatibility + +The evaluator builds the reachable input/grammar DAG in reusable flat workspaces and settles predecessors in topological order. +Precomputed next-byte sets prune impossible paths. +Forced literal and epsilon chains become output fragments while retaining merge predecessors and the sequence of nonzero floating-point weight additions. +LIFO execution uses the uncompressed graph because compression can change visitation order. +Mode-specialized loops avoid repeated execution-option checks. +Direct composition starts uncompressed to avoid rebuilding the predominantly unweighted English verbalizer lattices. +Classification reproduces prepared-lookahead arc ordering; postprocessing reproduces output projection and epsilon removal before shortest-path selection. +These transformations retain the original graph; they do not replace the linguistic rules with heuristics. + +A single native program should preserve stage boundaries. +Globally minimizing a composed classifier/verbalizer graph would not necessarily reproduce selecting the classifier first and then the first accepted field permutation. +Exact output compatibility with the pinned reference remains required before production use. +Equal-cost alternatives depend on float accumulation and traversal order; a higher-precision accumulator is not a compatible replacement. +Lookahead and output-projection modes reject reachable input-epsilon cycles rather than use an incompatible fallback. +The direct-transducer mode retains a bounded cyclic evaluator; its tests check path cost and membership, not universal tie-order equivalence. +Chain compression accounts for the two possible lookahead copies of a grammar state when preserving merges. + +The complete entry point implements the cased, deterministic English reference with both punctuation options enabled, postprocessing enabled and a 729-permutation split limit. +It collapses Unicode whitespace before and after normalization, as the corpus wrapper does. +NeMo's failed-verbalization early return, including escaped brackets and backslashes, is preserved. +This is not a configurable replacement for every Normalizer option. +The audited dependency set is NeMo text processing 1.2.0, Pynini 2.1.6.post1, Python Unicode 15.0.0 and sacremoses 0.2.0. +Changing grammar exports or dependencies requires fresh differential validation. +TNBYTE02 exports retain cached acyclicity rather than recomputing it, since that cache affects reference queue choice. +Re-export legacy TNBYTE01 grammars before direct-mode unweighted execution; missing queue metadata is an error there. +Malformed UTF-8, unsupported labels and resource-limit failures are explicit errors, not silent fallbacks. + +Relevant implementation references: + +- [RE2 NFA](https://github.com/google/re2/blob/main/re2/nfa.cc): sparse active states and next-byte filtering. +- [Rust regex-automata](https://docs.rs/regex-automata/latest/regex_automata/): multiple execution strategies, bounded lazy DFA caches and one-pass specialization. +- [Hyperscan](https://github.com/intel/hyperscan) and [VectorScan](https://github.com/VectorCamp/vectorscan): hybrid automata, literal matching and architecture-specific acceleration. +- [PCRE2 JIT](https://www.pcre.org/current/doc/html/pcre2jit.html): compiling repeated matching work into machine code. + +These are architectural references, not interchangeable implementations of weighted text transduction. +No code from these engines is copied here. + +English punctuation processing uses build-time Unicode property tables from export_text_tables.py; regenerate text_tables.h with --output when deliberately changing the reference Python/sacremoses version. +The checked tables identify their source versions, and LICENSE.sacremoses preserves the MIT notice for derived detokenization code and tables. +ASCII decoding has NEON and SSE2 paths with a scalar fallback; define TN_DISABLE_SIMD to compare outputs and timing. +Compare SIMD and scalar execution on both ARM64 and x86-64 when changing text processing. + +This backend is opt-in and does not change Normalizer or Sparrowhawk. +Before enabling a new grammar or dependency version, run differential corpus tests and the repository's applicable NeMo/Sparrowhawk checks. +The reference's prepared-lookahead classifier can choose a different equally weighted output from ordinary Pynini composition; this prototype is not a claim of byte-for-byte equivalence to every Normalizer execution mode. diff --git a/tools/text_processing_deployment/native_runtime/__init__.py b/tools/text_processing_deployment/native_runtime/__init__.py new file mode 100644 index 000000000..7f5ad1f33 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Experimental native execution of exported NeMo byte transducers.""" diff --git a/tools/text_processing_deployment/native_runtime/benchmark.py b/tools/text_processing_deployment/native_runtime/benchmark.py new file mode 100644 index 000000000..4ee108ea2 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/benchmark.py @@ -0,0 +1,75 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Measure native normalization against supplied JSONL written/spoken references.""" + +import json +import re +import time +from argparse import ArgumentParser +from collections import Counter +from pathlib import Path + +from byte_transducer import NativePipeline + + +def main() -> None: + parser = ArgumentParser(description=__doc__) + parser.add_argument("--library", type=Path, required=True) + parser.add_argument("--grammars", type=Path, required=True) + parser.add_argument("--input", type=Path, nargs="+", required=True) + parser.add_argument("--report", type=Path, required=True) + parser.add_argument("--mismatches", type=Path, required=True) + args = parser.parse_args() + started = time.perf_counter() + native = NativePipeline(args.library, args.grammars) + setup_seconds = time.perf_counter() - started + counts: Counter[str] = Counter() + seconds: Counter[str] = Counter() + started = time.perf_counter() + try: + with args.mismatches.open("w") as differences: + for path in args.input: + with path.open() as source: + for line in source: + row = json.loads(line) + written = row["written"] + counts["rows"] += 1 + begin = time.perf_counter() + output = native.normalize(written) + seconds["native_normalization"] += time.perf_counter() - begin + if output is None: + counts["no_path"] += 1 + differences.write(json.dumps({"written": written, "error": "no path"}) + "\n") + continue + reference = row["spoken"] + if output != reference: + counts["exact_differences"] += 1 + counts["word_token_differences"] += re.findall(r"\w+", output) != re.findall( + r"\w+", reference + ) + differences.write( + json.dumps( + { + "written": written, + "candidate": output, + "reference": reference, + } + ) + + "\n" + ) + report = { + "counts": dict(counts), + "seconds": dict(seconds), + "setup_seconds": setup_seconds, + "elapsed_seconds": time.perf_counter() - started, + "scope": "Includes JSONL input and mismatch output; excludes source-token decoding and corpus output.", + "remaining_python": "JSONL benchmark input/output and call wrapper; use normalize_stream for a native record loop.", + } + args.report.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2)) + finally: + native.close() + + +if __name__ == "__main__": + main() diff --git a/tools/text_processing_deployment/native_runtime/byte_transducer.cc b/tools/text_processing_deployment/native_runtime/byte_transducer.cc new file mode 100644 index 000000000..da97da2ed --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/byte_transducer.cc @@ -0,0 +1,1210 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Experimental byte-transducer evaluator. +// Grammar construction remains in Pynini; this file has no OpenFst dependency. +// State merging follows weighted Viterbi semantics, not regexp leftmost-first semantics. +#include "text_processing.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +constexpr float kInf = std::numeric_limits::infinity(); +constexpr uint32_t kNone = UINT32_MAX; +struct Arc { + uint32_t input, output, target; + float weight; +}; +struct State { + uint32_t begin, end; + float final_weight; +}; +struct Trace { + uint32_t parent, output; +}; +struct Tail { + uint32_t target, output; + std::vector weights; + std::string input{}; + bool last_epsilon = false; +}; +struct Frontier { + std::vector costs; + std::vector paths, active; + std::vector queued; + explicit Frontier(size_t n) : costs(n, kInf), paths(n, kNone), queued(n, 0) {} + void clear() { + for (uint32_t s : active) { + costs[s] = kInf; + paths[s] = kNone; + queued[s] = 0; + } + active.clear(); + } +}; + +struct DagNode { + uint32_t state, pos, begin, end, parent = kNone, via = kNone; + float cost = kInf; + uint8_t mark = 0; +}; +struct DagEdge { + uint32_t node, arc; +}; +// Reuse per-handle storage and a flat state/position index across records. +struct DagWorkspace { + std::vector nodes; + std::vector edges; + std::vector stack, order, slots, used; + static size_t hash(uint64_t key) { + key ^= key >> 33; + key *= 0xff51afd7ed558ccdULL; + key ^= key >> 33; + return static_cast(key); + } + void reset() { + for (uint32_t i : used) + slots[i] = kNone; + used.clear(); + nodes.clear(); + edges.clear(); + stack.clear(); + order.clear(); + } + void grow() { + slots.assign(slots.empty() ? 4096 : slots.size() * 2, kNone); + used.clear(); + for (uint32_t i = 0; i < nodes.size(); ++i) { + auto &n = nodes[i]; + uint64_t key = (uint64_t(n.pos) << 32) | n.state; + size_t j = hash(key) & (slots.size() - 1); + while (slots[j] != kNone) + j = (j + 1) & (slots.size() - 1); + slots[j] = i; + used.push_back(j); + } + } + uint32_t get(uint32_t state, uint32_t pos) { + if ((nodes.size() + 1) * 2 >= slots.size()) + grow(); + uint64_t key = (uint64_t(pos) << 32) | state; + size_t j = hash(key) & (slots.size() - 1); + while (slots[j] != kNone) { + uint32_t id = slots[j]; + const auto &n = nodes[id]; + if (n.state == state && n.pos == pos) + return id; + j = (j + 1) & (slots.size() - 1); + } + if (nodes.size() >= 10000000) + throw std::runtime_error("DAG workspace limit"); + uint32_t id = nodes.size(); + nodes.push_back({state, pos, 0, 0}); + slots[j] = id; + used.push_back(j); + return id; + } +}; + +class Program { +public: + std::vector states; + std::vector arcs; + uint32_t start; + bool grammar_zero = true, known_acyclic = false, has_queue_properties = false; + std::vector> next_labels; + std::vector tails, literal_tails; + std::vector fragments; + std::vector traces; + std::vector queue; + Frontier first, second; + std::string result; + size_t relaxations = 0; + + explicit Program(const char *path) : start(0), first(0), second(0) { + std::ifstream f(path, std::ios::binary); + if (!f) + throw std::runtime_error("Cannot open transducer"); + auto read = [&](void *p, size_t n) { + if (!f.read(static_cast(p), n)) + throw std::runtime_error("Truncated transducer"); + }; + const uint32_t endian = 1; + if (*reinterpret_cast(&endian) != 1) + throw std::runtime_error("This prototype requires a little-endian host"); + char magic[8]; + read(magic, 8); + has_queue_properties = std::memcmp(magic, "TNBYTE02", 8) == 0; + if (!has_queue_properties && std::memcmp(magic, "TNBYTE01", 8)) + throw std::runtime_error("Invalid transducer format"); + uint32_t n, m; + read(&n, 4); + read(&m, 4); + read(&start, 4); + if (has_queue_properties) { + uint32_t flags; + read(&flags, 4); + if (flags & ~1u) + throw std::runtime_error("Unsupported queue properties"); + known_acyclic = flags & 1; + } + if (!n || start >= n || n > 10000000 || m > 100000000) + throw std::runtime_error("Invalid transducer dimensions"); + states.resize(n); + arcs.resize(m); + for (auto &s : states) { + read(&s.begin, 4); + read(&s.end, 4); + read(&s.final_weight, 4); + if (s.begin > s.end || s.end > m || std::isnan(s.final_weight) || + s.final_weight == -kInf) + throw std::runtime_error("Invalid state"); + } + for (auto &a : arcs) { + read(&a.input, 4); + read(&a.output, 4); + read(&a.target, 4); + read(&a.weight, 4); + if (a.input > 255 || a.output > 255 || a.target >= n || + !std::isfinite(a.weight)) + throw std::runtime_error("Unsupported arc"); + } + for (const auto &s : states) { + for (uint32_t i = s.begin + (s.begin < s.end); i < s.end; ++i) + if (arcs[i - 1].input > arcs[i].input) + throw std::runtime_error("Unsorted arcs"); + } + for (const auto &arc : arcs) + grammar_zero &= arc.weight == 0; + // Compute possible next bytes through epsilon paths once, rather than + // exploring paths that cannot consume the current byte on every call. + next_labels.resize(n); + std::vector> predecessors(n); + std::deque pending; + std::vector queued(n, 1); + for (uint32_t s = 0; s < n; ++s) { + if (states[s].final_weight != kInf) + next_labels[s][4] = 1; + for (uint32_t j = states[s].begin; j < states[s].end; ++j) { + const auto &a = arcs[j]; + if (!a.input) + predecessors[a.target].push_back(s); + else + next_labels[s][a.input / 64] |= uint64_t{1} << (a.input % 64); + } + pending.push_back(s); + } + while (!pending.empty()) { + uint32_t s = pending.front(); + pending.pop_front(); + queued[s] = 0; + for (uint32_t before : predecessors[s]) { + bool changed = false; + for (size_t k = 0; k < 5; ++k) { + uint64_t combined = next_labels[before][k] | next_labels[s][k]; + changed |= combined != next_labels[before][k]; + next_labels[before][k] = combined; + } + if (changed && !queued[before]) { + pending.push_back(before); + queued[before] = 1; + } + } + } + // Fuse forced epsilon chains into output fragments without changing the + // order of floating-point weight additions. + // A lookahead state can have both an ordinary and a pending-label copy. + // Their outgoing arcs can merge even when the grammar target has one predecessor. + std::vector phases(n, 0); + phases[start] = 1; + for (const auto &a : arcs) + phases[a.target] |= a.input ? 1 : 2; + std::vector indegree(n, 0); + for (uint32_t source = 0; source < n; ++source) + for (uint32_t a = states[source].begin; a < states[source].end; ++a) + indegree[arcs[a].target] += phases[source] == 3 ? 2 : 1; + auto compile_tails = [&](bool include_input) { + std::vector compiled; + compiled.reserve(n); + for (uint32_t s = 0; s < n; ++s) { + Tail tail{s, 0, {}}; + std::string fragment; + size_t steps = 0; + while (steps < 256) { + const auto &st = states[tail.target]; + // Keep merge points: a preference can disappear if a large later cost + // is added before the competing prefix costs are compared. + if (indegree[tail.target] > 1 || st.final_weight != kInf || + st.end - st.begin != 1 || indegree[arcs[st.begin].target] > 1 || + (!include_input && arcs[st.begin].input)) + break; + const auto &a = arcs[st.begin]; + if (a.output) + fragment.push_back(static_cast(a.output)); + if (a.input) + tail.input.push_back(static_cast(a.input)); + tail.last_epsilon = !a.input; + if (a.weight != 0) + tail.weights.push_back(a.weight); + tail.target = a.target; + ++steps; + } + // Leave long chains and cycles to the bounded general evaluator. + if (steps == 256) { + tail = Tail{s, 0, {}}; + fragment.clear(); + } + if (!fragment.empty()) { + tail.output = static_cast(fragments.size() + 256); + fragments.push_back(std::move(fragment)); + } + compiled.push_back(std::move(tail)); + } + return compiled; + }; + tails = compile_tails(false); + literal_tails = compile_tails(true); + first = Frontier(n); + second = Frontier(n); + traces.reserve(65536); + queue.reserve(4096); + } + + bool relax(Frontier &f, uint32_t s, float cost, uint32_t parent, + uint32_t output) { + const auto &tail = tails[s]; + for (float w : tail.weights) + cost += w; + s = tail.target; + if (!(cost < f.costs[s])) + return false; + if (++relaxations > 10000000) + throw std::runtime_error("Relaxation limit reached; possible epsilon " + "cycle or excessive ambiguity"); + if (f.costs[s] == kInf) + f.active.push_back(s); + if (output) { + if (traces.size() >= 10000000) + throw std::runtime_error("Trace memory limit reached"); + traces.push_back({parent, output}); + parent = static_cast(traces.size() - 1); + } + if (tail.output) { + if (traces.size() >= 10000000) + throw std::runtime_error("Trace memory limit reached"); + traces.push_back({parent, tail.output}); + parent = static_cast(traces.size() - 1); + } + f.costs[s] = cost; + f.paths[s] = parent; + return true; + } + + bool can_consume(uint32_t s, uint32_t label) const { + return next_labels[s][label / 64] & (uint64_t{1} << (label % 64)); + } + + void epsilon_closure(Frontier &f, uint32_t label) { + queue.clear(); + for (uint32_t s : f.active) + if (can_consume(s, label)) { + queue.push_back(s); + f.queued[s] = 1; + } + for (size_t p = 0; p < queue.size(); ++p) { + uint32_t s = queue[p]; + f.queued[s] = 0; + const float cost = f.costs[s]; + const uint32_t path = f.paths[s]; + const State &st = states[s]; + for (uint32_t j = st.begin; j < st.end && arcs[j].input == 0; ++j) { + const Arc &a = arcs[j]; + if (can_consume(a.target, label) && + relax(f, a.target, cost + a.weight, path, a.output) && + !f.queued[tails[a.target].target]) { + uint32_t terminal = tails[a.target].target; + queue.push_back(terminal); + f.queued[terminal] = 1; + } + } + } + } + + bool project_output = false; + bool rewrite_projected(float &best_cost, size_t input_size) { + struct PArc { + uint32_t target, output; + float weight; + }; + const auto &nodes = dag.nodes; + size_t n = nodes.size(); + std::vector> outgoing(n); + std::vector final(n, kInf), distance(n, kInf); + std::vector incoming(n, 0), seen(n, 0); + std::vector rank(n), pending, closure; + incoming[0] = 1; + for (uint32_t i = 0; i < dag.order.size(); ++i) + rank[dag.order[i]] = i; + for (uint32_t i = 0; i < n; ++i) { + if (nodes[i].pos == input_size) + final[i] = states[nodes[i].state / 2].final_weight; + for (uint32_t e = nodes[i].begin; e < nodes[i].end; ++e) { + auto edge = dag.edges[e]; + const auto &arc = arcs[edge.arc]; + outgoing[i].push_back({edge.node, arc.output, arc.weight}); + if (arc.output) + incoming[edge.node] = 1; + } + } + // NeMo's top_rewrite projects output labels, removes epsilon paths, then + // selects a path; direct transducer traversal has different tie behavior. + for (uint32_t source : dag.order) { + if (!incoming[source]) + continue; + if (std::none_of(outgoing[source].begin(), outgoing[source].end(), + [](const PArc &arc) { return !arc.output; })) { + std::reverse(outgoing[source].begin(), outgoing[source].end()); + continue; + } + closure.clear(); + pending.clear(); + pending.push_back(source); + while (!pending.empty()) { + uint32_t id = pending.back(); + pending.pop_back(); + if (seen[id]) + continue; + seen[id] = 1; + closure.push_back(id); + for (const auto &arc : outgoing[id]) + if (!arc.output) + pending.push_back(arc.target); + } + for (uint32_t id : closure) { + seen[id] = 0; + distance[id] = kInf; + } + std::sort(closure.begin(), closure.end(), + [&](uint32_t a, uint32_t b) { return rank[a] > rank[b]; }); + distance[source] = 0; + for (uint32_t id : closure) + for (const auto &arc : outgoing[id]) + if (!arc.output) + distance[arc.target] = + std::min(distance[arc.target], distance[id] + arc.weight); + std::vector expanded; + std::unordered_map positions; + pending.push_back(source); + float final_weight = kInf; + while (!pending.empty()) { + uint32_t id = pending.back(); + pending.pop_back(); + if (seen[id]) + continue; + seen[id] = 1; + for (const auto &arc : outgoing[id]) { + if (!arc.output) { + if (!seen[arc.target]) + pending.push_back(arc.target); + continue; + } + uint64_t key = (uint64_t(arc.output) << 32) | arc.target; + auto [it, inserted] = positions.emplace(key, expanded.size()); + float weight = distance[id] + arc.weight; + if (inserted) + expanded.push_back({arc.target, arc.output, weight}); + else + expanded[it->second].weight = + std::min(expanded[it->second].weight, weight); + } + final_weight = std::min(final_weight, distance[id] + final[id]); + } + for (uint32_t id : closure) + seen[id] = 0; + // OpenFst restores the collected arcs by popping from the back. + std::reverse(expanded.begin(), expanded.end()); + outgoing[source] = std::move(expanded); + final[source] = final_weight; + } + // Epsilon removal changes arc order, so recompute the reference DFS order. + std::vector cursor(n, 0), order; + pending.push_back(0); + seen[0] = 1; + while (!pending.empty()) { + uint32_t id = pending.back(); + if (cursor[id] < outgoing[id].size()) { + uint32_t target = outgoing[id][cursor[id]++].target; + if (!seen[target]) { + seen[target] = 1; + pending.push_back(target); + } + } else { + order.push_back(id); + pending.pop_back(); + } + } + std::vector coaccessible(n, 0); + bool unweighted = true; + for (uint32_t id : order) { + coaccessible[id] = final[id] != kInf; + for (const auto &arc : outgoing[id]) { + coaccessible[id] |= coaccessible[arc.target]; + if (coaccessible[arc.target]) + unweighted &= arc.weight == 0; + } + } + // RmEpsilon clears the known acyclic property; ignore dead paths as Connect does. + if (unweighted) { + std::fill(seen.begin(), seen.end(), 0); + pending.clear(); + order.clear(); + pending.push_back(0); + seen[0] = 1; + while (!pending.empty()) { + uint32_t id = pending.back(); + pending.pop_back(); + order.push_back(id); + for (const auto &arc : outgoing[id]) + if (coaccessible[arc.target] && !seen[arc.target]) { + seen[arc.target] = 1; + pending.push_back(arc.target); + } + } + std::reverse(order.begin(), order.end()); + } + std::fill(distance.begin(), distance.end(), kInf); + distance[0] = 0; + std::vector parent(n, kNone), output(n, 0); + best_cost = kInf; + uint32_t last = kNone; + for (auto it = order.rbegin(); it != order.rend(); ++it) { + uint32_t id = *it; + float cost = distance[id] + final[id]; + if (cost < best_cost) { + best_cost = cost; + last = id; + } + for (const auto &arc : outgoing[id]) { + cost = distance[id] + arc.weight; + if (cost < distance[arc.target]) { + distance[arc.target] = cost; + parent[arc.target] = id; + output[arc.target] = arc.output; + } + } + } + result.clear(); + if (last == kNone) + return false; + for (uint32_t id = last; parent[id] != kNone; id = parent[id]) + result.push_back(output[id]); + std::reverse(result.begin(), result.end()); + return true; + } + + DagWorkspace dag; + bool push_prefix = false; + // Removed input epsilon arcs still affect AutoQueue and terminal lookahead phase. + bool input_had_nul = false, trailing_nul = false; + // Specialize record execution so each arc does not retest the same mode flags. + template + bool rewrite_dag_impl(const char *data, size_t size, float &best_cost) { + if (size > UINT32_MAX) + throw std::runtime_error("Input length limit"); + auto &nodes = dag.nodes; + auto &edges = dag.edges; + auto &stack = dag.stack; + auto &order = dag.order; + dag.reset(); + for (size_t i = 0; i < size; ++i) + if (!data[i]) + throw std::runtime_error("Embedded NUL is unsupported"); + dag.get(start * 2, 0); + stack.push_back(0); + while (!stack.empty()) { + uint32_t id = stack.back(); + if (!nodes[id].mark) { + uint32_t state = nodes[id].state / 2, pos = nodes[id].pos; + uint32_t label = + pos < size ? static_cast(data[pos]) : 256; + nodes[id].mark = 1; + nodes[id].begin = edges.size(); + const auto &st = states[state]; + auto add = [&](uint32_t a) { + const auto &arc = arcs[a]; + uint32_t nextpos = pos + (arc.input != 0); + uint32_t nextlabel = + nextpos < size ? static_cast(data[nextpos]) : 256; + if (!can_consume(arc.target, nextlabel)) + return; + const auto &tail = literal_tails[arc.target]; + uint32_t terminal = arc.target; + bool last_epsilon = !arc.input; + if (compress && tail.target != arc.target) { + if (tail.input.size() > size - nextpos || + std::memcmp(data + nextpos, tail.input.data(), + tail.input.size())) + return; + nextpos += tail.input.size(); + terminal = tail.target; + if (terminal != arc.target) + last_epsilon = tail.last_epsilon; + nextlabel = nextpos < size + ? static_cast(data[nextpos]) + : 256; + if (!can_consume(terminal, nextlabel)) + return; + } + bool prefix = lookahead && (nextpos < size || trailing_nul) && last_epsilon; + uint32_t target = dag.get(terminal * 2 + prefix, nextpos); + if (edges.size() >= 10000000) + throw std::runtime_error("DAG edge limit"); + edges.push_back({target, a}); + }; + uint32_t epsilon_end = st.begin; + while (epsilon_end < st.end && !arcs[epsilon_end].input) + ++epsilon_end; + auto add_eps = [&]() { + for (uint32_t a = st.begin; a < epsilon_end; ++a) + add(a); + }; + auto add_match = [&]() { + if (label == 256) + return; + auto it = std::lower_bound( + arcs.begin() + epsilon_end, arcs.begin() + st.end, label, + [](const Arc &arc, uint32_t value) { return arc.input < value; }); + for (; it != arcs.begin() + st.end && it->input == label; ++it) + add(it - arcs.begin()); + }; + // Prepared label lookahead lists the pushed label before epsilon arcs. + // That order determines the selected output when costs tie. + if (nodes[id].state & 1) { + add_match(); + add_eps(); + } else { + add_eps(); + add_match(); + } + nodes[id].end = edges.size(); + } + if (nodes[id].parent == kNone) + nodes[id].parent = nodes[id].begin; + if (nodes[id].parent < nodes[id].end) { + uint32_t child = edges[nodes[id].parent++].node; + if (nodes[child].mark == 1) { + if (lookahead || project_output) + throw std::runtime_error( + "Input-epsilon cycles are unsupported in compatibility modes"); + return rewrite_fast(data, size, best_cost); + } + if (nodes[child].mark == 0) + stack.push_back(child); + } else { + nodes[id].mark = 2; + nodes[id].parent = kNone; + order.push_back(id); + stack.pop_back(); + } + } + if (project_output) + return rewrite_projected(best_cost, size); + // Only direct composition needs Connect pruning to choose its queue. + std::vector coaccessible; + if (!lookahead && !known_acyclic) { + coaccessible.assign(nodes.size(), 0); + for (uint32_t id : order) { + coaccessible[id] = nodes[id].pos == size && + states[nodes[id].state / 2].final_weight != kInf; + for (uint32_t e = nodes[id].begin; e < nodes[id].end; ++e) + coaccessible[id] |= coaccessible[edges[e].node]; + } + } + bool unweighted = grammar_zero; + if (!unweighted && (lookahead || !known_acyclic)) { + unweighted = true; + for (const auto &edge : edges) { + const auto &arc = arcs[edge.arc]; + if (!lookahead && !coaccessible[edge.node]) + continue; + if (arc.weight != 0 || + (compress && !literal_tails[arc.target].weights.empty())) { + unweighted = false; + break; + } + } + } + // AutoQueue tests known acyclicity before falling back to unweighted LIFO. + // Prepared lookahead retains the acyclic property for empty input only. + bool lifo = unweighted && !(lookahead ? size == 0 && !input_had_nul : known_acyclic); + if (lifo && !lookahead && !has_queue_properties) + throw std::runtime_error( + "Re-export legacy grammar with cached queue properties"); + if (lifo && compress) + return rewrite_dag_impl(data, size, best_cost); + if (lifo) { + std::vector seen(nodes.size(), 0); + stack.clear(); + order.clear(); + stack.push_back(0); + seen[0] = 1; + while (!stack.empty()) { + uint32_t id = stack.back(); + stack.pop_back(); + order.push_back(id); + for (uint32_t e = nodes[id].begin; e < nodes[id].end; ++e) { + uint32_t target = edges[e].node; + if (!lookahead && !coaccessible[target]) + continue; + if (!seen[target]) { + seen[target] = 1; + stack.push_back(target); + } + } + } + std::reverse(order.begin(), order.end()); + } + // Settle every predecessor before propagating a cost or choosing a tie. + nodes[0].cost = 0; + best_cost = kInf; + uint32_t last = kNone; + for (auto it = order.rbegin(); it != order.rend(); ++it) { + uint32_t id = *it; + const auto &node = nodes[id]; + if (node.pos == size) { + float cost = node.cost + states[node.state / 2].final_weight; + if (cost < best_cost) { + best_cost = cost; + last = id; + } + } + for (uint32_t e = node.begin; e < node.end; ++e) { + const auto &edge = edges[e]; + const auto &arc = arcs[edge.arc]; + float cost = node.cost + arc.weight; + if (compress) + for (float weight : literal_tails[arc.target].weights) + cost += weight; + auto &next = nodes[edge.node]; + if (cost < next.cost) { + next.cost = cost; + next.parent = id; + next.via = edge.arc; + } + } + } + result.clear(); + if (last == kNone) + return false; + for (uint32_t id = last; nodes[id].parent != kNone; id = nodes[id].parent) { + const auto &arc = arcs[nodes[id].via]; + const auto &tail = literal_tails[arc.target]; + if (compress && tail.output) { + const auto &fragment = fragments[tail.output - 256]; + result.append(fragment.rbegin(), fragment.rend()); + } + if (arc.output) + result.push_back(arc.output); + } + std::reverse(result.begin(), result.end()); + return true; + } + bool rewrite_dag(const char *data, size_t size, float &cost) { + if (push_prefix) { + if (!grammar_zero && !project_output) + return rewrite_dag_impl(data, size, cost); + return rewrite_dag_impl(data, size, cost); + } + // Verbalizer lattices are usually unweighted; build once without compression. + return rewrite_dag_impl(data, size, cost); + } + bool rewrite(const char *data, size_t size, float &cost) { + // Pynini compiles byte zero as epsilon, not a consuming input label. + // Keep it in the original text for punctuation and failed-verbalization fallback. + input_had_nul = std::memchr(data, 0, size) != nullptr; + trailing_nul = size && !data[size - 1]; + if (input_had_nul) { + if (!push_prefix && !project_output) + throw std::runtime_error("NUL requires lookahead or output-projected mode"); + std::string input; + input.reserve(size); + for (size_t i = 0; i < size; ++i) + if (data[i]) + input.push_back(data[i]); + return rewrite_dag(input.data(), input.size(), cost); + } + return rewrite_dag(data, size, cost); + } + + bool rewrite_fast(const char *data, size_t size, float &best_cost) { + first.clear(); + second.clear(); + traces.clear(); + result.clear(); + relaxations = 0; + Frontier *current = &first; + Frontier *next = &second; + relax(*current, start, 0, kNone, 0); + for (size_t p = 0; p < size; ++p) { + const uint32_t ch = static_cast(data[p]); + epsilon_closure(*current, ch); + // Label zero denotes epsilon, so literal NUL requires a different alphabet. + if (!ch) + throw std::runtime_error("Embedded NUL is unsupported"); + next->clear(); + for (uint32_t s : current->active) { + const State &st = states[s]; + auto a = std::lower_bound( + arcs.begin() + st.begin, arcs.begin() + st.end, ch, + [](const Arc &a, uint32_t label) { return a.input < label; }); + for (; a != arcs.begin() + st.end && a->input == ch; ++a) + relax(*next, a->target, current->costs[s] + a->weight, + current->paths[s], a->output); + } + std::swap(current, next); + if (current->active.empty()) { + best_cost = kInf; + return false; + } + } + epsilon_closure(*current, 256); + best_cost = kInf; + uint32_t path = kNone; + for (uint32_t s : current->active) { + float cost = current->costs[s] + states[s].final_weight; + if (cost < best_cost) { + best_cost = cost; + path = current->paths[s]; + } + } + if (best_cost == kInf) + return false; + for (; path != kNone; path = traces[path].parent) { + uint32_t output = traces[path].output; + if (output < 256) + result.push_back(static_cast(output)); + else { + const auto &fragment = fragments[output - 256]; + result.append(fragment.rbegin(), fragment.rend()); + } + } + std::reverse(result.begin(), result.end()); + return true; + } +}; +thread_local std::string error; +} // namespace + +extern "C" { +void *tn_load(const char *path) { + try { + return new Program(path); + } catch (const std::exception &e) { + error = e.what(); + return nullptr; + } +} +void tn_delete(void *p) { delete static_cast(p); } +const char *tn_error() { return error.c_str(); } +// Result memory belongs to the handle and is valid until its next rewrite. +// One handle is not safe for simultaneous calls. +int tn_rewrite(void *p, const char *text, size_t length, const char **output, + size_t *size, float *cost) { + try { + if (!p || !text || !output || !size || !cost) + throw std::runtime_error("Null argument"); + auto &program = *static_cast(p); + bool found = program.rewrite(text, length, *cost); + *output = program.result.data(); + *size = program.result.size(); + return found ? 0 : 1; + } catch (const std::exception &e) { + error = e.what(); + return -1; + } +} +} + +namespace { +struct Field { + enum Kind { String, Boolean, Object, Missing } kind; + std::string key, value; + std::vector children; +}; + +class TagParser { + const std::string &text; + size_t pos = 0; + void spaces() { + while (pos < text.size() && text[pos] == ' ') + ++pos; + } + void expect(char c) { + if (pos == text.size() || text[pos++] != c) + throw std::runtime_error("Invalid tagged text"); + } + +public: + explicit TagParser(const std::string &input) : text(input) {} + std::vector parse(size_t depth = 0) { + if (depth > 64) + throw std::runtime_error("Tagged nesting limit exceeded"); + std::vector result; + spaces(); + while (pos < text.size() && text[pos] != '}') { + size_t begin = pos; + while (pos < text.size() && + ((text[pos] >= 'a' && text[pos] <= 'z') || + (text[pos] >= 'A' && text[pos] <= 'Z') || text[pos] == '_')) + ++pos; + if (pos == begin) + throw std::runtime_error("Invalid tag key"); + Field f; + f.key = text.substr(begin, pos - begin); + spaces(); + if (f.key == "preserve_order") { + expect(':'); + spaces(); + if (text.compare(pos, 4, "true") != 0) + throw std::runtime_error("Invalid preserve_order"); + pos += 4; + f.kind = Field::Boolean; + } else if (pos < text.size() && text[pos] == '{') { + ++pos; + f.kind = Field::Object; + f.children = parse(depth + 1); + expect('}'); + } else { + expect(':'); + spaces(); + expect('"'); + begin = pos; + while (pos + 1 < text.size() && + !(text[pos] == '"' && text[pos + 1] == ' ')) + ++pos; + if (pos + 1 >= text.size()) + throw std::runtime_error("Unterminated tag string"); + f.kind = pos == begin ? Field::Missing : Field::String; + f.value = text.substr(begin, pos - begin); + ++pos; + } + // NeMo's top-level list preserves repeated "tokens"; nested dictionaries overwrite duplicate keys. + auto old = + depth ? std::find_if(result.begin(), result.end(), + [&](const Field &x) { return x.key == f.key; }) + : result.end(); + if (old != result.end()) + *old = std::move(f); + else + result.push_back(std::move(f)); + spaces(); + } + if (!depth && pos != text.size()) + throw std::runtime_error("Trailing tag text"); + return result; + } +}; + +size_t permutation_count(const std::vector &fields) { + size_t result = 1; + for (size_t i = 2; i <= fields.size(); ++i) { + if (result > 729 / i) + return 730; + result *= i; + } + for (const auto &f : fields) + if (f.kind == Field::Object) { + size_t sub = permutation_count(f.children); + if (result > 729 / sub) + return 730; + result *= sub; + } + return result; +} + +std::vector permutations(const std::vector &fields) { + bool preserve = false; + std::vector order; + std::vector> pieces; + for (size_t i = 0; i < fields.size(); ++i) { + const auto &f = fields[i]; + order.push_back(i); + preserve |= f.key == "preserve_order"; + if (f.kind == Field::Missing) + return {}; + if (f.kind == Field::String) + pieces.push_back({f.key + ": \"" + f.value + "\" "}); + else if (f.kind == Field::Boolean) + pieces.push_back({f.key + ": true "}); + else { + auto values = permutations(f.children); + for (auto &v : values) + v = " " + f.key + " { " + v + " } "; + pieces.push_back(std::move(values)); + } + } + std::vector result; + do { + std::vector partial(1); + for (size_t i : order) { + std::vector expanded; + for (const auto &before : partial) + for (const auto &after : pieces[i]) + expanded.push_back(before + after); + partial = std::move(expanded); + } + result.insert(result.end(), partial.begin(), partial.end()); + if (result.size() > 729) + throw std::runtime_error("Permutation limit exceeded"); + } while (!preserve && std::next_permutation(order.begin(), order.end())); + return result; +} + +std::string collapse_spaces(const std::string &text) { + std::string result; + result.reserve(text.size()); + for (char c : text) { + if (c != ' ' || (!result.empty() && result.back() != ' ')) + result.push_back(c); + } + if (!result.empty() && result.back() == ' ') + result.pop_back(); + return result; +} + +class Pipeline { +public: + Program tagger, verbalizer, postprocessor; + std::string result, batch_result; + bool verbalization_failed = false; + Pipeline(const char *tagger_path, const char *verbalizer_path, + const char *post_path) + : tagger(tagger_path), verbalizer(verbalizer_path), + postprocessor(post_path) { + tagger.push_prefix = true; + postprocessor.project_output = true; + } + + bool rewrite(const char *text, size_t length) { + float cost; + result.clear(); + verbalization_failed = false; + if (!tagger.rewrite(text, length, cost)) + return false; + auto tokens = TagParser(tagger.result).parse(); + // NeMo attempts verbalization even when the tag parser returns no tokens. + size_t begin = 0; + do { + size_t end = begin, count = 1; + while (end < tokens.size()) { + size_t n = permutation_count({tokens[end]}); + if (n > 729) + throw std::runtime_error("Unsplittable tag permutation group"); + if (count > 729 / n) + break; + count *= n; + ++end; + } + std::vector candidates(1); + for (size_t i = begin; i < end; ++i) { + auto choices = permutations({tokens[i]}); + std::vector expanded; + for (const auto &prefix : candidates) + for (const auto &choice : choices) + expanded.push_back(prefix + choice); + candidates = std::move(expanded); + } + bool found = false; + for (const auto &candidate : candidates) { + if (verbalizer.rewrite(candidate.data(), candidate.size(), cost)) { + if (!result.empty()) + result += ' '; + result += verbalizer.result; + found = true; + break; + } + } + if (!found) { + verbalization_failed = true; + return false; + } + begin = end; + } while (begin < tokens.size()); + result = collapse_spaces(result); + if (!result.empty()) { + if (!postprocessor.rewrite(result.data(), result.size(), cost)) + return false; + result = postprocessor.result; + } + return true; + } +}; +} // namespace +extern "C" { +void *tn_pipeline_load(const char *t, const char *v, const char *p) { + try { + return new Pipeline(t, v, p); + } catch (const std::exception &e) { + error = e.what(); + return nullptr; + } +} +void tn_pipeline_delete(void *p) { delete static_cast(p); } +// Input whitespace/punctuation preprocessing and optional Moses postprocessing +// are outside this experimental core and must be supplied by the caller. +int tn_pipeline_rewrite(void *p, const char *text, size_t length, + const char **out, size_t *size) { + try { + if (!p || !text || !out || !size) + throw std::runtime_error("Null argument"); + auto &pipeline = *static_cast(p); + bool found = pipeline.rewrite(text, length); + *out = pipeline.result.data(); + *size = pipeline.result.size(); + return found ? 0 : 1; + } catch (const std::exception &e) { + error = e.what(); + return -1; + } +} +} + +extern "C" { +int tn_pipeline_normalize(void *p, const char *text, size_t length, + const char **out, size_t *size) { + try { + if (!p || !text || !out || !size) + throw std::runtime_error("Null argument"); + auto &pipeline = *static_cast(p); + auto original = + tn_text::collapse(tn_text::decode(std::string_view(text, length))); + bool found = true; + if (original.empty()) + pipeline.result.clear(); + else { + auto prepared = tn_text::prepare(original); + found = pipeline.rewrite(prepared.data(), prepared.size()); + if (found) + pipeline.result = tn_text::finish(original, pipeline.result); + else if (pipeline.verbalization_failed) { + // NeMo returns its escaped prepared input on failed verbalization. + // It skips punctuation postprocessing on this early-return path. + pipeline.result.clear(); + for (char c : prepared) { + if (c == '[' || c == ']' || c == '\\') + pipeline.result += '\\'; + pipeline.result += c; + } + found = true; + } + } + *out = pipeline.result.data(); + *size = pipeline.result.size(); + return found ? 0 : 1; + } catch (const std::exception &e) { + error = e.what(); + return -1; + } +} +// Consume a bounded batch without Python callbacks between records. +// No partial output is published when any record or frame fails. +int tn_pipeline_normalize_batch(void *p, const char *frames, size_t length, + const char **out, size_t *size) { + if (out) + *out = nullptr; + if (size) + *size = 0; + try { + if (!p || !frames || !out || !size) + throw std::runtime_error("Null argument"); + if (length > 64 * 1024 * 1024) + throw std::runtime_error("Batch exceeds 64 MiB"); + auto &batch = static_cast(p)->batch_result; + batch.clear(); + size_t position = 0, record = 0; + while (position < length) { + if (length - position < 4) + throw std::runtime_error("Truncated batch frame header"); + const auto *h = + reinterpret_cast(frames + position); + uint32_t n = uint32_t(h[0]) | (uint32_t(h[1]) << 8) | + (uint32_t(h[2]) << 16) | (uint32_t(h[3]) << 24); + position += 4; + if (n > 16 * 1024 * 1024 || n > length - position) + throw std::runtime_error("Invalid batch frame length"); + const char *value = nullptr; + size_t value_size = 0; + int status = + tn_pipeline_normalize(p, frames + position, n, &value, &value_size); + if (status) + throw std::runtime_error("Batch record " + std::to_string(record) + + ": " + + (status < 0 ? error : "No accepting path")); + if (value_size > UINT32_MAX || + value_size + 4 > 64 * 1024 * 1024 - batch.size()) + throw std::runtime_error("Batch output exceeds 64 MiB"); + for (size_t i = 0; i < 4; ++i) + batch.push_back(static_cast((value_size >> (8 * i)) & 255)); + batch.append(value, value_size); + position += n; + ++record; + } + *out = batch.data(); + *size = batch.size(); + return 0; + } catch (const std::exception &e) { + if (p) + static_cast(p)->batch_result.clear(); + error = e.what(); + return -1; + } +} +// Independent stage entry points let differential tests isolate text processing +// from weighted-path selection; result memory lasts until the next stage call. +int tn_text_stage(int stage, const char *input, size_t n, const char *output, + size_t m, const char **result, size_t *length) { + try { + if (!input || !output || !result || !length) + throw std::runtime_error("Null argument"); + static thread_local std::string value; + auto original = + tn_text::collapse(tn_text::decode(std::string_view(input, n))); + if (stage == 0) + value = tn_text::prepare(original); + else if (stage == 1) + value = tn_text::encode( + tn_text::detokenize(tn_text::decode(std::string_view(output, m)))); + else if (stage == 2) + value = tn_text::finish(original, std::string_view(output, m)); + else + throw std::runtime_error("Invalid text stage"); + *result = value.data(); + *length = value.size(); + return 0; + } catch (const std::exception &e) { + error = e.what(); + return -1; + } +} +} + +extern "C" int tn_set_mode(void *handle, int mode) { + if (!handle || mode < 0 || mode > 2) { + error = "Invalid evaluator mode"; + return -1; + } + auto &program = *static_cast(handle); + program.push_prefix = mode == 1; + program.project_output = mode == 2; + return 0; +} diff --git a/tools/text_processing_deployment/native_runtime/byte_transducer.py b/tools/text_processing_deployment/native_runtime/byte_transducer.py new file mode 100644 index 000000000..680b43482 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/byte_transducer.py @@ -0,0 +1,209 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Export tropical byte transducers and exercise the experimental native evaluator.""" + +from __future__ import annotations + +import ctypes +import struct +from pathlib import Path +from typing import TYPE_CHECKING, Literal + +if TYPE_CHECKING: + import pynini + + +def export_fst(fst: pynini.Fst, path: Path) -> None: + """Export byte arcs and cached acyclicity used by reference queue selection.""" + import pynini + + if fst.arc_type() != "standard" or fst.weight_type() != "tropical": + raise ValueError("Only standard tropical arcs are supported") + states = list(fst.states()) + if states != list(range(len(states))) or fst.start() < 0: + raise ValueError("Expected dense states and a valid start") + arcs = [] + records = [] + for state in states: + begin = len(arcs) + for arc in sorted(fst.arcs(state), key=lambda a: a.ilabel): + if not 0 <= arc.ilabel <= 255 or not 0 <= arc.olabel <= 255: + raise ValueError("Only epsilon and byte labels are supported") + weight = float(arc.weight) + arcs.append((arc.ilabel, arc.olabel, arc.nextstate, weight)) + records.append((begin, len(arcs), float(fst.final(state)))) + with path.open("wb") as out: + out.write( + struct.pack( + "<8sIIII", + b"TNBYTE02", + len(records), + len(arcs), + fst.start(), + int(bool(fst.properties(pynini.ACYCLIC, False))), + ) + ) + for record in records: + out.write(struct.pack(" None: + """Select direct, prepared-lookahead, or projected-output path semantics.""" + modes = {"transducer": 0, "lookahead": 1, "output": 2} + if mode not in modes: + raise ValueError(f"Unsupported evaluator mode: {mode}") + self.lib = ctypes.CDLL(str(library.absolute())) + self.lib.tn_load.argtypes = [ctypes.c_char_p] + self.lib.tn_load.restype = ctypes.c_void_p + self.lib.tn_delete.argtypes = [ctypes.c_void_p] + self.lib.tn_delete.restype = None + self.lib.tn_error.restype = ctypes.c_char_p + self.lib.tn_rewrite.argtypes = [ + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(ctypes.c_float), + ] + self.lib.tn_rewrite.restype = ctypes.c_int + self.lib.tn_set_mode.argtypes = [ctypes.c_void_p, ctypes.c_int] + self.lib.tn_set_mode.restype = ctypes.c_int + self.handle = self.lib.tn_load(str(grammar.absolute()).encode()) + if not self.handle: + raise RuntimeError(self.lib.tn_error().decode()) + if self.lib.tn_set_mode(self.handle, modes[mode]): + self.close() + raise RuntimeError(self.lib.tn_error().decode()) + + def close(self) -> None: + """Release native memory.""" + if self.handle: + self.lib.tn_delete(self.handle) + self.handle = None + + def rewrite(self, text: str) -> tuple[str, float] | None: + """Return a minimum-cost output, or None if no accepting path exists.""" + if not self.handle: + raise RuntimeError("Transducer is closed") + encoded = text.encode() + output = ctypes.c_void_p() + size = ctypes.c_size_t() + cost = ctypes.c_float() + status = self.lib.tn_rewrite( + self.handle, + encoded, + len(encoded), + ctypes.byref(output), + ctypes.byref(size), + ctypes.byref(cost), + ) + if status < 0: + raise RuntimeError(self.lib.tn_error().decode()) + if status: + return None + return ctypes.string_at(output.value, size.value).decode(), cost.value + + +class NativePipeline: + """Native classifier, tag parser, reordering, verbalizer and postprocessing FST. + + normalize includes English punctuation handling and Unicode whitespace collapse. + normalize_core exposes the transducer stages for compatibility diagnostics. + """ + + def __init__(self, library: Path, grammar_directory: Path) -> None: + self.lib = ctypes.CDLL(str(library.absolute())) + self.lib.tn_error.restype = ctypes.c_char_p + self.lib.tn_pipeline_load.argtypes = [ctypes.c_char_p] * 3 + self.lib.tn_pipeline_load.restype = ctypes.c_void_p + self.lib.tn_pipeline_delete.argtypes = [ctypes.c_void_p] + self.lib.tn_pipeline_delete.restype = None + self.lib.tn_pipeline_rewrite.argtypes = [ + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(ctypes.c_size_t), + ] + self.lib.tn_pipeline_rewrite.restype = ctypes.c_int + self.lib.tn_pipeline_normalize.argtypes = self.lib.tn_pipeline_rewrite.argtypes + self.lib.tn_pipeline_normalize.restype = ctypes.c_int + paths = [ + (grammar_directory / (name + ".bin")).absolute() for name in ("tagger", "verbalizer", "postprocessor") + ] + self.handle = self.lib.tn_pipeline_load(*(str(path).encode() for path in paths)) + if not self.handle: + raise RuntimeError(self.lib.tn_error().decode()) + + def close(self) -> None: + """Release native grammar and workspace storage.""" + if self.handle: + self.lib.tn_pipeline_delete(self.handle) + self.handle = None + + def normalize_core(self, text: str) -> str | None: + """Normalize preprocessed UTF-8 input with no Python work between stages.""" + return self._normalize(text, complete=False) + + def normalize(self, text: str) -> str | None: + """Run all English normalization stages, including punctuation, in native code.""" + return self._normalize(text, complete=True) + + def normalize_batch(self, texts: list[str]) -> list[str]: + """Run a bounded batch in native code and fail without returning partial results. + + Embedded NUL follows Pynini's epsilon interpretation at transducer input. + Original text is retained for punctuation handling and failed-verbalization fallback. + A failing record raises RuntimeError with its zero-based batch index. + The handle remains usable after failure; no partial result is returned. + """ + if not self.handle: + raise RuntimeError("Pipeline is closed") + encoded = [text.encode() for text in texts] + frames = b"".join(struct.pack(" len(data) - position: + raise RuntimeError("Truncated native batch record") + result.append(data[position : position + length].decode()) + position += length + if len(result) != len(texts): + raise RuntimeError("Native batch record count changed") + return result + + def _normalize(self, text: str, *, complete: bool) -> str | None: + if not self.handle: + raise RuntimeError("Pipeline is closed") + encoded = text.encode() + output = ctypes.c_void_p() + size = ctypes.c_size_t() + function = self.lib.tn_pipeline_normalize if complete else self.lib.tn_pipeline_rewrite + status = function(self.handle, encoded, len(encoded), ctypes.byref(output), ctypes.byref(size)) + if status < 0: + raise RuntimeError(self.lib.tn_error().decode()) + if status: + return None + return ctypes.string_at(output.value, size.value).decode() diff --git a/tools/text_processing_deployment/native_runtime/export_grammars.py b/tools/text_processing_deployment/native_runtime/export_grammars.py new file mode 100644 index 000000000..9fe413314 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/export_grammars.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Export an existing deterministic English NeMo normalizer for native evaluation.""" + +from argparse import ArgumentParser +from pathlib import Path + +from byte_transducer import export_fst +from nemo_text_processing.text_normalization.normalize import Normalizer + + +def main() -> None: + parser = ArgumentParser(description=__doc__) + parser.add_argument("--cache-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + normalizer = Normalizer( + input_case="cased", + lang="en", + cache_dir=str(args.cache_dir), + overwrite_cache=False, + ) + args.output_dir.mkdir(parents=True, exist_ok=True) + for name, grammar in ( + ("tagger", normalizer.tagger.fst), + ("verbalizer", normalizer.verbalizer.fst), + ("postprocessor", normalizer.post_processor.fst), + ): + export_fst(grammar, args.output_dir / (name + ".bin")) + + +if __name__ == "__main__": + main() diff --git a/tools/text_processing_deployment/native_runtime/export_text_tables.py b/tools/text_processing_deployment/native_runtime/export_text_tables.py new file mode 100644 index 000000000..7f8b948c5 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/export_text_tables.py @@ -0,0 +1,52 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Export the installed Python/sacremoses character properties at build time.""" + +import unicodedata +from argparse import ArgumentParser +from importlib.metadata import version +from pathlib import Path + +from sacremoses import MosesDetokenizer +from sacremoses.util import CJKChars + + +def ranges(values: list[int]) -> list[tuple[int, int]]: + """Coalesce exact codepoint membership into sorted closed intervals.""" + result: list[tuple[int, int]] = [] + for value in sorted(set(values)): + if result and result[-1][1] + 1 == value: + result[-1] = (result[-1][0], value) + else: + result.append((value, value)) + return result + + +def export(path: Path) -> None: + """Write immutable C++ tables; runtime processing needs no Python.""" + properties = { + "whitespace": ranges([c for c in range(0x110000) if chr(c).isspace()]), + "alpha": ranges(list(map(ord, MosesDetokenizer.IsAlpha))), + "currency_symbols": ranges(list(map(ord, MosesDetokenizer.IsSc))), + "cjk": CJKChars.ranges, + } + lines = [ + "// Generated by export_text_tables.py; do not edit.", + f"// Python Unicode {unicodedata.unidata_version}; sacremoses {version('sacremoses')}.", + "// Sacremoses-derived tables are covered by LICENSE.sacremoses.", + "#pragma once", + "namespace tn_text {", + "struct Range { char32_t first, last; };", + ] + for name, items in properties.items(): + lines.append(f"inline constexpr Range {name}[] = {{") + lines.extend(f" {{0x{a:x},0x{b:x}}}," for a, b in items) + lines.append("};") + lines.extend(["}", ""]) + path.write_text("\n".join(lines)) + + +if __name__ == "__main__": + parser = ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + export(parser.parse_args().output) diff --git a/tools/text_processing_deployment/native_runtime/normalize_stream.cc b/tools/text_processing_deployment/native_runtime/normalize_stream.cc new file mode 100644 index 000000000..686d8ed7d --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/normalize_stream.cc @@ -0,0 +1,82 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// Binary framing keeps embedded newlines intact and amortizes process startup. +// Each record is a little-endian uint32 byte length followed by UTF-8 bytes. +#include +#include +#include +#include +#include +#include +#include +extern "C" { +void *tn_pipeline_load(const char *, const char *, const char *); +void tn_pipeline_delete(void *); +const char *tn_error(); +int tn_pipeline_normalize(void *, const char *, size_t, const char **, + size_t *); +} +int main(int argc, char **argv) { + try { + if (argc != 4) + throw std::runtime_error("Usage: normalize_stream GRAMMAR_DIRECTORY " + "INPUT.frames OUTPUT.frames"); + std::string root = argv[1]; + std::unique_ptr pipeline( + tn_pipeline_load((root + "/tagger.bin").c_str(), + (root + "/verbalizer.bin").c_str(), + (root + "/postprocessor.bin").c_str()), + tn_pipeline_delete); + if (!pipeline) + throw std::runtime_error(tn_error()); + std::ifstream in(argv[2], std::ios::binary); + std::ofstream out(argv[3], std::ios::binary); + if (!in || !out) + throw std::runtime_error("Cannot open input/output"); + auto start = std::chrono::steady_clock::now(); + size_t count = 0; + std::string input; + for (;;) { + unsigned char h[4]; + in.read(reinterpret_cast(h), 4); + if (in.gcount() == 0 && in.eof()) + break; + if (in.gcount() != 4) + throw std::runtime_error("Truncated input frame header"); + uint32_t n = uint32_t(h[0]) | (uint32_t(h[1]) << 8) | + (uint32_t(h[2]) << 16) | (uint32_t(h[3]) << 24); + if (n > 16 * 1024 * 1024) + throw std::runtime_error("Input frame exceeds 16 MiB"); + input.resize(n); + if (!in.read(input.data(), n)) + throw std::runtime_error("Truncated input frame"); + const char *result = nullptr; + size_t length = 0; + int status = tn_pipeline_normalize(pipeline.get(), input.data(), + input.size(), &result, &length); + if (status) + throw std::runtime_error(status < 0 ? tn_error() : "No accepting path"); + if (length > UINT32_MAX) + throw std::runtime_error("Output frame too long"); + for (size_t i = 0; i < 4; ++i) + h[i] = (length >> (8 * i)) & 255; + out.write(reinterpret_cast(h), 4); + out.write(result, length); + if (!out) + throw std::runtime_error("Output write failed"); + ++count; + } + out.flush(); + if (!out) + throw std::runtime_error("Output flush failed"); + std::cerr << "{\"records\":" << count << ",\"processing_seconds\":" + << std::chrono::duration( + std::chrono::steady_clock::now() - start) + .count() + << "}\n"; + return 0; + } catch (const std::exception &e) { + std::cerr << e.what() << '\n'; + return 1; + } +} diff --git a/tools/text_processing_deployment/native_runtime/test_byte_transducer.py b/tools/text_processing_deployment/native_runtime/test_byte_transducer.py new file mode 100644 index 000000000..241d389b2 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/test_byte_transducer.py @@ -0,0 +1,468 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Compare the native evaluator against OpenFst on bounded weighted graphs.""" + +import random +import subprocess +import tempfile +import unittest +from pathlib import Path + +import pynini +from .byte_transducer import ByteTransducer, NativePipeline, export_fst + + +class ByteTransducerTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.directory = tempfile.TemporaryDirectory() + cls.root = Path(cls.directory.name) + cls.library = cls.root / "libtn_byte_transducer.so" + subprocess.run( + [ + "g++", + "-std=c++17", + "-O3", + "-Wall", + "-Wextra", + "-shared", + "-fPIC", + str(Path(__file__).with_suffix(".cc").with_name("byte_transducer.cc")), + "-o", + str(cls.library), + ], + check=True, + ) + + @classmethod + def tearDownClass(cls) -> None: + cls.directory.cleanup() + + def check_graph(self, graph: pynini.Fst, inputs: list[str]) -> None: + path = self.root / "grammar.bin" + export_fst(graph, path) + runtime = ByteTransducer(self.library, path) + try: + for text in inputs: + with self.subTest(text=text): + actual = runtime.rewrite(text) + lattice = pynini.compose(pynini.accep(pynini.escape(text)), graph) + expected_cost = ( + float(pynini.shortestdistance(lattice, reverse=True)[lattice.start()]) + if lattice.start() >= 0 + else float("inf") + ) + self.assertEqual(actual is None, expected_cost == float("inf")) + if actual is not None: + output, cost = actual + self.assertAlmostEqual(cost, expected_cost, places=5) + constrained = pynini.compose(lattice, pynini.accep(pynini.escape(output))) + self.assertGreaterEqual(constrained.start(), 0) + output_cost = float(pynini.shortestdistance(constrained, reverse=True)[constrained.start()]) + self.assertAlmostEqual(output_cost, expected_cost, places=5) + finally: + runtime.close() + + def test_random_acyclic_weighted_graphs(self) -> None: + rng = random.Random(81) + for _ in range(50): + graph = pynini.Fst() + for _ in range(12): + graph.add_state() + graph.set_start(0) + graph.set_final(11, pynini.Weight("tropical", rng.randrange(-3, 4) / 8)) + for source in range(11): + for target in range(source + 1, 12): + if rng.random() < 0.22 or target == source + 1: + graph.add_arc( + source, + pynini.Arc( + rng.choice([0, 97, 98]), + rng.choice([0, 120, 121]), + pynini.Weight("tropical", rng.randrange(-8, 9) / 8), + target, + ), + ) + self.check_graph(graph, ["", "a", "b", "ab", "ba", "aaa", "abab"]) + + def test_unicode_and_literal_metacharacters(self) -> None: + graph = pynini.union( + pynini.cross(pynini.escape("café [x]"), pynini.escape("café [output]")), + pynini.cross("12", "twelve"), + pynini.cross("", "empty"), + ) + self.check_graph(graph, ["café [x]", "12", "", "no match", "12"]) + + def test_consuming_cycles(self) -> None: + graph = pynini.closure(pynini.cross("a", "xyz")) + self.check_graph(graph, ["", "a", "aaaa", "b", "a"]) + + def test_epsilon_cycle_with_nonnegative_cost(self) -> None: + graph = pynini.Fst() + graph.add_state() + graph.add_state() + graph.set_start(0) + graph.set_final(1) + graph.add_arc(0, pynini.Arc(0, 120, pynini.Weight("tropical", 1), 0)) + graph.add_arc(0, pynini.Arc(97, 121, pynini.Weight("tropical", 0), 1)) + self.check_graph(graph, ["a", "", "aa", "a"]) + + def test_invalid_and_closed_handles(self) -> None: + path = self.root / "invalid.bin" + path.write_bytes(b"not a transducer") + with self.assertRaises(RuntimeError): + ByteTransducer(self.library, path) + export_fst(pynini.accep("x"), path) + runtime = ByteTransducer(self.library, path) + runtime.close() + runtime.close() + with self.assertRaises(RuntimeError): + runtime.rewrite("x") + + def test_native_pipeline_field_reordering(self) -> None: + tags = 'tokens { money { currency: "dollars" integer: "twelve" } }' + reordered = ' tokens { money { integer: "twelve" currency: "dollars" } } ' + export_fst(pynini.cross("input", tags), self.root / "tagger.bin") + export_fst(pynini.cross(reordered, "twelve dollars"), self.root / "verbalizer.bin") + export_fst( + pynini.cross("twelve dollars", "TWELVE DOLLARS"), + self.root / "postprocessor.bin", + ) + native = NativePipeline(self.library, self.root) + try: + self.assertEqual(native.normalize_core("input"), "TWELVE DOLLARS") + self.assertIsNone(native.normalize_core("no path")) + self.assertEqual(native.normalize_core("input"), "TWELVE DOLLARS") + finally: + native.close() + + def test_native_pipeline_preserves_explicit_field_order(self) -> None: + tags = 'tokens { money { currency: "dollars" integer: "twelve" preserve_order: true } }' + forbidden = ' tokens { money { integer: "twelve" currency: "dollars" preserve_order: true } } ' + export_fst(pynini.cross("input", tags), self.root / "tagger.bin") + export_fst(pynini.cross(forbidden, "twelve dollars"), self.root / "verbalizer.bin") + export_fst(pynini.accep("twelve dollars"), self.root / "postprocessor.bin") + native = NativePipeline(self.library, self.root) + try: + self.assertIsNone(native.normalize_core("input")) + finally: + native.close() + + def test_nonbyte_labels_are_rejected(self) -> None: + graph = pynini.Fst() + graph.add_state() + graph.add_state() + graph.set_start(0) + graph.set_final(1) + graph.add_arc(0, pynini.Arc(1000, 97, pynini.Weight("tropical", 0), 1)) + with self.assertRaises(ValueError): + export_fst(graph, self.root / "unsupported.bin") + + def test_float_tie_after_shared_merge(self) -> None: + graph = pynini.Fst() + for _ in range(4): + graph.add_state() + graph.set_start(0) + graph.set_final(3) + for src, dst, ilabel, olabel, weight in [ + (0, 1, 97, 72, 10.0), + (0, 2, 97, 76, 9.0), + (2, 1, 0, 0, 0.0), + (1, 3, 0, 0, 1e8), + ]: + graph.add_arc(src, pynini.Arc(ilabel, olabel, pynini.Weight("tropical", weight), dst)) + path = self.root / "merge.bin" + export_fst(graph, path) + runtime = ByteTransducer(self.library, path) + try: + expected = pynini.shortestpath(pynini.accep("a") @ graph).string() + self.assertEqual(expected, "L") + self.assertEqual(runtime.rewrite("a")[0], expected) + finally: + runtime.close() + + def test_prepared_lookahead_arc_order(self) -> None: + graph = pynini.Fst() + for _ in range(5): + graph.add_state() + graph.set_start(0) + graph.set_final(4) + for src, dst, ilabel, olabel, weight in [ + (0, 1, 0, 0, 1.0), + (1, 2, 0, 120, 0.0), + (1, 3, 97, 121, 0.0), + (2, 4, 97, 122, 0.0), + (3, 4, 0, 113, 0.0), + ]: + graph.add_arc(src, pynini.Arc(ilabel, olabel, pynini.Weight("tropical", weight), dst)) + path = self.root / "lookahead.bin" + export_fst(graph, path) + for mode, expected in [("transducer", "yq"), ("lookahead", "xz")]: + runtime = ByteTransducer(self.library, path, mode=mode) + try: + self.assertEqual(runtime.rewrite("a")[0], expected) + finally: + runtime.close() + + def test_projected_output_against_top_rewrite(self) -> None: + from pynini.lib import rewrite + + rng = random.Random(173) + for _ in range(30): + graph = pynini.Fst() + for _ in range(10): + graph.add_state() + graph.set_start(0) + graph.set_final(9) + for src in range(9): + for dst in range(src + 1, 10): + if dst == src + 1 or rng.random() < 0.2: + graph.add_arc( + src, + pynini.Arc( + rng.choice([0, 97, 98]), + rng.choice([0, 120, 121]), + pynini.Weight("tropical", rng.randrange(-3, 4) / 8), + dst, + ), + ) + path = self.root / "projected.bin" + export_fst(graph, path) + runtime = ByteTransducer(self.library, path, mode="output") + try: + for text in ["", "a", "b", "ab", "ba", "aaa", "abab"]: + try: + expected = rewrite.top_rewrite(text, graph) + except rewrite.Error: + expected = None + actual = runtime.rewrite(text) + with self.subTest(text=text): + self.assertEqual(None if actual is None else actual[0], expected) + finally: + runtime.close() + + def test_lookahead_phase_merge_before_float_rounding(self) -> None: + graph = pynini.Fst() + for _ in range(5): + graph.add_state() + graph.set_start(0) + for state, weight in [(0, 0), (1, 1), (4, 0)]: + graph.set_final(state, pynini.Weight("tropical", weight)) + for src, dst, ilabel, olabel, weight in [ + (0, 4, 98, 120, -1e-8), + (0, 2, 98, 122, 0), + (2, 3, 98, 122, 0), + (3, 1, 98, 121, -10), + (4, 2, 0, 120, 0), + ]: + graph.add_arc(src, pynini.Arc(ilabel, olabel, pynini.Weight("tropical", weight), dst)) + path = self.root / "phase-merge.bin" + export_fst(graph, path) + runtime = ByteTransducer(self.library, path, mode="lookahead") + try: + # The prepared OpenFst reference retains the cheaper prefix at the merge. + # Compressing the next path first rounds both totals to the same float. + self.assertEqual(runtime.rewrite("bbb"), ("xxzy", -9.0)) + finally: + runtime.close() + + def test_compatibility_modes_reject_epsilon_cycles(self) -> None: + graph = pynini.closure(pynini.cross("", "x")) + pynini.cross("a", "y") + path = self.root / "cycle.bin" + export_fst(graph, path) + for mode in ["lookahead", "output"]: + runtime = ByteTransducer(self.library, path, mode=mode) + try: + with self.assertRaisesRegex(RuntimeError, "Input-epsilon cycles"): + runtime.rewrite("a") + finally: + runtime.close() + + def test_random_exact_outputs(self) -> None: + from pynini.lib import rewrite + + rng = random.Random(20261017) + for case in range(1200): + graph = pynini.Fst() + n = rng.randrange(4, 15) + for _ in range(n): + graph.add_state() + ids = list(range(n)) + if case % 2: + rng.shuffle(ids) + graph.set_start(ids[0]) + for pos, state in enumerate(ids): + if pos == n - 1 or rng.random() < 0.15: + graph.set_final( + state, + pynini.Weight("tropical", rng.choice([0, 0, 0.125, 1, 2])), + ) + for later in range(pos + 1, n): + if later == pos + 1 or rng.random() < 0.25: + graph.add_arc( + state, + pynini.Arc( + rng.choice([0, 0, 97, 98]), + rng.choice([0, 120, 121, 122]), + pynini.Weight( + "tropical", + 0 if case >= 600 else rng.choice([0, 0, 0, -1e-8, 0.1, 0.2, 1e8, -10]), + ), + ids[later], + ), + ) + if case >= 600: + if case % 3 == 0: + target = graph.add_state() + graph.set_final(target) + graph.add_arc( + graph.start(), + pynini.Arc(99, 120, pynini.Weight("tropical", 1), target), + ) + if case % 2: + graph.properties(pynini.ACYCLIC, True) + graph.arcsort("ilabel") + path = self.root / "random-exact.bin" + export_fst(graph, path) + for mode in ["transducer", "output"]: + runtime = ByteTransducer(self.library, path, mode=mode) + try: + for text in ["", "a", "b", "ab", "ba", "aa", "bbb", "abab", "\0", "a\0", "a\0b", "\0b"]: + if mode == "transducer" and "\0" in text: + with self.assertRaisesRegex(RuntimeError, "NUL requires"): + runtime.rewrite(text) + continue + if mode == "output": + try: + expected = rewrite.top_rewrite(text, graph) + except rewrite.Error: + expected = None + else: + lattice = pynini.accep(text) @ graph + expected = pynini.shortestpath(lattice).string() if lattice.start() >= 0 else None + actual = runtime.rewrite(text) + with self.subTest(case=case, mode=mode, text=text): + self.assertEqual(None if actual is None else actual[0], expected) + finally: + runtime.close() + + def test_nul_preserves_lookahead_queue_and_terminal_phase(self) -> None: + # Expected outputs were checked against OpenFst's prepared lookahead matcher. + cases = [ + ( + 10, + {0: 2, 4: 0, 9: 0}, + "a\0", + "zxyzy", + [ + (0, 0, 122, 1), + (0, 98, 120, 9), + (1, 0, 120, 2), + (2, 0, 0, 3), + (2, 0, 0, 8), + (2, 97, 0, 9), + (3, 98, 0, 4), + (3, 0, 0, 5), + (3, 0, 121, 7), + (4, 98, 120, 5), + (4, 0, 0, 7), + (4, 0, 121, 9), + (5, 0, 120, 6), + (5, 97, 121, 7), + (5, 0, 122, 8), + (6, 98, 0, 7), + (7, 0, 122, 8), + (8, 0, 121, 9), + ], + ), + ( + 12, + {4: 0, 6: 2, 7: 0, 11: 0}, + "\0", + "yy", + [ + (0, 98, 0, 1), + (0, 0, 121, 2), + (0, 98, 120, 11), + (1, 0, 0, 2), + (2, 0, 120, 3), + (2, 0, 121, 10), + (3, 0, 0, 4), + (4, 98, 121, 5), + (4, 97, 121, 9), + (4, 0, 122, 11), + (5, 97, 120, 6), + (5, 98, 122, 9), + (6, 97, 122, 7), + (7, 97, 122, 8), + (7, 98, 122, 10), + (7, 97, 0, 11), + (8, 0, 0, 9), + (9, 98, 122, 10), + (9, 0, 121, 11), + (10, 0, 0, 11), + ], + ), + ] + for count, finals, text, expected, arcs in cases: + graph = pynini.Fst() + for _ in range(count): + graph.add_state() + graph.set_start(0) + for state, weight in finals.items(): + graph.set_final(state, pynini.Weight("tropical", weight)) + for source, ilabel, olabel, target in arcs: + graph.add_arc(source, pynini.Arc(ilabel, olabel, pynini.Weight.one("tropical"), target)) + graph.arcsort("ilabel") + path = self.root / "nul-queue.bin" + export_fst(graph, path) + runtime = ByteTransducer(self.library, path, mode="lookahead") + try: + self.assertEqual(runtime.rewrite(text)[0], expected) + runtime.rewrite(text.replace("\0", "")) + self.assertEqual(runtime.rewrite(text)[0], expected) + finally: + runtime.close() + + def test_unweighted_reference_queue_order(self) -> None: + # OpenFst AutoQueue chooses LIFO when acyclicity is not already known. + graph = pynini.Fst() + for _ in range(7): + graph.add_state() + graph.set_start(2) + graph.set_final(4) + for src, dst, ilabel, olabel in [ + (0, 1, 0, 122), + (0, 4, 98, 122), + (1, 4, 98, 120), + (2, 0, 0, 120), + (2, 1, 0, 121), + ]: + graph.add_arc(src, pynini.Arc(ilabel, olabel, pynini.Weight.one("tropical"), dst)) + graph.arcsort("ilabel") + expected = pynini.shortestpath(pynini.accep("b") @ graph).string() + self.assertEqual(expected, "yx") + path = self.root / "unweighted.bin" + for known_acyclic in [False, True]: + for unused_weighted_branch in [False, True]: + variant = graph.copy() + if unused_weighted_branch: + target = variant.add_state() + variant.set_final(target) + variant.add_arc( + variant.start(), + pynini.Arc(99, 120, pynini.Weight("tropical", 1), target), + ) + if known_acyclic: + variant.properties(pynini.ACYCLIC, True) + expected = pynini.shortestpath(pynini.accep("b") @ variant).string() + export_fst(variant, path) + runtime = ByteTransducer(self.library, path) + try: + self.assertEqual(runtime.rewrite("b")[0], expected) + finally: + runtime.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/text_processing_deployment/native_runtime/test_text_processing.py b/tools/text_processing_deployment/native_runtime/test_text_processing.py new file mode 100644 index 000000000..8ecec2550 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/test_text_processing.py @@ -0,0 +1,310 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Differential tests for native English text stages against their Python references.""" + +import ctypes +import logging +import random +import struct +import subprocess +import tempfile +import unittest +from pathlib import Path + +import pynini +from sacremoses import MosesDetokenizer + +from nemo_text_processing.text_normalization.data_loader_utils import post_process_punct, pre_process + +from .byte_transducer import NativePipeline, export_fst + + +class TextProcessingTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.previous_logging_disable = logging.root.manager.disable + logging.disable(logging.CRITICAL) + cls.directory = tempfile.TemporaryDirectory() + cls.root = Path(cls.directory.name) + source = Path(__file__).parent + cls.library = cls.root / "native.so" + cls.scalar = cls.root / "scalar.so" + for library, flags in ((cls.library, []), (cls.scalar, ["-DTN_DISABLE_SIMD"])): + subprocess.run( + [ + "g++", + "-std=c++17", + "-O3", + "-Wall", + "-Wextra", + "-shared", + "-fPIC", + *flags, + str(source / "byte_transducer.cc"), + "-o", + str(library), + ], + check=True, + ) + cls.libs = [ctypes.CDLL(str(p)) for p in (cls.library, cls.scalar)] + for lib in cls.libs: + lib.tn_text_stage.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_size_t, + ctypes.c_char_p, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(ctypes.c_size_t), + ] + lib.tn_text_stage.restype = ctypes.c_int + cls.detokenizer = MosesDetokenizer(lang="en") + cls.executable = cls.root / "normalize_stream" + subprocess.run( + [ + "g++", + "-std=c++17", + "-O3", + str(source / "byte_transducer.cc"), + str(source / "normalize_stream.cc"), + "-o", + str(cls.executable), + ], + check=True, + ) + + @classmethod + def tearDownClass(cls) -> None: + cls.directory.cleanup() + logging.disable(cls.previous_logging_disable) + + def stage(self, stage: int, written: str, output: str) -> str: + a, b = written.encode(), output.encode() + results = [] + for lib in self.libs: + result, length = ctypes.c_void_p(), ctypes.c_size_t() + status = lib.tn_text_stage(stage, a, len(a), b, len(b), ctypes.byref(result), ctypes.byref(length)) + self.assertEqual(status, 0) + results.append(ctypes.string_at(result.value, length.value).decode()) + self.assertEqual(results[0], results[1], "SIMD/scalar mismatch") + return results[0] + + def check_pair(self, written: str, output: str) -> None: + written = " ".join(written.split()) + self.assertEqual(self.stage(0, written, ""), pre_process(written).strip()) + detokenized = self.detokenizer.detokenize([output], unescape=False) + self.assertEqual(self.stage(1, "", output), detokenized) + expected = " ".join(post_process_punct(input=written, normalized_text=detokenized).split()) + self.assertEqual(self.stage(2, written, output), expected) + + def test_public_punctuation_and_unicode_cases(self) -> None: + pairs = [ + ("[12] test' example", "[ twelve ] test ' example"), + ("The Jones' house.", "The Jones ' house ."), + (chr(96) * 2 + "A test''", "\" A test ' '"), + ("a:b:c", "a : b c"), + ("a:: b", "a : : b"), + ("a [x] b", "a [ x ] b"), + ("a\u00a0b\u2003c", "a\u00a0b\u2003c"), + ("$5.00 and €3", "$ five . zero zero and € three"), + ("你好 世界 한 국", "你好 世界 한 국"), + ("a @-@ b @-@ c", "a @-@ b @-@ c"), + ("' é ' 中文", "' é ' 中文"), + ("'\U00010400", "'\U00010400"), + ("", ""), + ] + for a, b in pairs: + with self.subTest(written=a, output=b): + self.check_pair(a, b) + + def test_seeded_text_stage_equivalence(self) -> None: + rng = random.Random(914) + units = [ + "word", + "James", + "café", + "你好", + "국", + "'s", + "'é", + "a", + "", + " ", + "\t", + "\u00a0", + "\u2003", + "[", + "]", + "(", + ")", + "$", + "€", + "¿", + "¡", + ":", + ";", + ".", + ",", + "!", + "?", + "\\", + "%", + "'", + '"', + chr(96) * 2, + "''", + "„", + "“", + "”", + "a:b", + "@-@", + "\U00010400", + "\u1100", + "\uffdc", + "\U0002ffff", + ] + for _ in range(2000): + a = " ".join(rng.choices(units, k=rng.randrange(30))) + b = " ".join(rng.choices(units, k=rng.randrange(30))) + with self.subTest(written=a, output=b): + self.check_pair(a, b) + + def test_invalid_utf8(self) -> None: + for value in ( + b"\xc0\x80", + b"\xed\xa0\x80", + b"\xf4\x90\x80\x80", + b"\xff", + b"\xe2\x82", + ): + for lib in self.libs: + result, length = ctypes.c_void_p(), ctypes.c_size_t() + status = lib.tn_text_stage( + 0, + value, + len(value), + b"", + 0, + ctypes.byref(result), + ctypes.byref(length), + ) + self.assertEqual(status, -1) + + def test_stream_and_complete_api(self) -> None: + export_fst(pynini.cross("x", 'tokens { name: "word" }'), self.root / "tagger.bin") + export_fst( + pynini.cross(' tokens { name: "word" } ', "word"), + self.root / "verbalizer.bin", + ) + export_fst(pynini.accep("word"), self.root / "postprocessor.bin") + values = ["x", " \nx\t", "", "\u2003x\u00a0"] + native = NativePipeline(self.library, self.root) + try: + self.assertEqual([native.normalize(s) for s in values], ["word", "word", "", "word"]) + self.assertEqual(native.normalize_batch(values), ["word", "word", "", "word"]) + self.assertEqual(native.normalize_batch([]), []) + with self.assertRaisesRegex(RuntimeError, "Batch record 1"): + native.normalize_batch(["x", "unmatched", "x"]) + self.assertEqual(native.normalize_batch(["x"]), ["word"]) + for malformed in [b"\x02\x00", struct.pack(" None: + from types import SimpleNamespace + + from nemo_text_processing.text_normalization.normalize import Normalizer + from nemo_text_processing.text_normalization.token_parser import TokenParser + + values = ["a" * n + "\0word" for n in (0, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65)] + values += ["René\0 word", "x\0", "\u2003\0\u00a0", "\0", " \0 \0 ", "[\0x]", "x \0 y"] + normalizer = Normalizer.__new__(Normalizer) + normalizer.lang = "en" + normalizer.parser = TokenParser() + normalizer.max_number_of_permutations_per_split = 729 + normalizer.moses_detokenizer = self.detokenizer + prepared = {pre_process(" ".join(value.split())).strip().replace("\0", "") for value in values} + normalizer.tagger = SimpleNamespace( + fst=pynini.union(*(pynini.cross(pynini.escape(value), 'tokens { name: "word" }') for value in prepared)) + ) + normalizer.verbalizer = SimpleNamespace(fst=pynini.cross(' tokens { name: "word" } ', "word")) + normalizer.post_processor = SimpleNamespace(fst=pynini.accep("word")) + # Also check the early-return path, which must retain the original NUL. + for failed_verbalization in (False, True): + if failed_verbalization: + normalizer.verbalizer.fst = pynini.cross("unreachable", "word") + for name in ("tagger", "verbalizer", "post_processor"): + filename = "postprocessor" if name == "post_processor" else name + export_fst(getattr(normalizer, name).fst, self.root / (filename + ".bin")) + expected = [ + normalizer.normalize(" ".join(value.split()), punct_pre_process=True, punct_post_process=True) + for value in values + ] + for library in (self.library, self.scalar): + native = NativePipeline(library, self.root) + try: + with self.subTest(library=library.name, failed_verbalization=failed_verbalization): + self.assertEqual([native.normalize(value) for value in values], expected) + self.assertEqual(native.normalize_batch(values), expected) + self.assertEqual(native.normalize_batch(values), expected) + finally: + native.close() + + def test_failed_verbalization_matches_reference(self) -> None: + from types import SimpleNamespace + + from nemo_text_processing.text_normalization.normalize import Normalizer + from nemo_text_processing.text_normalization.token_parser import TokenParser + + normalizer = Normalizer.__new__(Normalizer) + normalizer.lang = "en" + normalizer.parser = TokenParser() + normalizer.max_number_of_permutations_per_split = 729 + normalizer.moses_detokenizer = self.detokenizer + normalizer.verbalizer = SimpleNamespace(fst=pynini.cross("unreachable", "word")) + normalizer.post_processor = SimpleNamespace(fst=pynini.accep("word")) + for tags in [" ", 'tokens { name: "word" }', 'tokens { name: "" }']: + normalizer.tagger = SimpleNamespace(fst=pynini.cross(pynini.escape("[ x ]"), tags)) + for name, component in [ + ("tagger", normalizer.tagger), + ("verbalizer", normalizer.verbalizer), + ("postprocessor", normalizer.post_processor), + ]: + export_fst(component.fst, self.root / (name + ".bin")) + native = NativePipeline(self.library, self.root) + try: + expected = normalizer.normalize("[x]", punct_pre_process=True, punct_post_process=True) + self.assertEqual(native.normalize("[x]"), expected) + finally: + native.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/text_processing_deployment/native_runtime/text_processing.h b/tools/text_processing_deployment/native_runtime/text_processing.h new file mode 100644 index 000000000..1833e65f0 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/text_processing.h @@ -0,0 +1,317 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 AND MIT +// English detokenization follows sacremoses; see LICENSE.sacremoses. +// Punctuation restoration follows NeMo's character-slot semantics, including deletions. +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(__aarch64__) && !defined(TN_DISABLE_SIMD) +#include +#elif defined(__SSE2__) && !defined(TN_DISABLE_SIMD) +#include +#endif +#include "text_tables.h" + +namespace tn_text { +template bool contains(char32_t c, const Range (&ranges)[N]) { + auto p = + std::lower_bound(std::begin(ranges), std::end(ranges), c, + [](const Range &r, char32_t v) { return r.last < v; }); + return p != std::end(ranges) && p->first <= c; +} +bool space(char32_t c) { + if (c < 128) + return c == ' ' || (c >= 9 && c <= 13) || (c >= 28 && c <= 31); + return contains(c, whitespace); +} +std::u32string decode(std::string_view s) { + std::u32string out; + out.reserve(s.size()); + size_t i = 0; + while (i < s.size()) { + // Widen ASCII blocks without a branch or UTF-8 decoder call per character. +#if defined(__aarch64__) && !defined(TN_DISABLE_SIMD) + if (i + 16 <= s.size()) { + auto v = vld1q_u8(reinterpret_cast(s.data() + i)); + if (vmaxvq_u8(v) < 128) { + size_t n = out.size(); + out.resize(n + 16); + auto lo = vmovl_u8(vget_low_u8(v)), hi = vmovl_u8(vget_high_u8(v)); + auto *p = reinterpret_cast(out.data() + n); + vst1q_u32(p, vmovl_u16(vget_low_u16(lo))); + vst1q_u32(p + 4, vmovl_u16(vget_high_u16(lo))); + vst1q_u32(p + 8, vmovl_u16(vget_low_u16(hi))); + vst1q_u32(p + 12, vmovl_u16(vget_high_u16(hi))); + i += 16; + continue; + } + } +#elif defined(__SSE2__) && !defined(TN_DISABLE_SIMD) + if (i + 16 <= s.size()) { + auto v = _mm_loadu_si128(reinterpret_cast(s.data() + i)); + if (!_mm_movemask_epi8(v)) { + size_t n = out.size(); + out.resize(n + 16); + auto z = _mm_setzero_si128(), lo = _mm_unpacklo_epi8(v, z), + hi = _mm_unpackhi_epi8(v, z); + auto *p = reinterpret_cast<__m128i *>(out.data() + n); + _mm_storeu_si128(p, _mm_unpacklo_epi16(lo, z)); + _mm_storeu_si128(p + 1, _mm_unpackhi_epi16(lo, z)); + _mm_storeu_si128(p + 2, _mm_unpacklo_epi16(hi, z)); + _mm_storeu_si128(p + 3, _mm_unpackhi_epi16(hi, z)); + i += 16; + continue; + } + } +#endif + uint32_t c = static_cast(s[i++]), minimum = 0; + size_t extra = 0; + if (c < 128) { + out.push_back(c); + continue; + } + if (c >= 0xc2 && c <= 0xdf) { + c &= 31; + extra = 1; + minimum = 0x80; + } else if (c >= 0xe0 && c <= 0xef) { + c &= 15; + extra = 2; + minimum = 0x800; + } else if (c >= 0xf0 && c <= 0xf4) { + c &= 7; + extra = 3; + minimum = 0x10000; + } else + throw std::runtime_error("Invalid UTF-8 lead byte"); + if (i + extra > s.size()) + throw std::runtime_error("Truncated UTF-8"); + for (size_t j = 0; j < extra; ++j) { + uint8_t b = static_cast(s[i++]); + if ((b & 0xc0) != 0x80) + throw std::runtime_error("Invalid UTF-8 continuation"); + c = (c << 6) | (b & 63); + } + if (c < minimum || c > 0x10ffff || (c >= 0xd800 && c <= 0xdfff)) + throw std::runtime_error("Invalid UTF-8 code point"); + out.push_back(c); + } + return out; +} +std::string encode(const std::u32string &s) { + std::string out; + out.reserve(s.size()); + for (uint32_t c : s) { + if (c < 0x80) + out.push_back(c); + else if (c < 0x800) { + out.push_back(0xc0 | (c >> 6)); + out.push_back(0x80 | (c & 63)); + } else if (c < 0x10000) { + out.push_back(0xe0 | (c >> 12)); + out.push_back(0x80 | ((c >> 6) & 63)); + out.push_back(0x80 | (c & 63)); + } else { + out.push_back(0xf0 | (c >> 18)); + out.push_back(0x80 | ((c >> 12) & 63)); + out.push_back(0x80 | ((c >> 6) & 63)); + out.push_back(0x80 | (c & 63)); + } + } + return out; +} +std::u32string collapse(const std::u32string &s) { + std::u32string out; + out.reserve(s.size()); + bool pending = false; + for (char32_t c : s) { + if (space(c)) { + pending = !out.empty(); + continue; + } + if (pending) + out += U' '; + pending = false; + out += c; + } + return out; +} +std::u32string replace_all(const std::u32string &s, std::u32string_view from, + std::u32string_view to) { + std::u32string out; + out.reserve(s.size()); + size_t p = 0; + while (p < s.size()) { + size_t k = s.find(from, p); + if (k == std::u32string::npos) { + out.append(s, p, s.size() - p); + break; + } + out.append(s, p, k - p); + out += to; + p = k + from.size(); + } + return out; +} +std::string prepare(const std::u32string &input) { + std::u32string out; + out.reserve(input.size() + 8); + for (char32_t c : input) { + if (c == U'[' || c == U']') { + out += U' '; + out += c; + out += U' '; + } else + out += c; + } + return encode(collapse(out)); +} +bool all_of(std::u32string_view s, std::u32string_view set) { + return std::all_of(s.begin(), s.end(), + [&](char32_t c) { return set.find(c) != set.npos; }); +} +std::u32string detokenize(const std::u32string &input) { + auto text = replace_all(U" " + input + U" ", U" @-@ ", U"-"); + std::vector tokens; + for (size_t i = 0; i < text.size();) { + if (space(text[i])) { + ++i; + continue; + } + size_t begin = i; + while (i < text.size() && !space(text[i])) + ++i; + tokens.emplace_back(text.data() + begin, i - begin); + } + std::u32string out; + out.reserve(text.size()); + bool prepend = true; + std::unordered_map quotes; + for (size_t i = 0; i < tokens.size(); ++i) { + auto token = tokens[i]; + bool currency = std::all_of(token.begin(), token.end(), [](char32_t c) { + return contains(c, currency_symbols) || + std::u32string_view(U"([{¿¡").find(c) != std::u32string_view::npos; + }); + if (contains(token.front(), cjk)) { + if ((!i || !contains(tokens[i - 1].back(), cjk)) && prepend) + out += U' '; + out += token; + prepend = true; + } else if (currency) { + if (prepend) + out += U' '; + out += token; + prepend = false; + } else if (all_of(token, U",.?!:;\\%}])") || + (i && token.size() > 1 && token[0] == U'\'' && + contains(token[1], alpha))) { + out += token; + prepend = true; + } else if (all_of(token, U"'\"„“\x60")) { + std::u32string key = + all_of(token, U"„“”") ? U"\"" : std::u32string(token); + auto &count = quotes[key]; + if (count % 2 == 0) { + if (token == U"'" && i && tokens[i - 1].back() == U's') { + out += token; + prepend = true; + } else { + if (prepend) + out += U' '; + out += token; + prepend = false; + ++count; + } + } else { + out += token; + prepend = true; + ++count; + } + } else { + if (prepend) + out += U' '; + out += token; + prepend = true; + } + } + return collapse(out); +} +struct Cell { + char32_t value; + bool appended_space = false; + bool equals(char32_t c) const { return value == c && !appended_space; } + void append_space() { + if (!value) { + value = U' '; + appended_space = false; + } else + appended_space = true; + } +}; +std::u32string restore(std::u32string input, const std::u32string &normalized) { + if (input.find(U"\x60\x60") != input.npos && + normalized.find(U"\x60\x60") == normalized.npos) + input = replace_all(input, U"\x60\x60", U"\""); + std::vector cells; + cells.reserve(normalized.size()); + for (auto c : normalized) + cells.push_back({c, false}); + // Preserve Python's stable character slots: removal does not shift later indices. + for (char32_t punct : + std::u32string_view(U"!\"#$%&'()*+,-./:;<=>?@[\\]^_\x60{|}~")) { + size_t count_in = std::count(input.begin(), input.end(), punct); + if (!count_in) + continue; + size_t count_out = + std::count_if(cells.begin(), cells.end(), + [&](const Cell &c) { return c.equals(punct); }); + bool equal = count_in == count_out; + size_t j = 0; + for (size_t i = 0; i < input.size(); ++i) { + if (input[i] != punct) + continue; + while (j < cells.size() && !cells[j].equals(punct)) + ++j; + if (j == cells.size()) + break; + bool valid = (i && j && cells[j - 1].equals(input[i - 1])) || + (i + 1 < input.size() && j + 1 < cells.size() && + cells[j + 1].equals(input[i + 1])); + if (!equal && !valid) + continue; + if (i && j) { + if (cells[j - 1].equals(U' ') && input[i - 1] != U' ') + cells[j - 1] = {0, false}; + else if (!cells[j - 1].equals(U' ') && input[i - 1] == U' ') + cells[j - 1].append_space(); + } + if (i + 1 < input.size() && j + 1 < cells.size()) { + if (cells[j + 1].equals(U' ') && input[i + 1] != U' ') + cells[j + 1] = {0, false}; + else if (!cells[j + 1].equals(U' ') && input[i + 1] == U' ') + cells[j].append_space(); + } + ++j; + } + } + std::u32string out; + out.reserve(cells.size()); + for (auto cell : cells) { + if (cell.value) + out += cell.value; + if (cell.appended_space) + out += U' '; + } + return collapse(out); +} +std::string finish(const std::u32string &original, std::string_view output) { + return encode(restore(original, detokenize(decode(output)))); +} +} // namespace tn_text diff --git a/tools/text_processing_deployment/native_runtime/text_tables.h b/tools/text_processing_deployment/native_runtime/text_tables.h new file mode 100644 index 000000000..7c049d4be --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/text_tables.h @@ -0,0 +1,473 @@ +// Generated by export_text_tables.py; do not edit. +// Python Unicode 15.0.0; sacremoses 0.2.0. +// Sacremoses-derived tables are covered by LICENSE.sacremoses. +#pragma once +namespace tn_text { +struct Range { char32_t first, last; }; +inline constexpr Range whitespace[] = { + {0x9,0xd}, + {0x1c,0x20}, + {0x85,0x85}, + {0xa0,0xa0}, + {0x1680,0x1680}, + {0x2000,0x200a}, + {0x2028,0x2029}, + {0x202f,0x202f}, + {0x205f,0x205f}, + {0x3000,0x3000}, +}; +inline constexpr Range alpha[] = { + {0x41,0x5a}, + {0x61,0x7a}, + {0xaa,0xaa}, + {0xb5,0xb5}, + {0xba,0xba}, + {0xc0,0xd6}, + {0xd8,0xf6}, + {0xf8,0x2c1}, + {0x2c6,0x2d1}, + {0x2e0,0x2e4}, + {0x2ec,0x2ec}, + {0x2ee,0x2ee}, + {0x345,0x345}, + {0x370,0x374}, + {0x376,0x377}, + {0x37a,0x37d}, + {0x37f,0x37f}, + {0x386,0x386}, + {0x388,0x38a}, + {0x38c,0x38c}, + {0x38e,0x3a1}, + {0x3a3,0x3f5}, + {0x3f7,0x481}, + {0x48a,0x52f}, + {0x531,0x556}, + {0x559,0x559}, + {0x561,0x587}, + {0x5b0,0x5bd}, + {0x5bf,0x5bf}, + {0x5c1,0x5c2}, + {0x5c4,0x5c5}, + {0x5c7,0x5c7}, + {0x5d0,0x5ea}, + {0x5f0,0x5f2}, + {0x610,0x61a}, + {0x620,0x657}, + {0x659,0x65f}, + {0x66e,0x6d3}, + {0x6d5,0x6dc}, + {0x6e1,0x6e8}, + {0x6ed,0x6ef}, + {0x6fa,0x6fc}, + {0x6ff,0x6ff}, + {0x710,0x73f}, + {0x74d,0x7b1}, + {0x7ca,0x7ea}, + {0x7f4,0x7f5}, + {0x7fa,0x7fa}, + {0x800,0x817}, + {0x81a,0x82c}, + {0x840,0x858}, + {0x8a0,0x8b2}, + {0x8e4,0x8e9}, + {0x8f0,0x93b}, + {0x93d,0x94c}, + {0x94e,0x950}, + {0x955,0x963}, + {0x971,0x983}, + {0x985,0x98c}, + {0x98f,0x990}, + {0x993,0x9a8}, + {0x9aa,0x9b0}, + {0x9b2,0x9b2}, + {0x9b6,0x9b9}, + {0x9bd,0x9c4}, + {0x9c7,0x9c8}, + {0x9cb,0x9cc}, + {0x9ce,0x9ce}, + {0x9d7,0x9d7}, + {0x9dc,0x9dd}, + {0x9df,0x9e3}, + {0x9f0,0x9f1}, + {0xa01,0xa03}, + {0xa05,0xa0a}, + {0xa0f,0xa10}, + {0xa13,0xa28}, + {0xa2a,0xa30}, + {0xa32,0xa33}, + {0xa35,0xa36}, + {0xa38,0xa39}, + {0xa3e,0xa42}, + {0xa47,0xa48}, + {0xa4b,0xa4c}, + {0xa51,0xa51}, + {0xa59,0xa5c}, + {0xa5e,0xa5e}, + {0xa70,0xa75}, + {0xa81,0xa83}, + {0xa85,0xa8d}, + {0xa8f,0xa91}, + {0xa93,0xaa8}, + {0xaaa,0xab0}, + {0xab2,0xab3}, + {0xab5,0xab9}, + {0xabd,0xac5}, + {0xac7,0xac9}, + {0xacb,0xacc}, + {0xad0,0xad0}, + {0xae0,0xae3}, + {0xb01,0xb03}, + {0xb05,0xb0c}, + {0xb0f,0xb10}, + {0xb13,0xb28}, + {0xb2a,0xb30}, + {0xb32,0xb33}, + {0xb35,0xb39}, + {0xb3d,0xb44}, + {0xb47,0xb48}, + {0xb4b,0xb4c}, + {0xb56,0xb57}, + {0xb5c,0xb5d}, + {0xb5f,0xb63}, + {0xb71,0xb71}, + {0xb82,0xb83}, + {0xb85,0xb8a}, + {0xb8e,0xb90}, + {0xb92,0xb95}, + {0xb99,0xb9a}, + {0xb9c,0xb9c}, + {0xb9e,0xb9f}, + {0xba3,0xba4}, + {0xba8,0xbaa}, + {0xbae,0xbb9}, + {0xbbe,0xbc2}, + {0xbc6,0xbc8}, + {0xbca,0xbcc}, + {0xbd0,0xbd0}, + {0xbd7,0xbd7}, + {0xc00,0xc03}, + {0xc05,0xc0c}, + {0xc0e,0xc10}, + {0xc12,0xc28}, + {0xc2a,0xc39}, + {0xc3d,0xc44}, + {0xc46,0xc48}, + {0xc4a,0xc4c}, + {0xc55,0xc56}, + {0xc58,0xc59}, + {0xc60,0xc63}, + {0xc81,0xc83}, + {0xc85,0xc8c}, + {0xc8e,0xc90}, + {0xc92,0xca8}, + {0xcaa,0xcb3}, + {0xcb5,0xcb9}, + {0xcbd,0xcc4}, + {0xcc6,0xcc8}, + {0xcca,0xccc}, + {0xcd5,0xcd6}, + {0xcde,0xcde}, + {0xce0,0xce3}, + {0xcf1,0xcf2}, + {0xd01,0xd03}, + {0xd05,0xd0c}, + {0xd0e,0xd10}, + {0xd12,0xd3a}, + {0xd3d,0xd44}, + {0xd46,0xd48}, + {0xd4a,0xd4c}, + {0xd4e,0xd4e}, + {0xd57,0xd57}, + {0xd60,0xd63}, + {0xd7a,0xd7f}, + {0xd82,0xd83}, + {0xd85,0xd96}, + {0xd9a,0xdb1}, + {0xdb3,0xdbb}, + {0xdbd,0xdbd}, + {0xdc0,0xdc6}, + {0xdcf,0xdd4}, + {0xdd6,0xdd6}, + {0xdd8,0xddf}, + {0xdf2,0xdf3}, + {0xe01,0xe3a}, + {0xe40,0xe46}, + {0xe4d,0xe4d}, + {0xe81,0xe82}, + {0xe84,0xe84}, + {0xe87,0xe88}, + {0xe8a,0xe8a}, + {0xe8d,0xe8d}, + {0xe94,0xe97}, + {0xe99,0xe9f}, + {0xea1,0xea3}, + {0xea5,0xea5}, + {0xea7,0xea7}, + {0xeaa,0xeab}, + {0xead,0xeb9}, + {0xebb,0xebd}, + {0xec0,0xec4}, + {0xec6,0xec6}, + {0xecd,0xecd}, + {0xedc,0xedf}, + {0xf00,0xf00}, + {0xf40,0xf47}, + {0xf49,0xf6c}, + {0xf71,0xf81}, + {0xf88,0xf97}, + {0xf99,0xfbc}, + {0x1000,0x1036}, + {0x1038,0x1038}, + {0x103b,0x103f}, + {0x1050,0x1062}, + {0x1065,0x1068}, + {0x106e,0x1086}, + {0x108e,0x108e}, + {0x109c,0x109d}, + {0x10a0,0x10c5}, + {0x10c7,0x10c7}, + {0x10cd,0x10cd}, + {0x10d0,0x10fa}, + {0x10fc,0x1248}, + {0x124a,0x124d}, + {0x1250,0x1256}, + {0x1258,0x1258}, + {0x125a,0x125d}, + {0x1260,0x1288}, + {0x128a,0x128d}, + {0x1290,0x12b0}, + {0x12b2,0x12b5}, + {0x12b8,0x12be}, + {0x12c0,0x12c0}, + {0x12c2,0x12c5}, + {0x12c8,0x12d6}, + {0x12d8,0x1310}, + {0x1312,0x1315}, + {0x1318,0x135a}, + {0x135f,0x135f}, + {0x1380,0x138f}, + {0x13a0,0x13f4}, + {0x1401,0x166c}, + {0x166f,0x167f}, + {0x1681,0x169a}, + {0x16a0,0x16ea}, + {0x16ee,0x16f8}, + {0x1700,0x170c}, + {0x170e,0x1713}, + {0x1720,0x1733}, + {0x1740,0x1753}, + {0x1760,0x176c}, + {0x176e,0x1770}, + {0x1772,0x1773}, + {0x1780,0x17b3}, + {0x17b6,0x17c8}, + {0x17d7,0x17d7}, + {0x17dc,0x17dc}, + {0x1820,0x1877}, + {0x1880,0x18aa}, + {0x18b0,0x18f5}, + {0x1900,0x191e}, + {0x1920,0x192b}, + {0x1930,0x1938}, + {0x1950,0x196d}, + {0x1970,0x1974}, + {0x1980,0x19ab}, + {0x19b0,0x19c9}, + {0x1a00,0x1a1b}, + {0x1a20,0x1a5e}, + {0x1a61,0x1a74}, + {0x1aa7,0x1aa7}, + {0x1b00,0x1b33}, + {0x1b35,0x1b43}, + {0x1b45,0x1b4b}, + {0x1b80,0x1ba9}, + {0x1bac,0x1baf}, + {0x1bba,0x1be5}, + {0x1be7,0x1bf1}, + {0x1c00,0x1c35}, + {0x1c4d,0x1c4f}, + {0x1c5a,0x1c7d}, + {0x1ce9,0x1cec}, + {0x1cee,0x1cf3}, + {0x1cf5,0x1cf6}, + {0x1d00,0x1dbf}, + {0x1de7,0x1df4}, + {0x1e00,0x1f15}, + {0x1f18,0x1f1d}, + {0x1f20,0x1f45}, + {0x1f48,0x1f4d}, + {0x1f50,0x1f57}, + {0x1f59,0x1f59}, + {0x1f5b,0x1f5b}, + {0x1f5d,0x1f5d}, + {0x1f5f,0x1f7d}, + {0x1f80,0x1fb4}, + {0x1fb6,0x1fbc}, + {0x1fbe,0x1fbe}, + {0x1fc2,0x1fc4}, + {0x1fc6,0x1fcc}, + {0x1fd0,0x1fd3}, + {0x1fd6,0x1fdb}, + {0x1fe0,0x1fec}, + {0x1ff2,0x1ff4}, + {0x1ff6,0x1ffc}, + {0x2071,0x2071}, + {0x207f,0x207f}, + {0x2090,0x209c}, + {0x2102,0x2102}, + {0x2107,0x2107}, + {0x210a,0x2113}, + {0x2115,0x2115}, + {0x2119,0x211d}, + {0x2124,0x2124}, + {0x2126,0x2126}, + {0x2128,0x2128}, + {0x212a,0x212d}, + {0x212f,0x2139}, + {0x213c,0x213f}, + {0x2145,0x2149}, + {0x214e,0x214e}, + {0x2160,0x2188}, + {0x24b6,0x24e9}, + {0x25cc,0x25cc}, + {0x2c00,0x2c2e}, + {0x2c30,0x2c5e}, + {0x2c60,0x2ce4}, + {0x2ceb,0x2cee}, + {0x2cf2,0x2cf3}, + {0x2d00,0x2d25}, + {0x2d27,0x2d27}, + {0x2d2d,0x2d2d}, + {0x2d30,0x2d67}, + {0x2d6f,0x2d6f}, + {0x2d80,0x2d96}, + {0x2da0,0x2da6}, + {0x2da8,0x2dae}, + {0x2db0,0x2db6}, + {0x2db8,0x2dbe}, + {0x2dc0,0x2dc6}, + {0x2dc8,0x2dce}, + {0x2dd0,0x2dd6}, + {0x2dd8,0x2dde}, + {0x2de0,0x2dff}, + {0x2e2f,0x2e2f}, + {0x3006,0x3006}, + {0x3031,0x3035}, + {0x303c,0x303c}, + {0x3041,0x3096}, + {0x309d,0x309f}, + {0x30a1,0x30fa}, + {0x30fc,0x30ff}, + {0x3105,0x312d}, + {0x3131,0x318e}, + {0x31a0,0x31ba}, + {0x31f0,0x31ff}, + {0xa000,0xa48c}, + {0xa4d0,0xa4fd}, + {0xa500,0xa60c}, + {0xa610,0xa61f}, + {0xa62a,0xa62b}, + {0xa640,0xa66e}, + {0xa674,0xa67b}, + {0xa67f,0xa69d}, + {0xa69f,0xa6ef}, + {0xa717,0xa71f}, + {0xa722,0xa788}, + {0xa78b,0xa78e}, + {0xa790,0xa7ad}, + {0xa7b0,0xa7b1}, + {0xa7f7,0xa801}, + {0xa803,0xa805}, + {0xa807,0xa80a}, + {0xa80c,0xa827}, + {0xa840,0xa873}, + {0xa880,0xa8c3}, + {0xa8f2,0xa8f7}, + {0xa8fb,0xa8fb}, + {0xa90a,0xa92a}, + {0xa930,0xa952}, + {0xa960,0xa97c}, + {0xa980,0xa9b2}, + {0xa9b4,0xa9bf}, + {0xa9cf,0xa9cf}, + {0xa9e0,0xa9e4}, + {0xa9e6,0xa9ef}, + {0xa9fa,0xa9fe}, + {0xaa00,0xaa36}, + {0xaa40,0xaa4d}, + {0xaa60,0xaa76}, + {0xaa7a,0xaa7a}, + {0xaa7e,0xaabe}, + {0xaac0,0xaac0}, + {0xaac2,0xaac2}, + {0xaadb,0xaadd}, + {0xaae0,0xaaef}, + {0xaaf2,0xaaf5}, + {0xab01,0xab06}, + {0xab09,0xab0e}, + {0xab11,0xab16}, + {0xab20,0xab26}, + {0xab28,0xab2e}, + {0xab30,0xab5a}, + {0xab5c,0xab5f}, + {0xab64,0xab65}, + {0xabc0,0xabea}, + {0xd7b0,0xd7c6}, + {0xd7cb,0xd7fb}, + {0xfb00,0xfb06}, + {0xfb13,0xfb17}, + {0xfb1d,0xfb28}, + {0xfb2a,0xfb36}, + {0xfb38,0xfb3c}, + {0xfb3e,0xfb3e}, + {0xfb40,0xfb41}, + {0xfb43,0xfb44}, + {0xfb46,0xfbb1}, + {0xfbd3,0xfd3d}, + {0xfd50,0xfd8f}, + {0xfd92,0xfdc7}, + {0xfdf0,0xfdfb}, + {0xfe70,0xfe74}, + {0xfe76,0xfefc}, + {0xff21,0xff3a}, + {0xff41,0xff5a}, + {0xff66,0xffbe}, + {0xffc2,0xffc7}, + {0xffca,0xffcf}, + {0xffd2,0xffd7}, + {0xffda,0xffdc}, +}; +inline constexpr Range currency_symbols[] = { + {0xa,0xa}, + {0x24,0x24}, + {0xa2,0xa5}, + {0x58f,0x58f}, + {0x60b,0x60b}, + {0x9f2,0x9f3}, + {0x9fb,0x9fb}, + {0xaf1,0xaf1}, + {0xbf9,0xbf9}, + {0xe3f,0xe3f}, + {0x17db,0x17db}, + {0x20a0,0x20bd}, + {0xa838,0xa838}, + {0xfdfc,0xfdfc}, + {0xfe69,0xfe69}, + {0xff04,0xff04}, + {0xffe0,0xffe1}, + {0xffe5,0xffe6}, +}; +inline constexpr Range cjk[] = { + {0x1100,0x11ff}, + {0x2e80,0xa4cf}, + {0xa840,0xa87f}, + {0xac00,0xd7af}, + {0xf900,0xfaff}, + {0xfe30,0xfe4f}, + {0xff65,0xffdc}, + {0x17000,0x18aff}, + {0x1b000,0x1b12f}, + {0x1b170,0x1b2ff}, + {0x20000,0x2ffff}, +}; +}