From 17b278303c3a8acc87117eacea73686c1041ec57 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Mon, 12 Oct 2026 14:35:49 -0400 Subject: [PATCH 1/2] Fix table evidence and add topology and order holdouts --- .github/workflows/ci.yml | 27 ++ CHANGELOG.md | 10 + README.md | 9 +- benchmark/eval/README.md | 7 + benchmark/eval/bounded_zig.py | 293 +++++++++++++++ benchmark/eval/reading_order_next/.gitignore | 3 + benchmark/eval/reading_order_next/README.md | 49 +++ .../order-next-accepted-separator-dev.pdf | Bin 0 -> 800 bytes .../order-next-accepted-separator-holdout.pdf | Bin 0 -> 812 bytes .../order-next-multipage-continuation-dev.pdf | Bin 0 -> 1017 bytes ...er-next-multipage-continuation-holdout.pdf | Bin 0 -> 1029 bytes ...er-next-quarter-turn-crop-userunit-dev.pdf | Bin 0 -> 778 bytes ...ext-quarter-turn-crop-userunit-holdout.pdf | Bin 0 -> 787 bytes ...r-next-rejected-separators-graphic-dev.pdf | Bin 0 -> 775 bytes ...xt-rejected-separators-graphic-holdout.pdf | Bin 0 -> 784 bytes .../corpus/order-next-reverse-stream-dev.pdf | Bin 0 -> 707 bytes .../order-next-reverse-stream-holdout.pdf | Bin 0 -> 716 bytes .../order-next-ruled-table-prose-dev.pdf | Bin 0 -> 1158 bytes .../order-next-ruled-table-prose-holdout.pdf | Bin 0 -> 1182 bytes .../order-next-section-column-reset-dev.pdf | Bin 0 -> 1153 bytes ...rder-next-section-column-reset-holdout.pdf | Bin 0 -> 1184 bytes benchmark/eval/reading_order_next/freeze.json | 71 ++++ benchmark/eval/reading_order_next/generate.py | 151 ++++++++ .../order-next-accepted-separator-dev.json | 43 +++ ...order-next-accepted-separator-holdout.json | 43 +++ ...order-next-multipage-continuation-dev.json | 39 ++ ...r-next-multipage-continuation-holdout.json | 39 ++ ...r-next-quarter-turn-crop-userunit-dev.json | 34 ++ ...xt-quarter-turn-crop-userunit-holdout.json | 34 ++ ...-next-rejected-separators-graphic-dev.json | 34 ++ ...t-rejected-separators-graphic-holdout.json | 34 ++ .../order-next-reverse-stream-dev.json | 34 ++ .../order-next-reverse-stream-holdout.json | 34 ++ .../order-next-ruled-table-prose-dev.json | 79 ++++ .../order-next-ruled-table-prose-holdout.json | 79 ++++ .../order-next-section-column-reset-dev.json | 97 +++++ ...der-next-section-column-reset-holdout.json | 97 +++++ .../eval/reading_order_next/manifest.tsv | 15 + .../eval/reading_order_next/metadata.jsonl | 14 + .../order-next-accepted-separator-dev.txt | 4 + .../order-next-accepted-separator-holdout.txt | 4 + .../order-next-multipage-continuation-dev.txt | 4 + ...er-next-multipage-continuation-holdout.txt | 4 + ...er-next-quarter-turn-crop-userunit-dev.txt | 3 + ...ext-quarter-turn-crop-userunit-holdout.txt | 3 + ...r-next-rejected-separators-graphic-dev.txt | 3 + ...xt-rejected-separators-graphic-holdout.txt | 3 + .../order-next-reverse-stream-dev.txt | 3 + .../order-next-reverse-stream-holdout.txt | 3 + .../order-next-ruled-table-prose-dev.txt | 8 + .../order-next-ruled-table-prose-holdout.txt | 8 + .../order-next-section-column-reset-dev.txt | 10 + ...rder-next-section-column-reset-holdout.txt | 10 + .../eval/reading_order_next/test_verify.py | 111 ++++++ .../order-next-accepted-separator-dev.json | 54 +++ ...order-next-accepted-separator-holdout.json | 54 +++ ...order-next-multipage-continuation-dev.json | 55 +++ ...r-next-multipage-continuation-holdout.json | 55 +++ ...r-next-quarter-turn-crop-userunit-dev.json | 41 ++ ...xt-quarter-turn-crop-userunit-holdout.json | 41 ++ ...-next-rejected-separators-graphic-dev.json | 41 ++ ...t-rejected-separators-graphic-holdout.json | 41 ++ .../truth/order-next-reverse-stream-dev.json | 41 ++ .../order-next-reverse-stream-holdout.json | 41 ++ .../order-next-ruled-table-prose-dev.json | 106 ++++++ .../order-next-ruled-table-prose-holdout.json | 106 ++++++ .../order-next-section-column-reset-dev.json | 132 +++++++ ...der-next-section-column-reset-holdout.json | 132 +++++++ benchmark/eval/reading_order_next/verify.py | 234 ++++++++++++ benchmark/eval/table_compare.py | 84 ++++- benchmark/eval/table_topology/README.md | 44 +++ benchmark/eval/table_topology/cases.json | 58 +++ .../table_topology/corpus/blank-zero-dev.pdf | Bin 0 -> 931 bytes .../corpus/blank-zero-holdout.pdf | Bin 0 -> 934 bytes .../corpus/boundary-ownership-dev.pdf | Bin 0 -> 979 bytes .../corpus/boundary-ownership-holdout.pdf | Bin 0 -> 982 bytes .../corpus/graphic-axis-dev.pdf | Bin 0 -> 708 bytes .../corpus/graphic-axis-holdout.pdf | Bin 0 -> 708 bytes .../corpus/reverse-multispan-dev.pdf | Bin 0 -> 1096 bytes .../corpus/reverse-multispan-holdout.pdf | Bin 0 -> 1102 bytes benchmark/eval/table_topology/freeze.json | 33 ++ benchmark/eval/table_topology/generate.py | 123 ++++++ benchmark/eval/table_topology/test_verify.py | 110 ++++++ .../table_topology/truth/blank-zero-dev.json | 66 ++++ .../truth/blank-zero-holdout.json | 66 ++++ .../truth/boundary-ownership-dev.json | 66 ++++ .../truth/boundary-ownership-holdout.json | 66 ++++ .../truth/graphic-axis-dev.json | 1 + .../truth/graphic-axis-holdout.json | 1 + .../truth/reverse-multispan-dev.json | 66 ++++ .../truth/reverse-multispan-holdout.json | 66 ++++ benchmark/eval/table_topology/verify.py | 136 +++++++ benchmark/eval/test_bounded_zig.py | 354 ++++++++++++++++++ docs/output-schema.md | 16 +- integrations/javascript/integration.test.mjs | 4 +- .../javascript/vector-routing.test.mjs | 4 +- src/adaptive.zig | 100 +++-- src/integration_test.zig | 18 +- src/layout.zig | 55 ++- src/main.zig | 4 +- src/native_layout.zig | 66 +++- src/specialist_protocol.zig | 2 +- 102 files changed, 4269 insertions(+), 64 deletions(-) create mode 100644 benchmark/eval/bounded_zig.py create mode 100644 benchmark/eval/reading_order_next/.gitignore create mode 100644 benchmark/eval/reading_order_next/README.md create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-accepted-separator-dev.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-accepted-separator-holdout.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-multipage-continuation-dev.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-multipage-continuation-holdout.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-quarter-turn-crop-userunit-dev.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-quarter-turn-crop-userunit-holdout.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-rejected-separators-graphic-dev.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-rejected-separators-graphic-holdout.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-reverse-stream-dev.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-reverse-stream-holdout.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-ruled-table-prose-dev.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-ruled-table-prose-holdout.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-section-column-reset-dev.pdf create mode 100644 benchmark/eval/reading_order_next/corpus/order-next-section-column-reset-holdout.pdf create mode 100644 benchmark/eval/reading_order_next/freeze.json create mode 100644 benchmark/eval/reading_order_next/generate.py create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-accepted-separator-dev.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-accepted-separator-holdout.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-multipage-continuation-dev.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-multipage-continuation-holdout.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-quarter-turn-crop-userunit-dev.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-quarter-turn-crop-userunit-holdout.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-rejected-separators-graphic-dev.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-rejected-separators-graphic-holdout.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-reverse-stream-dev.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-reverse-stream-holdout.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-ruled-table-prose-dev.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-ruled-table-prose-holdout.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-section-column-reset-dev.json create mode 100644 benchmark/eval/reading_order_next/graph_truth/order-next-section-column-reset-holdout.json create mode 100644 benchmark/eval/reading_order_next/manifest.tsv create mode 100644 benchmark/eval/reading_order_next/metadata.jsonl create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-accepted-separator-dev.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-accepted-separator-holdout.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-multipage-continuation-dev.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-multipage-continuation-holdout.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-quarter-turn-crop-userunit-dev.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-quarter-turn-crop-userunit-holdout.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-rejected-separators-graphic-dev.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-rejected-separators-graphic-holdout.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-reverse-stream-dev.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-reverse-stream-holdout.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-ruled-table-prose-dev.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-ruled-table-prose-holdout.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-section-column-reset-dev.txt create mode 100644 benchmark/eval/reading_order_next/page_text/order-next-section-column-reset-holdout.txt create mode 100644 benchmark/eval/reading_order_next/test_verify.py create mode 100644 benchmark/eval/reading_order_next/truth/order-next-accepted-separator-dev.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-accepted-separator-holdout.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-multipage-continuation-dev.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-multipage-continuation-holdout.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-quarter-turn-crop-userunit-dev.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-quarter-turn-crop-userunit-holdout.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-rejected-separators-graphic-dev.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-rejected-separators-graphic-holdout.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-reverse-stream-dev.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-reverse-stream-holdout.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-ruled-table-prose-dev.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-ruled-table-prose-holdout.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-section-column-reset-dev.json create mode 100644 benchmark/eval/reading_order_next/truth/order-next-section-column-reset-holdout.json create mode 100644 benchmark/eval/reading_order_next/verify.py create mode 100644 benchmark/eval/table_topology/README.md create mode 100644 benchmark/eval/table_topology/cases.json create mode 100644 benchmark/eval/table_topology/corpus/blank-zero-dev.pdf create mode 100644 benchmark/eval/table_topology/corpus/blank-zero-holdout.pdf create mode 100644 benchmark/eval/table_topology/corpus/boundary-ownership-dev.pdf create mode 100644 benchmark/eval/table_topology/corpus/boundary-ownership-holdout.pdf create mode 100644 benchmark/eval/table_topology/corpus/graphic-axis-dev.pdf create mode 100644 benchmark/eval/table_topology/corpus/graphic-axis-holdout.pdf create mode 100644 benchmark/eval/table_topology/corpus/reverse-multispan-dev.pdf create mode 100644 benchmark/eval/table_topology/corpus/reverse-multispan-holdout.pdf create mode 100644 benchmark/eval/table_topology/freeze.json create mode 100644 benchmark/eval/table_topology/generate.py create mode 100644 benchmark/eval/table_topology/test_verify.py create mode 100644 benchmark/eval/table_topology/truth/blank-zero-dev.json create mode 100644 benchmark/eval/table_topology/truth/blank-zero-holdout.json create mode 100644 benchmark/eval/table_topology/truth/boundary-ownership-dev.json create mode 100644 benchmark/eval/table_topology/truth/boundary-ownership-holdout.json create mode 100644 benchmark/eval/table_topology/truth/graphic-axis-dev.json create mode 100644 benchmark/eval/table_topology/truth/graphic-axis-holdout.json create mode 100644 benchmark/eval/table_topology/truth/reverse-multispan-dev.json create mode 100644 benchmark/eval/table_topology/truth/reverse-multispan-holdout.json create mode 100644 benchmark/eval/table_topology/verify.py create mode 100644 benchmark/eval/test_bounded_zig.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00b7983..1528011 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,6 +135,33 @@ jobs: - name: Build shared library run: zig build shared --summary all + - name: Verify frozen topology and emitted-order controls + run: | + python3 -m unittest discover -s benchmark/eval -p 'test_table_compare.py' + python3 -m unittest discover -s benchmark/eval -p 'test_bounded_zig.py' + python3 -m unittest discover -s benchmark/eval/table_topology -p 'test_*.py' + python3 -m unittest discover -s benchmark/eval/reading_order_next -p 'test_*.py' + python3 benchmark/eval/table_topology/generate.py --check + python3 benchmark/eval/reading_order_next/generate.py --check + for split in dev holdout; do + python3 benchmark/eval/table_topology/verify.py \ + --cli zig-out/bin/pdf-parser --split "$split" \ + --output "benchmark/eval/outputs/topology-ci/$split" + python3 benchmark/eval/reading_order_next/verify.py \ + --cli zig-out/bin/pdf-parser --split "$split" \ + --output "benchmark/eval/outputs/reading-order-ci/$split" + done + + - name: Upload topology and emitted-order evidence + if: always() + uses: actions/upload-artifact@v6 + with: + name: topology-and-emitted-order + path: | + benchmark/eval/outputs/topology-ci + benchmark/eval/outputs/reading-order-ci + if-no-files-found: warn + - uses: actions/setup-node@v6 with: node-version: "22" diff --git a/CHANGELOG.md b/CHANGELOG.md index d3947d9..d10679b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ### Added +- Frozen development and holdout suites check table topology, exact cell source + ownership, and emitted reading order independently of graph diagnostics. +- A project-local build monitor bounds observed compiler-cache allocation and + preserves logs and receipts outside the cache. - Schema 0.18 page geometry and pixel-to-PDF transforms preserve raw PDF coordinates through quarter-turn OCR, CropBox and UserUnit handling. - Schema 0.19 page and region route evidence explains bounded vector candidates @@ -16,6 +20,12 @@ ### Fixed +- Schema 0.19.1 keeps reconstructed table evidence at cell granularity, assigns + shared-rule text once, and retains prose surrounding a promoted table. +- Legacy native ordering resets columns at corroborated section headings. +- Topology comparison checks physical grid dimensions, streaming verification + checks page identity, and exact financial tokens cannot be confused with + substrings inside other values. - Rotated OCR preserves recognized word order. Inferred tables on rotated pages remain withheld until their topology is qualified. - Partially clipped native spans no longer suppress outlines outside their clip. diff --git a/README.md b/README.md index 79b81da..4d754c1 100644 --- a/README.md +++ b/README.md @@ -670,11 +670,12 @@ and ruling lines, invokes OCR only for scanned routes, then reconciles native, OCR, table, formula, and form spans with typed provenance. The versioned JSON, artifact JSONL, and streaming JSONL schema is currently -`0.19.0` in local development, with parser build identity `0.4.0-dev` after the +`0.19.1` in local development, with parser build identity `0.4.0-dev` after the `v0.3.0` release. Schema `0.17.0` landed in [PR #6](https://github.com/Tom-R-Main/pdf-parser/pull/6) -at `9cac608`, with all six CI checks green. Schemas `0.18` and `0.19` are locally -checked and committed on `codex/coordinate-vector-ocr`; remote CI and merge -remain separate. The `0.19` vector-routing continuation's 23 outlined-text and +at `9cac608`, with all six CI checks green. Schemas `0.18.0` and `0.19.0` landed +in [PR #7](https://github.com/Tom-R-Main/pdf-parser/pull/7) at `478d73a`, with all +six PR CI checks green. Schema `0.19.1` is a local working-tree continuation. +The `0.19` vector-routing continuation's 23 outlined-text and negative-control fixtures, 13 real CLI/SDK tests and both JSONL readers are qualified in the [implementation evidence](docs/parser-host-implementation-plan.md#bounded-vector-routing-019-locally-checked). Every emitted record carries a `provenance` envelope with document and diff --git a/benchmark/eval/README.md b/benchmark/eval/README.md index 72dc140..c041aca 100644 --- a/benchmark/eval/README.md +++ b/benchmark/eval/README.md @@ -1,5 +1,12 @@ # Evaluation Harness +New frozen [table topology controls](table_topology/README.md) and +[reading-order holdouts](reading_order_next/README.md) assess source ownership, +physical cell spans and emitted order separately. The earlier reading-order +promotion stop rule remains in force. Use `bounded_zig.py` for task-scoped +compiler caches and preserved verification logs; its observed budget is not a +filesystem quota. + `pdf-parser` treats evaluation as a first-class build target: ```sh diff --git a/benchmark/eval/bounded_zig.py b/benchmark/eval/bounded_zig.py new file mode 100644 index 0000000..ec9fb6d --- /dev/null +++ b/benchmark/eval/bounded_zig.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Run Zig builds with an observed allocation budget and retained evidence. + +This monitor is not a filesystem quota: allocation may overshoot between scans. +It never removes caches or evidence, and terminates only its own process group. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import sys +import time +import uuid +from datetime import datetime, timezone + + +GIB = 1024**3 +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CACHE = REPO_ROOT / ".zig-cache/topology-order-2026-10-12" +DEFAULT_OUTPUT = REPO_ROOT / "benchmark/eval/outputs/topology-order-2026-10-12" + + +def allocated_bytes(root: Path) -> int: + """Count allocated filesystem blocks, without following directory symlinks. + + Hard-linked names are counted separately, so the measurement is conservative. + Files disappearing during a compiler operation are expected; other errors fail + closed and stop the owned build. + """ + total = root.lstat().st_blocks * 512 + + def walk_error(error: OSError) -> None: + if not isinstance(error, FileNotFoundError): + raise error + + for directory, dirs, files in os.walk(root, followlinks=False, onerror=walk_error): + for name in dirs + files: + try: + total += (Path(directory) / name).lstat().st_blocks * 512 + except FileNotFoundError: + pass + return total + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for part in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(part) + return digest.hexdigest() + + +def checked_path(path: Path, repo_root: Path) -> Path: + """Reject paths outside the project and symlinked path components.""" + path = Path(os.path.abspath(path)) + path.relative_to(repo_root) + for component in (path, *path.parents): + if component.is_symlink(): + raise ValueError(f"symlinked path is not owned by this runner: {component}") + if component == repo_root: + break + return path + + +def stop_group(proc: subprocess.Popen, grace_seconds: float = 0.5) -> None: + """Keep the leader unreaped until its process group has been signalled.""" + def send(sig: int) -> None: + try: + os.killpg(proc.pid, sig) + except ProcessLookupError: + pass + except PermissionError: + # Darwin returns EPERM for a group containing only unreaped zombies. + # Ignore that case only after checking states; a real denial remains + # an error rather than being presented as successful termination. + states = subprocess.run( + ["ps", "-ax", "-o", "pgid=,stat="], capture_output=True, text=True, + check=True, + ).stdout.splitlines() + for row in states: + fields = row.split() + if len(fields) == 2 and fields[0] == str(proc.pid) and not fields[1].startswith("Z"): + raise + + send(signal.SIGTERM) + time.sleep(grace_seconds) + send(signal.SIGKILL) + proc.wait(timeout=5) + + +def run_command( + command: list[str], *, cache_dir: Path, log_path: Path, receipt_path: Path, + repo_root: Path = REPO_ROOT, budget_bytes: int = 6 * GIB, + min_free_bytes: int = 12 * GIB, poll_seconds: float = 1.0, +) -> int: + """Execute a command; the CLI supplies Zig's explicit cache flags as well.""" + if budget_bytes <= 0 or min_free_bytes < 0 or not 0 < poll_seconds <= 2: + raise ValueError("budget must be positive, free floor nonnegative, polling in (0, 2]") + repo_root = repo_root.resolve(strict=True) + cache_dir = checked_path(cache_dir, repo_root) + log_path = checked_path(log_path, repo_root) + receipt_path = checked_path(receipt_path, repo_root) + if cache_dir == repo_root or repo_root / ".zig-cache" == cache_dir: + raise ValueError("use an individual task cache, not the project or whole cache root") + if log_path == receipt_path: + raise ValueError("log and receipt must be distinct") + for output in (log_path, receipt_path): + if output == cache_dir or cache_dir in output.parents: + raise ValueError("verification evidence must be outside the monitored cache") + if output.exists(): + raise FileExistsError(f"preserved evidence already exists: {output}") + output.parent.mkdir(parents=True, exist_ok=True) + cache_dir.mkdir(parents=True, exist_ok=True) + global_cache = cache_dir / "global" + checked_path(global_cache, repo_root).mkdir(exist_ok=True) + executable = shutil.which(command[0]) + identity = { + "path": str(Path(executable).resolve()) if executable else command[0], + "sha256": sha256_file(Path(executable)) if executable else None, + } + head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=repo_root, capture_output=True, text=True, + check=False, + ) + diff = subprocess.run( + ["git", "diff", "--binary", "HEAD"], cwd=repo_root, capture_output=True, + check=False, + ) + receipt = { + "schema_version": "1.0", "command": command, "cwd": str(repo_root), + "executable": identity, "git_head": head.stdout.strip() if head.returncode == 0 else None, + "runner_sha256": sha256_file(Path(__file__)), + "tracked_diff_sha256": hashlib.sha256(diff.stdout).hexdigest() if diff.returncode == 0 else None, + "cache_dir": str(cache_dir), "global_cache_dir": str(global_cache), + "cache_environment": { + "ZIG_LOCAL_CACHE_DIR": str(cache_dir), + "ZIG_GLOBAL_CACHE_DIR": str(global_cache), + }, + "budget_allocated_bytes": budget_bytes, "minimum_free_bytes": min_free_bytes, + "poll_seconds": poll_seconds, "strict_filesystem_quota": False, + "started_at": datetime.now(timezone.utc).isoformat(), "sample_count": 0, + "peak_allocated_bytes": 0, "lowest_free_bytes": None, + "observed_budget_overshoot_bytes": 0, "status": "starting", "pid": None, + "deletions": [], "log_path": str(log_path), + } + proc = None + start = time.monotonic() + last_sample = start + receipt["maximum_sample_gap_seconds"] = 0.0 + + def sample() -> str | None: + nonlocal last_sample + size = allocated_bytes(cache_dir) + free = shutil.disk_usage(cache_dir).free + now = time.monotonic() + receipt["maximum_sample_gap_seconds"] = max(receipt["maximum_sample_gap_seconds"], now - last_sample) + last_sample = now + receipt["sample_count"] += 1 + receipt["peak_allocated_bytes"] = max(receipt["peak_allocated_bytes"], size) + receipt["lowest_free_bytes"] = min(receipt["lowest_free_bytes"], free) if receipt["lowest_free_bytes"] is not None else free + receipt["observed_budget_overshoot_bytes"] = max(receipt["observed_budget_overshoot_bytes"], size - budget_bytes, 0) + receipt["last_allocated_bytes"] = size + receipt["last_free_bytes"] = free + if size > budget_bytes: + return "allocation_budget_exceeded" + if free < min_free_bytes: + return "free_space_floor_crossed" + return None + + # Exclusive creation protects earlier verification results, including receipts + # from interrupted invocations. Keep the receipt handle through finalization. + with receipt_path.open("x", encoding="utf-8") as receipt_handle, log_path.open("xb") as log: + def interrupted(signum, _frame): + receipt["termination_signal"] = signum + raise KeyboardInterrupt + + previous_sigterm = signal.signal(signal.SIGTERM, interrupted) + result = 125 + try: + breach = sample() + if breach: + receipt["status"] = "preflight_" + breach + log.write((receipt["status"] + ": build not started\n").encode()) + else: + env = os.environ.copy() + env["ZIG_LOCAL_CACHE_DIR"] = str(cache_dir) + env["ZIG_GLOBAL_CACHE_DIR"] = str(global_cache) + proc = subprocess.Popen( + command, cwd=repo_root, env=env, stdin=subprocess.DEVNULL, + stdout=log, stderr=subprocess.STDOUT, start_new_session=True, + ) + receipt["pid"] = proc.pid + receipt["status"] = "running" + receipt_handle.write(json.dumps(receipt, indent=2) + "\n") + receipt_handle.flush() + while True: + # Sample before poll so a just-finished build cannot bypass the + # final budget check. + breach = sample() + if breach: + receipt["status"] = breach + stop_group(proc) + result = 124 + break + child_exit = proc.poll() + if child_exit is not None: + receipt["status"] = "completed" if child_exit == 0 else "command_failed" + result = child_exit if child_exit >= 0 else 128 - child_exit + break + time.sleep(poll_seconds) + except KeyboardInterrupt: + receipt["status"] = "interrupted" + result = 130 + if proc is not None: + stop_group(proc) + except Exception as error: + receipt["status"] = "runner_error" + receipt["error"] = f"{type(error).__name__}: {error}" + if proc is not None: + stop_group(proc) + finally: + signal.signal(signal.SIGTERM, previous_sigterm) + try: + final_breach = sample() + if final_breach: + receipt["final_breach"] = final_breach + # The compiler can finish between a loop observation and + # poll(). A final observation must still reject that success. + # Keep an earlier failure's reason and exit code intact. + if result == 0: + receipt["status"] = final_breach + result = 124 + except Exception as error: + receipt["final_measurement_error"] = f"{type(error).__name__}: {error}" + if result == 0: + receipt["status"] = "final_measurement_failed" + result = 125 + log.flush() + receipt["exit_code"] = result + receipt["command_exit_code"] = proc.returncode if proc is not None else None + receipt["elapsed_seconds"] = time.monotonic() - start + receipt["finished_at"] = datetime.now(timezone.utc).isoformat() + receipt["log_sha256"] = sha256_file(log_path) + receipt_handle.seek(0) + receipt_handle.truncate() + receipt_handle.write(json.dumps(receipt, indent=2, sort_keys=True) + "\n") + receipt_handle.flush() + return result + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + separator = argv.index("--") if "--" in argv else len(argv) + extra = argv[separator + 1:] if separator < len(argv) else [] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("step", choices=("build", "test")) + parser.add_argument("--zig", default="zig") + parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE) + parser.add_argument("--budget-gib", type=float, default=6) + parser.add_argument("--min-free-gib", type=float, default=12) + parser.add_argument("--poll-seconds", type=float, default=1) + parser.add_argument("--log", type=Path) + parser.add_argument("--receipt", type=Path) + args = parser.parse_args(argv[:separator]) + for value in extra: + if value.startswith(("--cache-dir", "--global-cache-dir", "-j")): + parser.error("extra arguments cannot override cache paths or job limit") + run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + "-" + uuid.uuid4().hex[:8] + cache = checked_path(args.cache_dir, REPO_ROOT) + command = [args.zig, "build"] + (["test"] if args.step == "test" else []) + # Zig 0.17 build accepts --cache-dir; its global cache is selected through + # ZIG_GLOBAL_CACHE_DIR, unlike compile subcommands accepting the global flag. + command += ["-j2", "--cache-dir", str(cache)] + extra + log = args.log or DEFAULT_OUTPUT / (run_id + ".log") + receipt = args.receipt or DEFAULT_OUTPUT / (run_id + ".json") + result = run_command( + command, cache_dir=cache, log_path=log, receipt_path=receipt, + budget_bytes=int(args.budget_gib * GIB), min_free_bytes=int(args.min_free_gib * GIB), + poll_seconds=args.poll_seconds, + ) + print(f"build exit {result}; receipt: {receipt}; log: {log}") + return result + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/eval/reading_order_next/.gitignore b/benchmark/eval/reading_order_next/.gitignore new file mode 100644 index 0000000..4accdf1 --- /dev/null +++ b/benchmark/eval/reading_order_next/.gitignore @@ -0,0 +1,3 @@ +# These small deterministic inputs and independent text truth belong in the pack. +!corpus/*.pdf +!page_text/*.txt diff --git a/benchmark/eval/reading_order_next/README.md b/benchmark/eval/reading_order_next/README.md new file mode 100644 index 0000000..d308c0a --- /dev/null +++ b/benchmark/eval/reading_order_next/README.md @@ -0,0 +1,49 @@ +# Fresh reading-order experiment + +This pack adds fourteen small PDFs in seven development/holdout families: +ruled table cells between prose, two-column sections that reset under a new +heading, reversed content-stream order, a gutter separator, rejected separators +crossing text or inside a graphic, an equivalent quarter-turn/CropBox/UserUnit +page, and a two-page continuation. Every text anchor is an independently +positioned PDF text object. Table cells contain separate objects and real rules. +The holdouts change anchor text and horizontal placement; these synthetic cases +do not establish performance on real documents. + +The generator writes `freeze.json` with acceptance criteria and PDF/truth SHA256 +before evaluation. It refuses to overwrite changed frozen data. Existing +reading-order packs and their failed promotion gates remain frozen. The default +extraction order remains `legacy`; this experiment does not promote graph order. + +```sh +python3 benchmark/eval/reading_order_next/generate.py --check +python3 -m unittest discover -s benchmark/eval/reading_order_next -p 'test_*.py' +python3 benchmark/eval/reading_order_next/verify.py \ + --cli zig-out/bin/pdf-parser --eval-cli zig-out/bin/pdf-parser-eval \ + --split dev \ + --output benchmark/eval/outputs/topology-order-2026-10-12/reading-order-baseline +``` + +After implementation and independent review, evaluate `--split holdout` once +into a new evidence directory. Preserve diagnostic failures and failed public +order checks; do not revise truth or thresholds using heldout output. A failed +check exits nonzero and retains stdout, stderr, commands, binary hashes and +the freeze hash. The verifier requires an empty output directory so a later run +cannot replace earlier evidence. No compiler invocation is needed by this pack. + +`report.json` scores anchor uniqueness and page assignment separately in spans +and blocks. Only block records supply public reading order. Page-local required +pairs and cross-page continuation pairs have separate scores. An optional eval +runner captures internal diagnostic scores independently, using graph truth +that excludes cross-page pairs because the current graph has page-local edges. +It reports unmet required-edge recall, invalid projection and cycles as explicit +diagnostic failures, separately from public-lane failures. +Diagnostic scores cannot qualify emitted order or document continuation. +Multiple anchors in one fused block can satisfy public order by text position; +internal graph anchor collisions remain visible in the separate diagnostic. + +Stream checks require a first manifest, sequential page starts/finishes, page +artifacts inside their lifecycle, and one completed document end. Artifact JSONL +requires a first manifest and excludes streaming lifecycle records. Both reject +CLI errors and lifecycle error records. OCR is disabled in both public lanes and +the diagnostic lane. Stored evaluation output is ignored by Git and retained +outside the frozen pack. diff --git a/benchmark/eval/reading_order_next/corpus/order-next-accepted-separator-dev.pdf b/benchmark/eval/reading_order_next/corpus/order-next-accepted-separator-dev.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c97679c2bdd28ecf3cb5de7937fe0607e4e3cf0a GIT binary patch literal 800 zcma)5%TB{E5WM><=28g`5IZ3WNQgsAD^WY)MhrHH9m52#H@mf!u#;zcQs?5 zSzOvpd1tmD&Ng@JozpPYP4kQ!$?=i$5g+({cwH*ZtQc@vuHH(e+9)`8u1xoEDY@lr zCBg-|EZ&`$>SvUX?F4=(oxRc)q^IIR$lVZ62&6=~FEOolaU_wUCz=V1DzKeo#GP{9 zSUF&a{3cUu9k;Jhq~nRjATuRxBe_F6`JESU@XS5sf6~}WT`i;0f5ezo*Q!``sVmo^ zMGqE(@?6kJu$Yjmg(wVz5F@NG#T|7E`5KG(AOFCV&)xco9c*HOiq~iZ4W?ebEtL@6*j= literal 0 HcmV?d00001 diff --git a/benchmark/eval/reading_order_next/corpus/order-next-accepted-separator-holdout.pdf b/benchmark/eval/reading_order_next/corpus/order-next-accepted-separator-holdout.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c715ee90db5925e08cb2edaee42e7598de5d96b4 GIT binary patch literal 812 zcma)4%TB{E5WM><=28g`5IZ3Wl`14kTOk4B(cG#YOuH?q(ijq#*H3cb58y{wr-hal z5S9=2cxPvJy|s&0XGiVEqW1Rw@+uT0%=JVx8t{^*TY`6_t99m^&++;z#9)KiF--8fvuj%kJ9(xzbxHG=W6JKyrUSi_ z#iiYpcV-LXY;(8X84Xk2G>_;>j*pa&xbFAiT~TRf#emCl^;Rm?LBY9mWxCC!8ZFAayP^S0x1#hOH8X%97$y8ie|#13T!7Cai^R& zR^C&H0*Pdbt#149wB16Hj&CdmnJH--$pr1>cWV3-)!b74Co|sA%xLrW1o zOV_PMKNdstSkOrDM+o_~5QSk7VuUYDad+K9zQiK_?=?*M+_`nEzKI1YUdGu%rx_L1 fn0M1>Ld3&hjGNcRc$Zcew(=SIby2Ie&pYA+ELYI2 literal 0 HcmV?d00001 diff --git a/benchmark/eval/reading_order_next/corpus/order-next-multipage-continuation-dev.pdf b/benchmark/eval/reading_order_next/corpus/order-next-multipage-continuation-dev.pdf new file mode 100644 index 0000000000000000000000000000000000000000..066d4443c1b24a4fac769a88d2f359906d575d8b GIT binary patch literal 1017 zcmcIjO;5r=5WVlOmbi0MNkQfgr#6*pXjThr#g#l}zi`}hXf6{~hK>vs{3q=Gy zXkycBy07zg-n^Nuo$hzH)OI3jZ|^U!LO}rL`c~}hfJ~pJ2y#z5JvJkdr+S1ILJH`? z?ykUmz=6E~N|?pTJyYh6!sTH$uyEP%yqMRkC3=00-Zey%=jFBY=~;vV+13_ke-3bb zk1p%$rBnFubrCGNKrezd%VQj5x@{ieGNAZag^;wuE08_3rY!nY99QkL_zwlFj*I1& z3d$XwW+9rRl?A-7vL9hSayOvDI?J=JKs^z6LbOvlSrHED5ZL5^);NHu89+LLjs4CA zOvzv{IYOT@WqFC4klsQk{gIqBz9E|_<`wabCC)|6`^6oo#LgPb{rWGf63VtbExfO? zFRTjxXq9^~&>tK7d!LwDpe-j0>Nie37C00EG)1s@AdaGX1VdP0iki<4<_EHHB;5a3ZN(hNUDnJ8c6Qx3KkRO)?L;)*KVIL2f&k3To!H$4nLSStWR1)qMgynig3t=z$OQ@CJLf7fNTOAz0Uqg z=MbjkF_;3OPs#G4!cEBTpp*T|Pj-Ihn<*AG_kxAarOdm=eW)eR+RR=1FT)bbwmdVu zud^=m|UW*CJXitem#{q6al0nSUeC%Q8R)eEHFibzz-B(SWKh2 zf~k<|U&UJMSTm&YTZxMTx^F939Hw8GD|J4`(%;`+{(_Lwd>iBZ%=sQzr%UJSNrPB4 J8lAJQ_ykr44153p literal 0 HcmV?d00001 diff --git a/benchmark/eval/reading_order_next/corpus/order-next-quarter-turn-crop-userunit-dev.pdf b/benchmark/eval/reading_order_next/corpus/order-next-quarter-turn-crop-userunit-dev.pdf new file mode 100644 index 0000000000000000000000000000000000000000..419a2574e7f924f41883b6c1e9124f4b339cbb06 GIT binary patch literal 778 zcmah{!A`sVxmTc;>CDa>3}7ri@S?af6{}0pg-bF0Y!{Iuy<2&cnC6mo+HR(6HH+zATP}XJ;apIhod8b zWyXno{z+Ko$z!`H11ONEdFJ7!!MxiIVttF)vqX@C#a*Zxb93SAG2&ug`3SWdz{sDp^@_~6jY#DgG# z`w8svXDNtujb=i$Q);L{#gL9^s_a97rWE3*GuY{LuVGYJpO=$eNT<+E|Dq6HIBRAkWPdJ;anSguOk1 zRnCcg{z+Ko$pgEn0w|EjMegA$VcvCv*j^(JED_{raTDss+@9z?<8c`|!twz_F*c1V zTz0)Zi*-oVmTKiG=7s6mN4Sb8KZ!N86MaqIoSoOOiXf&yM1hzCLmNy$=tdwfJ-W-P z;97Es-Y(pj#=!HbRnh2+3p`XLz^QYPqtft`>)J~V}_~6jY#JwPa z+bL}EXDNtujb}piGHR$m#gI*Cs_a8SM?ta^K{kVpQGfrWe+c8!`l6a{LUsrJ>`#^^ z!wuG~$nz5aBXyD=Bs<5d(dvRtu57G!qfocOi`k%w|P`NBWS)_%WJvXVjqkPwN4!+qSfl34a5h4<;xWS literal 0 HcmV?d00001 diff --git a/benchmark/eval/reading_order_next/corpus/order-next-rejected-separators-graphic-dev.pdf b/benchmark/eval/reading_order_next/corpus/order-next-rejected-separators-graphic-dev.pdf new file mode 100644 index 0000000000000000000000000000000000000000..55ceb23dc88eaf8354c8bf2c0eb0cb32af0119c2 GIT binary patch literal 775 zcmah{!A{#i5WVkLyi25VDBiWRHjxmAKq92JDiEs|#9@gisgdoa_NJx%Ne=u$`yQ3akaJD=o1yC8~X=bYtBzijBFI&AiAQ?ui81=62Ep+` zi7~_RAqQ&6&2m^W$2qGrVBPtGM=B-^QqDZm$ZNbin|#5%@?F_150KwslK+#Ea;Gh& zE{md~|1mO_vt{^uYId4j${;pYH~a9=(n9f!m3}!$txYTzc)<)0P7az^EahMM1~V~x ueT$9uvBYGrd7*V>MQy*qv+@@q<^Sxxtm0;KyWrZ;>$Ia-4F;3XQ}qmKQN*VJ literal 0 HcmV?d00001 diff --git a/benchmark/eval/reading_order_next/corpus/order-next-rejected-separators-graphic-holdout.pdf b/benchmark/eval/reading_order_next/corpus/order-next-rejected-separators-graphic-holdout.pdf new file mode 100644 index 0000000000000000000000000000000000000000..adf20ad98f9f014781f37d99c5479b52ed458218 GIT binary patch literal 784 zcmah{O;6iE5WVlOc$Y}+0le!^kPuP>i7G)=Aig0EOFT)9Y%jGpK>H_q=s$oz!i*EZ z5LI3Iu={4-%zNV*ogGhi%x^=Be!Zi41$xg=x`Ye?`lAdwun>5gkZM1_I+b%&-AhFXdU@V)gAeA zZU~?z*+Y(n>s|~(UQQEP4z@_H=)P%_P!z%$+~ zGaQdOP(yABhb42IvpNIToiBK#V!|LZi1#$|8lPs9FQ=1Z%q!QG&GIwy8%*+lvQlm~ zVAW+&RP;Ym#-au!cT3Hn)wxV$<9V|f4=pVe4_oQigEUE^1Pi=kh9@Wo%^Q~TKmCB2 wkWGKY_P4RfWN&%CwPi(Zzsj@nHzDO8?wsu8W`#R%Ti@$UwNazd<_7VvW;5f~LN>vYi2?-F;MqH{M+-4h6rEy7IRQO2_`~mz3-ZX7W zD}?1kJlWZq@!ERm4KBoGz^t$Dk54AR1~px=K%;spiZQb%aj1t*V8yMTB-WLu8O=kjph=p3`GZTyQmMZ0k0CGn$X`3jSHr)#F zDs&f~&O7y6%wqiv!VAb=p>vc=_QF^vG@S()&2Vn6gA54pZE&w_2$yiy>p#IPRYj7; z=MXNSAO1=4MNL^IQIz6;jKHlKYK7T>$RmZQY*w?}bh|BWB_SpPB_?7-=#6k#=uk^a7u&_4f6{~hK>vt4rF_S@ z*+chr-n{pAw$|@-uf(;_YTrMfUrc}nYBFcdCh+imiNM=Z%T&d{`!dD?Y*R3R+gpZN zM1^!dCT!|>Tdgt;0{BA`6)>t3Z+t*>&WK%Q2zKdx)tD7 z=r%l^x9UC2V*LWb^T=MIbCe7A%2+Ejodp=paAvN93<&TA)Nd>Z7jQ8Q?s`E7rl~5D zEWU(r4ng=Q%M(YeMG{3R{zvlbM_F<7(o}w{p{{eBF>_JO+AZ*0*KuJ6J4~4L zv>@AKo|(r9CTtV>6l7$p?aGhNTp}r|S6?TIRa+={gRp)qY0y^6e$gEE~iK0IAVi8e}DaE0yx;mCw6uQLHy&Jf#5=Wo!bQjH+o@8 z7^1^1oS!pOq~u8Z*MyBFOG~&vA>Koe814|K774+udi1q0$s>KNOSAd& z$>h(*XNm4bp|HF1l&mutZIf;A*`%2s+ZVX!=&!^OqS5dHf?HGC$|YD9)my97h5*Wr z%A^lVDLh6iK{jYxJndWcYf~)z5)?nB^h)oHUa{|tA4jleibKRXtUyZCMk|LIfl9U@ zSuMErRGp{XdXm+6ZonouVIfy2K{3f{g-EpHx9B*=k_F&l#B=Nuos7Wr;pCJpjG+2SL_PcVtuURD@NkUg5SYw&9eE?X@9yUb`mF*oz~d8DYqE>Tjng*i-d zAMr57UX4qmQ{xU4+~xgOkGhJTY&($^-r2NDe6JT@X%3r7Yp&yqGjoP>#rSsNkV!nP zIczb3b4v?~4pVrO-e6*gQ}GrX?P8&bIzR7pmK#?;AhYbJ0fpDhT6zhZhokh``8v;0 MB-vmv`7&ky0K}OgJ^%m! literal 0 HcmV?d00001 diff --git a/benchmark/eval/reading_order_next/corpus/order-next-ruled-table-prose-holdout.pdf b/benchmark/eval/reading_order_next/corpus/order-next-ruled-table-prose-holdout.pdf new file mode 100644 index 0000000000000000000000000000000000000000..daa597fbdc8cd4e7219c5719a7058c19602e6089 GIT binary patch literal 1182 zcma)6%Wm5+5WMp%_F|wtG$JL7ihv+Mo!BVQ6is8@f*cr$wiU=^OHp;&pX{N3(Ednw zWyy~?NiF!W8qe(PYL^^bUryeMlY|Za{QdQt3E*JwAK2L$1lfCHuzsNdhY-IV7CJ65<%WF{)rnpp1Kv zq8{8Z<(3(5m=tx)HP|I5EaDO^C?`d|5t(lFJvz#;WdZmxvKh{azKnoG!0?nqwuCp+ z@%yXsCFIuRRa2MNyoKxm#uRU+L@${K|qW=L`R;`1D0>rDv~8$yY{u6_c+u z`Z(Vgw8#1G_q))l(C$dvXQlsj#qlIK-ktH3#yeXyx!)Pk7uwHm1G{7T==6-?zA?Ue zaQWgWh8b)yf%{Akm@OvpggwDTh}-fMOAoO~BwL<$s;snYFP3TfQ-j3gXD!_j-Gx$n Q<$PObnx!9zifbXJ&Rxw%BjQQoTrEb!?L# zvZ6gza}Y10ALe=g!TuZd)Zai7Ml8>Hi|S512+_@`vIOGHa%u+)pz=`&dr1h{9qb(U zuE62g=n2?jdgEbS8izk1yM(pbBOjmi{s-i?&-5r<3 c3qnFC%ox4|7I%j8%KA!YCk;ib)jR8pcLEtJO8@`> literal 0 HcmV?d00001 diff --git a/benchmark/eval/reading_order_next/corpus/order-next-section-column-reset-holdout.pdf b/benchmark/eval/reading_order_next/corpus/order-next-section-column-reset-holdout.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b93b5f08c3e69807b87139e5642f1184391857cb GIT binary patch literal 1184 zcmb7E%}&EG48G?ne3=9XXtS;@NC=6+CL}=otT)6VZ0;y32_zYRFUf&7fEU5;#@JYg zM$|*&`unl#q;Yo8-H@B9XncIWy$cB;nCn}yvjai)a*q(~E3Zm3gy2jK(Ls|EE?{?8 zpdN4{@8<#IykOsqv>E7*TMTbJ^h@J#9kN^KWXl<1xtzh=qyCd1UFM*~d=&$Puh*~eb@E%Y9bouE1AJjL z_$TwHaqe9b&sClm*7<+6E)ACw)LEbVd1Sp-fAz-Is;}I)biQ~RoT_g|PbcP_2Xr!m zlXLoPF&8xZBAhO&B93CnVH%S((7ZtZ#1cAUzhbgUbNoBDy%>vR`jh9aDoV7~&C)Ag b5fVCo#_$y}xlx=~)>l3yk&8y7bJ`VOOL{Yz literal 0 HcmV?d00001 diff --git a/benchmark/eval/reading_order_next/freeze.json b/benchmark/eval/reading_order_next/freeze.json new file mode 100644 index 0000000..9147dfe --- /dev/null +++ b/benchmark/eval/reading_order_next/freeze.json @@ -0,0 +1,71 @@ +{ + "acceptance": { + "anchors": "exactly once on declared page in spans and blocks", + "diagnostics": "page-local only, report scores separately from emitted order", + "emitted_order": "all page-local and document precedence pairs in blocks", + "lifecycle": "manifest first; sequential page starts/finishes; completed document last" + }, + "policy": "legacy output remains default; graph diagnostics cannot authorize promotion; retain heldout failures without retuning", + "scope": "fresh experiment; previous frozen stop rules remain in force", + "sha256": { + "corpus/order-next-accepted-separator-dev.pdf": "7a1acf8e72d764f61ad9f9c1d445f0a2cb73261f9e7c22a9d6af4535682898b1", + "corpus/order-next-accepted-separator-holdout.pdf": "99dcfc4295c67e834e1012d344cf935eb224ec58b23da47a592f58ec8ac02c37", + "corpus/order-next-multipage-continuation-dev.pdf": "a464b2cd78f11eec8c498e98df4c28f5026af40ed62b456fa43ca1a61fd5d7b8", + "corpus/order-next-multipage-continuation-holdout.pdf": "6ca0223f63398b5dcd13e5a771900eb41bc64ea689dee6b66dce5e8ee469927a", + "corpus/order-next-quarter-turn-crop-userunit-dev.pdf": "4a0b5926a9cddd5c37c7fa3a6b0062762411fc168f775682e7600b02228a82ba", + "corpus/order-next-quarter-turn-crop-userunit-holdout.pdf": "9fa6fb7c659f8c22461cdf856a7095479f34250a046e27f35355d30357cf8176", + "corpus/order-next-rejected-separators-graphic-dev.pdf": "c2e3906686f71f3eab7d2ee0b60e53d4a48c98fcf0b44ac99de858e72540ae3a", + "corpus/order-next-rejected-separators-graphic-holdout.pdf": "9b1f443a4ca6760d75144bbe9c68cbaaaa052aa66b0e4985ca6df062debffbb7", + "corpus/order-next-reverse-stream-dev.pdf": "3cf5bf21857f2c708f9d60c0f747203b7d4da190caa594937d051feaba326e3d", + "corpus/order-next-reverse-stream-holdout.pdf": "8bf1f6becb9be6492176b9490a90613086872f5c680462b433ebacb6ab606084", + "corpus/order-next-ruled-table-prose-dev.pdf": "66ab7d33a8736830bb43165e05ddff1182febe11351b527ff0607a9e7ecdc6ba", + "corpus/order-next-ruled-table-prose-holdout.pdf": "179914af882f0997294a6a31b1127ae8922eb431772d495af298325ef06415a0", + "corpus/order-next-section-column-reset-dev.pdf": "6db03b1f343499a7c376382610be53b12336d1cefde912ee989deb7cf3e09bbe", + "corpus/order-next-section-column-reset-holdout.pdf": "7789bc92ff875420e839fa92046350e323b397f770b1164f1e209a8f4167e0fd", + "graph_truth/order-next-accepted-separator-dev.json": "19c89b5854068f806620d90055c5e5d8e4a2c752813673303227a2f215301819", + "graph_truth/order-next-accepted-separator-holdout.json": "3efecf89d83dc6c8cb3a515304c2e979e66c3a39e9a31ee474aac1717c0a8ccf", + "graph_truth/order-next-multipage-continuation-dev.json": "f685d9e5738ad351ea05def5f483386df010f0d0f1b07261efe04caa59b28ea7", + "graph_truth/order-next-multipage-continuation-holdout.json": "6574af11af2775351fac82bd53bbfa495b7059baeb0027f7687c7fdbd02e641e", + "graph_truth/order-next-quarter-turn-crop-userunit-dev.json": "1754a0e62caf9d1db6753ac7be5faf54763c15a5eca9cb8a1522e9f75b83b1c4", + "graph_truth/order-next-quarter-turn-crop-userunit-holdout.json": "5a03d5360c461682eefad6abb2f2d5f7592c098d623877c8dff5bdf5f9f94190", + "graph_truth/order-next-rejected-separators-graphic-dev.json": "1754a0e62caf9d1db6753ac7be5faf54763c15a5eca9cb8a1522e9f75b83b1c4", + "graph_truth/order-next-rejected-separators-graphic-holdout.json": "5a03d5360c461682eefad6abb2f2d5f7592c098d623877c8dff5bdf5f9f94190", + "graph_truth/order-next-reverse-stream-dev.json": "1754a0e62caf9d1db6753ac7be5faf54763c15a5eca9cb8a1522e9f75b83b1c4", + "graph_truth/order-next-reverse-stream-holdout.json": "5a03d5360c461682eefad6abb2f2d5f7592c098d623877c8dff5bdf5f9f94190", + "graph_truth/order-next-ruled-table-prose-dev.json": "e7718fea1602be78c6748368d905a0ddb4cc1ed25182f7eba6769791426ce34f", + "graph_truth/order-next-ruled-table-prose-holdout.json": "12eef641c920f428b960df1b4d2faef08a060e24ac3b01314da4bc1855b1b0cc", + "graph_truth/order-next-section-column-reset-dev.json": "800657cfa37cfd20d3abb2fcbc44378c851c0b934629915129b5ac6eb28b5711", + "graph_truth/order-next-section-column-reset-holdout.json": "21d28ceecf46f70ce98ab2a35339988fba43260b3ae836b6fecd2c04f76b61b7", + "manifest.tsv": "4758090980f1f76275510b3d2eb6299f31a513532a927b62b3108931837d5583", + "metadata.jsonl": "6eca95d518326fee0d2cc706dce8d63d8ade701e8f1992c24dc20f930647a25c", + "page_text/order-next-accepted-separator-dev.txt": "a7f9367e3a7e135c88846f95f5aab3b88cce4ef723c80f4e6c5ec06e19f12dbd", + "page_text/order-next-accepted-separator-holdout.txt": "899c6425181bf413ac56f265265d35a0f329615c76a601be2cb94225ff3f6456", + "page_text/order-next-multipage-continuation-dev.txt": "2bdae52887f2a48d61d973026d75b0957a1f42962ee07ae3df46b2d832030d41", + "page_text/order-next-multipage-continuation-holdout.txt": "1a59d59704b0a7bc3abeb3e814e0a6095215698521483e331195b8a34e2daa65", + "page_text/order-next-quarter-turn-crop-userunit-dev.txt": "0e0271126801a174110110582ac156261fac96a8cd7bce4f7b21202cce8b5792", + "page_text/order-next-quarter-turn-crop-userunit-holdout.txt": "56be8a1cf45120d74999e62a8a70e521ec954c098c4912f323f97ebfc1b5ad98", + "page_text/order-next-rejected-separators-graphic-dev.txt": "0e0271126801a174110110582ac156261fac96a8cd7bce4f7b21202cce8b5792", + "page_text/order-next-rejected-separators-graphic-holdout.txt": "56be8a1cf45120d74999e62a8a70e521ec954c098c4912f323f97ebfc1b5ad98", + "page_text/order-next-reverse-stream-dev.txt": "0e0271126801a174110110582ac156261fac96a8cd7bce4f7b21202cce8b5792", + "page_text/order-next-reverse-stream-holdout.txt": "56be8a1cf45120d74999e62a8a70e521ec954c098c4912f323f97ebfc1b5ad98", + "page_text/order-next-ruled-table-prose-dev.txt": "8253b0f6f2f3863e11aa763331d55b97c04c1c7d71b8bef82ae93355953717f8", + "page_text/order-next-ruled-table-prose-holdout.txt": "72642c73aa92ec6ef0e0a1d1359ca0274c612ff89ea4037fd39c0ac4f6a56420", + "page_text/order-next-section-column-reset-dev.txt": "567235a52db437a582590e4eba749cb16b1dc4ba096ae72c6ae1106cda688caa", + "page_text/order-next-section-column-reset-holdout.txt": "6216943d773918ddcbc1f4bb8e9a5994a9ddedb41a7e2e4d2f95ed88b90f8656", + "truth/order-next-accepted-separator-dev.json": "66ce324721246071f7731d6eca09466f5a685d50f29107b0ddd4e7af00a458da", + "truth/order-next-accepted-separator-holdout.json": "8d4a48146613e290ca73208635383504398fc24f28f0add9cf9f73547e3d791a", + "truth/order-next-multipage-continuation-dev.json": "68f8b54e4c81c87f9835ac59b571fd9e6ab77b5761c77b65477000c104afc8b0", + "truth/order-next-multipage-continuation-holdout.json": "5b4a06853cbde6b8343325b6f40118a6d31af4f41469fa68797211e4095ea1c7", + "truth/order-next-quarter-turn-crop-userunit-dev.json": "842d145123197ef606543b83719372f3bcd7e736fca0b3159254593ddcee0370", + "truth/order-next-quarter-turn-crop-userunit-holdout.json": "0a63d94b6c48499e3cf2eeecf7160c3dd5924f627ce1a858cf66cc2b2bf3da65", + "truth/order-next-rejected-separators-graphic-dev.json": "842d145123197ef606543b83719372f3bcd7e736fca0b3159254593ddcee0370", + "truth/order-next-rejected-separators-graphic-holdout.json": "0a63d94b6c48499e3cf2eeecf7160c3dd5924f627ce1a858cf66cc2b2bf3da65", + "truth/order-next-reverse-stream-dev.json": "842d145123197ef606543b83719372f3bcd7e736fca0b3159254593ddcee0370", + "truth/order-next-reverse-stream-holdout.json": "0a63d94b6c48499e3cf2eeecf7160c3dd5924f627ce1a858cf66cc2b2bf3da65", + "truth/order-next-ruled-table-prose-dev.json": "f357fefbdbf3db4d39795b99bbd3479f734c23a5344af60c3f4628fd35ba08b4", + "truth/order-next-ruled-table-prose-holdout.json": "e8db54b1a6197c8f64842b2f96e675530784af68fd66266aa199c9b420b95659", + "truth/order-next-section-column-reset-dev.json": "69352430497bf72368d3aa90742c3d1111f1fc3d9a2a678a9ea454fa94d9e6d4", + "truth/order-next-section-column-reset-holdout.json": "e11f2e7fb0a4f22d1f1d57ea679fcc72845432386879de25e7ab211893489ea4" + }, + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/generate.py b/benchmark/eval/reading_order_next/generate.py new file mode 100644 index 0000000..8b4dd91 --- /dev/null +++ b/benchmark/eval/reading_order_next/generate.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Create an independent, frozen reading-order experiment without dependencies.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +PREFIX = "benchmark/eval/reading_order_next" + + +def encoded(value): + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() + + +def page(items, graphics="", transformed=False, reverse=False): + return dict(items=items, graphics=graphics, transformed=transformed, reverse=reverse) + + +def fixtures(): + result = [] + for split, prefix, shift in (("dev", "DEV", 0), ("holdout", "SEALED", 11)): + def texts(specs): + return [(name, f"{prefix} {name.replace('_', ' ')}", x + shift, y, size) + for name, x, y, size in specs] + ruled = texts([("introduction", 60, 750, 12), ("header_left", 75, 680, 12), + ("header_right", 335, 680, 12), ("row_one_left", 75, 630, 12), + ("row_one_right", 335, 630, 12), ("row_two_left", 75, 580, 12), + ("row_two_right", 335, 580, 12), ("conclusion", 60, 480, 12)]) + grid = "0.7 w\n" + "\n".join( + [f"60 {y} m 552 {y} l S" for y in (700, 650, 600, 550)] + + [f"{x} 550 m {x} 700 l S" for x in (60, 306, 552)]) + sections = texts([("section_one", 60, 750, 18), ("first_left_open", 60, 700, 12), + ("first_left_close", 60, 650, 12), ("first_right_open", 330, 700, 12), + ("first_right_close", 330, 650, 12), ("section_two", 60, 560, 18), + ("second_left_open", 60, 510, 12), ("second_left_close", 60, 460, 12), + ("second_right_open", 330, 510, 12), ("second_right_close", 330, 460, 12)]) + linear = texts([("opening", 60, 740, 12), ("middle", 60, 610, 12), ("closing", 60, 480, 12)]) + columns = texts([("left_open", 60, 730, 12), ("left_close", 60, 630, 12), + ("right_open", 330, 730, 12), ("right_close", 330, 630, 12)]) + # Lines crossing text or lying inside graphics must not manufacture a new text region. + rejected = "0.5 w 40 733 m 550 733 l S\n40 430 220 110 re S\n150 440 m 150 530 l S" + continuation = [page(texts([("page_one_open", 60, 740, 12), ("page_one_continue", 60, 580, 12)]), reverse=True), + page(texts([("page_two_resume", 60, 740, 12), ("page_two_close", 60, 580, 12)]), reverse=True)] + cases = [("ruled-table-prose", [page(ruled, grid, reverse=True)]), + ("section-column-reset", [page(sections, reverse=True)]), + ("reverse-stream", [page(linear, reverse=True)]), + ("accepted-separator", [page(columns, "0.5 w 300 580 m 300 760 l S", reverse=True)]), + ("rejected-separators-graphic", [page(linear, rejected, reverse=True)]), + ("quarter-turn-crop-userunit", [page(linear, transformed=True, reverse=True)]), + ("multipage-continuation", continuation)] + for family, pages in cases: + nodes = [dict(id=item[0], text_anchor=item[1], page_index=index) + for index, p in enumerate(pages) for item in p["items"]] + edges = [[a["id"], b["id"]] for a, b in zip(nodes, nodes[1:])] + local = [edge for edge in edges if next(n["page_index"] for n in nodes if n["id"] == edge[0]) == + next(n["page_index"] for n in nodes if n["id"] == edge[1])] + result.append(dict(doc_id=f"order-next-{family}-{split}", family=family, split=split, + pages=pages, truth=dict(version=1, nodes=nodes, required_precedence=edges, + page_local_precedence=local, document_precedence=[e for e in edges if e not in local]))) + return result + + +def build_pdf(pages): + # Each anchor uses its own positioned text object; no line is pre-concatenated. + objects = [b"<< /Type /Catalog /Pages 2 0 R >>", b"", b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"] + page_numbers = [] + for p in pages: + page_number = len(objects) + 1 + page_numbers.append(page_number) + geometry = "/MediaBox [0 0 612 792]" + if p["transformed"]: + geometry = "/MediaBox [0 0 426 346] /CropBox [10 20 406 326] /Rotate 90 /UserUnit 2" + objects.append(f"<< /Type /Page /Parent 2 0 R {geometry} /Resources << /Font << /F1 3 0 R >> >> /Contents {page_number+1} 0 R >>".encode()) + content = ["q"] + if p["transformed"]: + content.append("0 0.5 -0.5 0 406 20 cm") + content.append(p["graphics"]) + items = list(reversed(p["items"])) if p["reverse"] else p["items"] + for _, text, x, y, size in items: + content.append(f"BT /F1 {size} Tf 1 0 0 1 {x} {y} Tm ({text}) Tj ET") + content.append("Q") + stream = ("\n".join(content) + "\n").encode("ascii") + objects.append(f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"endstream") + kids = " ".join(f"{n} 0 R" for n in page_numbers) + objects[1] = f"<< /Type /Pages /Count {len(pages)} /Kids [{kids}] >>".encode() + out = bytearray(b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\n") + offsets = [0] + for index, obj in enumerate(objects, 1): + offsets.append(len(out)) + out.extend(f"{index} 0 obj\n".encode() + obj + b"\nendobj\n") + xref = len(out) + out.extend(f"xref\n0 {len(objects)+1}\n0000000000 65535 f \n".encode()) + for offset in offsets[1:]: + out.extend(f"{offset:010d} 00000 n \n".encode()) + out.extend(f"trailer\n<< /Size {len(objects)+1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()) + return bytes(out) + + +def generated(): + files = {} + manifest = ["# category\tdoc_id\tpdf_path\ttruth_text_path\ttruth_table_json_path_optional\ttruth_reading_order_path_optional\ttruth_formula_path_optional\ttruth_formula_json_path_optional\ttruth_form_json_path_optional\ttruth_reading_graph_path_optional"] + metadata = [] + for fixture in fixtures(): + doc_id, truth = fixture["doc_id"], fixture["truth"] + files[f"corpus/{doc_id}.pdf"] = build_pdf(fixture["pages"]) + files[f"truth/{doc_id}.json"] = encoded(truth) + files[f"page_text/{doc_id}.txt"] = ("\n".join(n["text_anchor"] for n in truth["nodes"]) + "\n").encode() + # Internal graph truth deliberately excludes cross-page document edges. + graph = {k: truth[k] for k in ("version", "nodes")} + graph.update(required_precedence=truth["page_local_precedence"], forbidden_precedence=[], ambiguous_pairs=[], relations=[], valid_orders=[]) + files[f"graph_truth/{doc_id}.json"] = encoded(graph) + manifest.append("\t".join(["visual_truth", doc_id, f"{PREFIX}/corpus/{doc_id}.pdf", f"{PREFIX}/page_text/{doc_id}.txt", "", "", "", "", "", f"{PREFIX}/graph_truth/{doc_id}.json"])) + metadata.append(json.dumps(dict(doc_id=doc_id, family=fixture["family"], reading_order_split=fixture["split"], expected_ocr_pages=0), sort_keys=True)) + files["manifest.tsv"] = ("\n".join(manifest) + "\n").encode() + files["metadata.jsonl"] = ("\n".join(metadata) + "\n").encode() + return files + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true", help="Check frozen bytes without writing") + args = parser.parse_args() + files = generated() + freeze = dict(version=1, scope="fresh experiment; previous frozen stop rules remain in force", + policy="legacy output remains default; graph diagnostics cannot authorize promotion; retain heldout failures without retuning", + acceptance=dict(anchors="exactly once on declared page in spans and blocks", emitted_order="all page-local and document precedence pairs in blocks", lifecycle="manifest first; sequential page starts/finishes; completed document last", diagnostics="page-local only, report scores separately from emitted order"), + sha256={name: hashlib.sha256(data).hexdigest() for name, data in sorted(files.items())}) + frozen = ROOT / "freeze.json" + if frozen.exists() and frozen.read_bytes() != encoded(freeze): + raise SystemExit("Frozen experiment changed; do not retune heldout truth or overwrite freeze") + for name, data in files.items(): + path = ROOT / name + if args.check: + if not path.exists() or path.read_bytes() != data: + raise SystemExit(f"Stale generated file: {name}") + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + if args.check: + if not frozen.exists(): + raise SystemExit("Missing pre-evaluation freeze") + else: + frozen.write_bytes(encoded(freeze)) + print(f"{'Checked' if args.check else 'Frozen'} {len(files)} deterministic files, 14 fixtures") + + +if __name__ == "__main__": + main() diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-accepted-separator-dev.json b/benchmark/eval/reading_order_next/graph_truth/order-next-accepted-separator-dev.json new file mode 100644 index 0000000..6fb1af3 --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-accepted-separator-dev.json @@ -0,0 +1,43 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "left_open", + "page_index": 0, + "text_anchor": "DEV left open" + }, + { + "id": "left_close", + "page_index": 0, + "text_anchor": "DEV left close" + }, + { + "id": "right_open", + "page_index": 0, + "text_anchor": "DEV right open" + }, + { + "id": "right_close", + "page_index": 0, + "text_anchor": "DEV right close" + } + ], + "relations": [], + "required_precedence": [ + [ + "left_open", + "left_close" + ], + [ + "left_close", + "right_open" + ], + [ + "right_open", + "right_close" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-accepted-separator-holdout.json b/benchmark/eval/reading_order_next/graph_truth/order-next-accepted-separator-holdout.json new file mode 100644 index 0000000..1b469c9 --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-accepted-separator-holdout.json @@ -0,0 +1,43 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "left_open", + "page_index": 0, + "text_anchor": "SEALED left open" + }, + { + "id": "left_close", + "page_index": 0, + "text_anchor": "SEALED left close" + }, + { + "id": "right_open", + "page_index": 0, + "text_anchor": "SEALED right open" + }, + { + "id": "right_close", + "page_index": 0, + "text_anchor": "SEALED right close" + } + ], + "relations": [], + "required_precedence": [ + [ + "left_open", + "left_close" + ], + [ + "left_close", + "right_open" + ], + [ + "right_open", + "right_close" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-multipage-continuation-dev.json b/benchmark/eval/reading_order_next/graph_truth/order-next-multipage-continuation-dev.json new file mode 100644 index 0000000..490ca95 --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-multipage-continuation-dev.json @@ -0,0 +1,39 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "page_one_open", + "page_index": 0, + "text_anchor": "DEV page one open" + }, + { + "id": "page_one_continue", + "page_index": 0, + "text_anchor": "DEV page one continue" + }, + { + "id": "page_two_resume", + "page_index": 1, + "text_anchor": "DEV page two resume" + }, + { + "id": "page_two_close", + "page_index": 1, + "text_anchor": "DEV page two close" + } + ], + "relations": [], + "required_precedence": [ + [ + "page_one_open", + "page_one_continue" + ], + [ + "page_two_resume", + "page_two_close" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-multipage-continuation-holdout.json b/benchmark/eval/reading_order_next/graph_truth/order-next-multipage-continuation-holdout.json new file mode 100644 index 0000000..56b01b6 --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-multipage-continuation-holdout.json @@ -0,0 +1,39 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "page_one_open", + "page_index": 0, + "text_anchor": "SEALED page one open" + }, + { + "id": "page_one_continue", + "page_index": 0, + "text_anchor": "SEALED page one continue" + }, + { + "id": "page_two_resume", + "page_index": 1, + "text_anchor": "SEALED page two resume" + }, + { + "id": "page_two_close", + "page_index": 1, + "text_anchor": "SEALED page two close" + } + ], + "relations": [], + "required_precedence": [ + [ + "page_one_open", + "page_one_continue" + ], + [ + "page_two_resume", + "page_two_close" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-quarter-turn-crop-userunit-dev.json b/benchmark/eval/reading_order_next/graph_truth/order-next-quarter-turn-crop-userunit-dev.json new file mode 100644 index 0000000..24e5671 --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-quarter-turn-crop-userunit-dev.json @@ -0,0 +1,34 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "opening", + "page_index": 0, + "text_anchor": "DEV opening" + }, + { + "id": "middle", + "page_index": 0, + "text_anchor": "DEV middle" + }, + { + "id": "closing", + "page_index": 0, + "text_anchor": "DEV closing" + } + ], + "relations": [], + "required_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-quarter-turn-crop-userunit-holdout.json b/benchmark/eval/reading_order_next/graph_truth/order-next-quarter-turn-crop-userunit-holdout.json new file mode 100644 index 0000000..be12b7a --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-quarter-turn-crop-userunit-holdout.json @@ -0,0 +1,34 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "opening", + "page_index": 0, + "text_anchor": "SEALED opening" + }, + { + "id": "middle", + "page_index": 0, + "text_anchor": "SEALED middle" + }, + { + "id": "closing", + "page_index": 0, + "text_anchor": "SEALED closing" + } + ], + "relations": [], + "required_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-rejected-separators-graphic-dev.json b/benchmark/eval/reading_order_next/graph_truth/order-next-rejected-separators-graphic-dev.json new file mode 100644 index 0000000..24e5671 --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-rejected-separators-graphic-dev.json @@ -0,0 +1,34 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "opening", + "page_index": 0, + "text_anchor": "DEV opening" + }, + { + "id": "middle", + "page_index": 0, + "text_anchor": "DEV middle" + }, + { + "id": "closing", + "page_index": 0, + "text_anchor": "DEV closing" + } + ], + "relations": [], + "required_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-rejected-separators-graphic-holdout.json b/benchmark/eval/reading_order_next/graph_truth/order-next-rejected-separators-graphic-holdout.json new file mode 100644 index 0000000..be12b7a --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-rejected-separators-graphic-holdout.json @@ -0,0 +1,34 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "opening", + "page_index": 0, + "text_anchor": "SEALED opening" + }, + { + "id": "middle", + "page_index": 0, + "text_anchor": "SEALED middle" + }, + { + "id": "closing", + "page_index": 0, + "text_anchor": "SEALED closing" + } + ], + "relations": [], + "required_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-reverse-stream-dev.json b/benchmark/eval/reading_order_next/graph_truth/order-next-reverse-stream-dev.json new file mode 100644 index 0000000..24e5671 --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-reverse-stream-dev.json @@ -0,0 +1,34 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "opening", + "page_index": 0, + "text_anchor": "DEV opening" + }, + { + "id": "middle", + "page_index": 0, + "text_anchor": "DEV middle" + }, + { + "id": "closing", + "page_index": 0, + "text_anchor": "DEV closing" + } + ], + "relations": [], + "required_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-reverse-stream-holdout.json b/benchmark/eval/reading_order_next/graph_truth/order-next-reverse-stream-holdout.json new file mode 100644 index 0000000..be12b7a --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-reverse-stream-holdout.json @@ -0,0 +1,34 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "opening", + "page_index": 0, + "text_anchor": "SEALED opening" + }, + { + "id": "middle", + "page_index": 0, + "text_anchor": "SEALED middle" + }, + { + "id": "closing", + "page_index": 0, + "text_anchor": "SEALED closing" + } + ], + "relations": [], + "required_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-ruled-table-prose-dev.json b/benchmark/eval/reading_order_next/graph_truth/order-next-ruled-table-prose-dev.json new file mode 100644 index 0000000..2c84d5b --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-ruled-table-prose-dev.json @@ -0,0 +1,79 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "introduction", + "page_index": 0, + "text_anchor": "DEV introduction" + }, + { + "id": "header_left", + "page_index": 0, + "text_anchor": "DEV header left" + }, + { + "id": "header_right", + "page_index": 0, + "text_anchor": "DEV header right" + }, + { + "id": "row_one_left", + "page_index": 0, + "text_anchor": "DEV row one left" + }, + { + "id": "row_one_right", + "page_index": 0, + "text_anchor": "DEV row one right" + }, + { + "id": "row_two_left", + "page_index": 0, + "text_anchor": "DEV row two left" + }, + { + "id": "row_two_right", + "page_index": 0, + "text_anchor": "DEV row two right" + }, + { + "id": "conclusion", + "page_index": 0, + "text_anchor": "DEV conclusion" + } + ], + "relations": [], + "required_precedence": [ + [ + "introduction", + "header_left" + ], + [ + "header_left", + "header_right" + ], + [ + "header_right", + "row_one_left" + ], + [ + "row_one_left", + "row_one_right" + ], + [ + "row_one_right", + "row_two_left" + ], + [ + "row_two_left", + "row_two_right" + ], + [ + "row_two_right", + "conclusion" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-ruled-table-prose-holdout.json b/benchmark/eval/reading_order_next/graph_truth/order-next-ruled-table-prose-holdout.json new file mode 100644 index 0000000..e89e289 --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-ruled-table-prose-holdout.json @@ -0,0 +1,79 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "introduction", + "page_index": 0, + "text_anchor": "SEALED introduction" + }, + { + "id": "header_left", + "page_index": 0, + "text_anchor": "SEALED header left" + }, + { + "id": "header_right", + "page_index": 0, + "text_anchor": "SEALED header right" + }, + { + "id": "row_one_left", + "page_index": 0, + "text_anchor": "SEALED row one left" + }, + { + "id": "row_one_right", + "page_index": 0, + "text_anchor": "SEALED row one right" + }, + { + "id": "row_two_left", + "page_index": 0, + "text_anchor": "SEALED row two left" + }, + { + "id": "row_two_right", + "page_index": 0, + "text_anchor": "SEALED row two right" + }, + { + "id": "conclusion", + "page_index": 0, + "text_anchor": "SEALED conclusion" + } + ], + "relations": [], + "required_precedence": [ + [ + "introduction", + "header_left" + ], + [ + "header_left", + "header_right" + ], + [ + "header_right", + "row_one_left" + ], + [ + "row_one_left", + "row_one_right" + ], + [ + "row_one_right", + "row_two_left" + ], + [ + "row_two_left", + "row_two_right" + ], + [ + "row_two_right", + "conclusion" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-section-column-reset-dev.json b/benchmark/eval/reading_order_next/graph_truth/order-next-section-column-reset-dev.json new file mode 100644 index 0000000..ddbe298 --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-section-column-reset-dev.json @@ -0,0 +1,97 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "section_one", + "page_index": 0, + "text_anchor": "DEV section one" + }, + { + "id": "first_left_open", + "page_index": 0, + "text_anchor": "DEV first left open" + }, + { + "id": "first_left_close", + "page_index": 0, + "text_anchor": "DEV first left close" + }, + { + "id": "first_right_open", + "page_index": 0, + "text_anchor": "DEV first right open" + }, + { + "id": "first_right_close", + "page_index": 0, + "text_anchor": "DEV first right close" + }, + { + "id": "section_two", + "page_index": 0, + "text_anchor": "DEV section two" + }, + { + "id": "second_left_open", + "page_index": 0, + "text_anchor": "DEV second left open" + }, + { + "id": "second_left_close", + "page_index": 0, + "text_anchor": "DEV second left close" + }, + { + "id": "second_right_open", + "page_index": 0, + "text_anchor": "DEV second right open" + }, + { + "id": "second_right_close", + "page_index": 0, + "text_anchor": "DEV second right close" + } + ], + "relations": [], + "required_precedence": [ + [ + "section_one", + "first_left_open" + ], + [ + "first_left_open", + "first_left_close" + ], + [ + "first_left_close", + "first_right_open" + ], + [ + "first_right_open", + "first_right_close" + ], + [ + "first_right_close", + "section_two" + ], + [ + "section_two", + "second_left_open" + ], + [ + "second_left_open", + "second_left_close" + ], + [ + "second_left_close", + "second_right_open" + ], + [ + "second_right_open", + "second_right_close" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/graph_truth/order-next-section-column-reset-holdout.json b/benchmark/eval/reading_order_next/graph_truth/order-next-section-column-reset-holdout.json new file mode 100644 index 0000000..8b8f6c9 --- /dev/null +++ b/benchmark/eval/reading_order_next/graph_truth/order-next-section-column-reset-holdout.json @@ -0,0 +1,97 @@ +{ + "ambiguous_pairs": [], + "forbidden_precedence": [], + "nodes": [ + { + "id": "section_one", + "page_index": 0, + "text_anchor": "SEALED section one" + }, + { + "id": "first_left_open", + "page_index": 0, + "text_anchor": "SEALED first left open" + }, + { + "id": "first_left_close", + "page_index": 0, + "text_anchor": "SEALED first left close" + }, + { + "id": "first_right_open", + "page_index": 0, + "text_anchor": "SEALED first right open" + }, + { + "id": "first_right_close", + "page_index": 0, + "text_anchor": "SEALED first right close" + }, + { + "id": "section_two", + "page_index": 0, + "text_anchor": "SEALED section two" + }, + { + "id": "second_left_open", + "page_index": 0, + "text_anchor": "SEALED second left open" + }, + { + "id": "second_left_close", + "page_index": 0, + "text_anchor": "SEALED second left close" + }, + { + "id": "second_right_open", + "page_index": 0, + "text_anchor": "SEALED second right open" + }, + { + "id": "second_right_close", + "page_index": 0, + "text_anchor": "SEALED second right close" + } + ], + "relations": [], + "required_precedence": [ + [ + "section_one", + "first_left_open" + ], + [ + "first_left_open", + "first_left_close" + ], + [ + "first_left_close", + "first_right_open" + ], + [ + "first_right_open", + "first_right_close" + ], + [ + "first_right_close", + "section_two" + ], + [ + "section_two", + "second_left_open" + ], + [ + "second_left_open", + "second_left_close" + ], + [ + "second_left_close", + "second_right_open" + ], + [ + "second_right_open", + "second_right_close" + ] + ], + "valid_orders": [], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/manifest.tsv b/benchmark/eval/reading_order_next/manifest.tsv new file mode 100644 index 0000000..1d36be0 --- /dev/null +++ b/benchmark/eval/reading_order_next/manifest.tsv @@ -0,0 +1,15 @@ +# category doc_id pdf_path truth_text_path truth_table_json_path_optional truth_reading_order_path_optional truth_formula_path_optional truth_formula_json_path_optional truth_form_json_path_optional truth_reading_graph_path_optional +visual_truth order-next-ruled-table-prose-dev benchmark/eval/reading_order_next/corpus/order-next-ruled-table-prose-dev.pdf benchmark/eval/reading_order_next/page_text/order-next-ruled-table-prose-dev.txt benchmark/eval/reading_order_next/graph_truth/order-next-ruled-table-prose-dev.json +visual_truth order-next-section-column-reset-dev benchmark/eval/reading_order_next/corpus/order-next-section-column-reset-dev.pdf benchmark/eval/reading_order_next/page_text/order-next-section-column-reset-dev.txt benchmark/eval/reading_order_next/graph_truth/order-next-section-column-reset-dev.json +visual_truth order-next-reverse-stream-dev benchmark/eval/reading_order_next/corpus/order-next-reverse-stream-dev.pdf benchmark/eval/reading_order_next/page_text/order-next-reverse-stream-dev.txt benchmark/eval/reading_order_next/graph_truth/order-next-reverse-stream-dev.json +visual_truth order-next-accepted-separator-dev benchmark/eval/reading_order_next/corpus/order-next-accepted-separator-dev.pdf benchmark/eval/reading_order_next/page_text/order-next-accepted-separator-dev.txt benchmark/eval/reading_order_next/graph_truth/order-next-accepted-separator-dev.json +visual_truth order-next-rejected-separators-graphic-dev benchmark/eval/reading_order_next/corpus/order-next-rejected-separators-graphic-dev.pdf benchmark/eval/reading_order_next/page_text/order-next-rejected-separators-graphic-dev.txt benchmark/eval/reading_order_next/graph_truth/order-next-rejected-separators-graphic-dev.json +visual_truth order-next-quarter-turn-crop-userunit-dev benchmark/eval/reading_order_next/corpus/order-next-quarter-turn-crop-userunit-dev.pdf benchmark/eval/reading_order_next/page_text/order-next-quarter-turn-crop-userunit-dev.txt benchmark/eval/reading_order_next/graph_truth/order-next-quarter-turn-crop-userunit-dev.json +visual_truth order-next-multipage-continuation-dev benchmark/eval/reading_order_next/corpus/order-next-multipage-continuation-dev.pdf benchmark/eval/reading_order_next/page_text/order-next-multipage-continuation-dev.txt benchmark/eval/reading_order_next/graph_truth/order-next-multipage-continuation-dev.json +visual_truth order-next-ruled-table-prose-holdout benchmark/eval/reading_order_next/corpus/order-next-ruled-table-prose-holdout.pdf benchmark/eval/reading_order_next/page_text/order-next-ruled-table-prose-holdout.txt benchmark/eval/reading_order_next/graph_truth/order-next-ruled-table-prose-holdout.json +visual_truth order-next-section-column-reset-holdout benchmark/eval/reading_order_next/corpus/order-next-section-column-reset-holdout.pdf benchmark/eval/reading_order_next/page_text/order-next-section-column-reset-holdout.txt benchmark/eval/reading_order_next/graph_truth/order-next-section-column-reset-holdout.json +visual_truth order-next-reverse-stream-holdout benchmark/eval/reading_order_next/corpus/order-next-reverse-stream-holdout.pdf benchmark/eval/reading_order_next/page_text/order-next-reverse-stream-holdout.txt benchmark/eval/reading_order_next/graph_truth/order-next-reverse-stream-holdout.json +visual_truth order-next-accepted-separator-holdout benchmark/eval/reading_order_next/corpus/order-next-accepted-separator-holdout.pdf benchmark/eval/reading_order_next/page_text/order-next-accepted-separator-holdout.txt benchmark/eval/reading_order_next/graph_truth/order-next-accepted-separator-holdout.json +visual_truth order-next-rejected-separators-graphic-holdout benchmark/eval/reading_order_next/corpus/order-next-rejected-separators-graphic-holdout.pdf benchmark/eval/reading_order_next/page_text/order-next-rejected-separators-graphic-holdout.txt benchmark/eval/reading_order_next/graph_truth/order-next-rejected-separators-graphic-holdout.json +visual_truth order-next-quarter-turn-crop-userunit-holdout benchmark/eval/reading_order_next/corpus/order-next-quarter-turn-crop-userunit-holdout.pdf benchmark/eval/reading_order_next/page_text/order-next-quarter-turn-crop-userunit-holdout.txt benchmark/eval/reading_order_next/graph_truth/order-next-quarter-turn-crop-userunit-holdout.json +visual_truth order-next-multipage-continuation-holdout benchmark/eval/reading_order_next/corpus/order-next-multipage-continuation-holdout.pdf benchmark/eval/reading_order_next/page_text/order-next-multipage-continuation-holdout.txt benchmark/eval/reading_order_next/graph_truth/order-next-multipage-continuation-holdout.json diff --git a/benchmark/eval/reading_order_next/metadata.jsonl b/benchmark/eval/reading_order_next/metadata.jsonl new file mode 100644 index 0000000..a398970 --- /dev/null +++ b/benchmark/eval/reading_order_next/metadata.jsonl @@ -0,0 +1,14 @@ +{"doc_id": "order-next-ruled-table-prose-dev", "expected_ocr_pages": 0, "family": "ruled-table-prose", "reading_order_split": "dev"} +{"doc_id": "order-next-section-column-reset-dev", "expected_ocr_pages": 0, "family": "section-column-reset", "reading_order_split": "dev"} +{"doc_id": "order-next-reverse-stream-dev", "expected_ocr_pages": 0, "family": "reverse-stream", "reading_order_split": "dev"} +{"doc_id": "order-next-accepted-separator-dev", "expected_ocr_pages": 0, "family": "accepted-separator", "reading_order_split": "dev"} +{"doc_id": "order-next-rejected-separators-graphic-dev", "expected_ocr_pages": 0, "family": "rejected-separators-graphic", "reading_order_split": "dev"} +{"doc_id": "order-next-quarter-turn-crop-userunit-dev", "expected_ocr_pages": 0, "family": "quarter-turn-crop-userunit", "reading_order_split": "dev"} +{"doc_id": "order-next-multipage-continuation-dev", "expected_ocr_pages": 0, "family": "multipage-continuation", "reading_order_split": "dev"} +{"doc_id": "order-next-ruled-table-prose-holdout", "expected_ocr_pages": 0, "family": "ruled-table-prose", "reading_order_split": "holdout"} +{"doc_id": "order-next-section-column-reset-holdout", "expected_ocr_pages": 0, "family": "section-column-reset", "reading_order_split": "holdout"} +{"doc_id": "order-next-reverse-stream-holdout", "expected_ocr_pages": 0, "family": "reverse-stream", "reading_order_split": "holdout"} +{"doc_id": "order-next-accepted-separator-holdout", "expected_ocr_pages": 0, "family": "accepted-separator", "reading_order_split": "holdout"} +{"doc_id": "order-next-rejected-separators-graphic-holdout", "expected_ocr_pages": 0, "family": "rejected-separators-graphic", "reading_order_split": "holdout"} +{"doc_id": "order-next-quarter-turn-crop-userunit-holdout", "expected_ocr_pages": 0, "family": "quarter-turn-crop-userunit", "reading_order_split": "holdout"} +{"doc_id": "order-next-multipage-continuation-holdout", "expected_ocr_pages": 0, "family": "multipage-continuation", "reading_order_split": "holdout"} diff --git a/benchmark/eval/reading_order_next/page_text/order-next-accepted-separator-dev.txt b/benchmark/eval/reading_order_next/page_text/order-next-accepted-separator-dev.txt new file mode 100644 index 0000000..5e3c7ac --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-accepted-separator-dev.txt @@ -0,0 +1,4 @@ +DEV left open +DEV left close +DEV right open +DEV right close diff --git a/benchmark/eval/reading_order_next/page_text/order-next-accepted-separator-holdout.txt b/benchmark/eval/reading_order_next/page_text/order-next-accepted-separator-holdout.txt new file mode 100644 index 0000000..dbe610e --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-accepted-separator-holdout.txt @@ -0,0 +1,4 @@ +SEALED left open +SEALED left close +SEALED right open +SEALED right close diff --git a/benchmark/eval/reading_order_next/page_text/order-next-multipage-continuation-dev.txt b/benchmark/eval/reading_order_next/page_text/order-next-multipage-continuation-dev.txt new file mode 100644 index 0000000..aa4b524 --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-multipage-continuation-dev.txt @@ -0,0 +1,4 @@ +DEV page one open +DEV page one continue +DEV page two resume +DEV page two close diff --git a/benchmark/eval/reading_order_next/page_text/order-next-multipage-continuation-holdout.txt b/benchmark/eval/reading_order_next/page_text/order-next-multipage-continuation-holdout.txt new file mode 100644 index 0000000..7988550 --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-multipage-continuation-holdout.txt @@ -0,0 +1,4 @@ +SEALED page one open +SEALED page one continue +SEALED page two resume +SEALED page two close diff --git a/benchmark/eval/reading_order_next/page_text/order-next-quarter-turn-crop-userunit-dev.txt b/benchmark/eval/reading_order_next/page_text/order-next-quarter-turn-crop-userunit-dev.txt new file mode 100644 index 0000000..357fb66 --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-quarter-turn-crop-userunit-dev.txt @@ -0,0 +1,3 @@ +DEV opening +DEV middle +DEV closing diff --git a/benchmark/eval/reading_order_next/page_text/order-next-quarter-turn-crop-userunit-holdout.txt b/benchmark/eval/reading_order_next/page_text/order-next-quarter-turn-crop-userunit-holdout.txt new file mode 100644 index 0000000..9e6097f --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-quarter-turn-crop-userunit-holdout.txt @@ -0,0 +1,3 @@ +SEALED opening +SEALED middle +SEALED closing diff --git a/benchmark/eval/reading_order_next/page_text/order-next-rejected-separators-graphic-dev.txt b/benchmark/eval/reading_order_next/page_text/order-next-rejected-separators-graphic-dev.txt new file mode 100644 index 0000000..357fb66 --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-rejected-separators-graphic-dev.txt @@ -0,0 +1,3 @@ +DEV opening +DEV middle +DEV closing diff --git a/benchmark/eval/reading_order_next/page_text/order-next-rejected-separators-graphic-holdout.txt b/benchmark/eval/reading_order_next/page_text/order-next-rejected-separators-graphic-holdout.txt new file mode 100644 index 0000000..9e6097f --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-rejected-separators-graphic-holdout.txt @@ -0,0 +1,3 @@ +SEALED opening +SEALED middle +SEALED closing diff --git a/benchmark/eval/reading_order_next/page_text/order-next-reverse-stream-dev.txt b/benchmark/eval/reading_order_next/page_text/order-next-reverse-stream-dev.txt new file mode 100644 index 0000000..357fb66 --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-reverse-stream-dev.txt @@ -0,0 +1,3 @@ +DEV opening +DEV middle +DEV closing diff --git a/benchmark/eval/reading_order_next/page_text/order-next-reverse-stream-holdout.txt b/benchmark/eval/reading_order_next/page_text/order-next-reverse-stream-holdout.txt new file mode 100644 index 0000000..9e6097f --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-reverse-stream-holdout.txt @@ -0,0 +1,3 @@ +SEALED opening +SEALED middle +SEALED closing diff --git a/benchmark/eval/reading_order_next/page_text/order-next-ruled-table-prose-dev.txt b/benchmark/eval/reading_order_next/page_text/order-next-ruled-table-prose-dev.txt new file mode 100644 index 0000000..e409929 --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-ruled-table-prose-dev.txt @@ -0,0 +1,8 @@ +DEV introduction +DEV header left +DEV header right +DEV row one left +DEV row one right +DEV row two left +DEV row two right +DEV conclusion diff --git a/benchmark/eval/reading_order_next/page_text/order-next-ruled-table-prose-holdout.txt b/benchmark/eval/reading_order_next/page_text/order-next-ruled-table-prose-holdout.txt new file mode 100644 index 0000000..a605607 --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-ruled-table-prose-holdout.txt @@ -0,0 +1,8 @@ +SEALED introduction +SEALED header left +SEALED header right +SEALED row one left +SEALED row one right +SEALED row two left +SEALED row two right +SEALED conclusion diff --git a/benchmark/eval/reading_order_next/page_text/order-next-section-column-reset-dev.txt b/benchmark/eval/reading_order_next/page_text/order-next-section-column-reset-dev.txt new file mode 100644 index 0000000..f6eb107 --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-section-column-reset-dev.txt @@ -0,0 +1,10 @@ +DEV section one +DEV first left open +DEV first left close +DEV first right open +DEV first right close +DEV section two +DEV second left open +DEV second left close +DEV second right open +DEV second right close diff --git a/benchmark/eval/reading_order_next/page_text/order-next-section-column-reset-holdout.txt b/benchmark/eval/reading_order_next/page_text/order-next-section-column-reset-holdout.txt new file mode 100644 index 0000000..ff1b086 --- /dev/null +++ b/benchmark/eval/reading_order_next/page_text/order-next-section-column-reset-holdout.txt @@ -0,0 +1,10 @@ +SEALED section one +SEALED first left open +SEALED first left close +SEALED first right open +SEALED first right close +SEALED section two +SEALED second left open +SEALED second left close +SEALED second right open +SEALED second right close diff --git a/benchmark/eval/reading_order_next/test_verify.py b/benchmark/eval/reading_order_next/test_verify.py new file mode 100644 index 0000000..55c8fcc --- /dev/null +++ b/benchmark/eval/reading_order_next/test_verify.py @@ -0,0 +1,111 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location("order_next_verify", Path(__file__).with_name("verify.py")) +VERIFY = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(VERIFY) + + +def truth(): + return dict(nodes=[dict(id="a", text_anchor="Anchor Alpha", page_index=0), + dict(id="b", text_anchor="Anchor Beta", page_index=0), + dict(id="c", text_anchor="Anchor Gamma", page_index=1)], + page_local_precedence=[["a", "b"]], document_precedence=[["b", "c"]]) + + +def records(): + result = [dict(record_type="document_manifest", page_count=2)] + for page, anchors in ((0, ["Anchor Alpha", "Anchor Beta"]), (1, ["Anchor Gamma"])): + result.append(dict(record_type="page_started", page_index=page, status="running")) + for kind in ("span", "block"): + result.extend(dict(record_type=kind, page_index=page, text=anchor) for anchor in anchors) + result.append(dict(record_type="page_finished", page_index=page, status="completed")) + result.append(dict(record_type="document_finished", status="completed")) + return result + + +class VerificationTest(unittest.TestCase): + def test_valid_stream_and_artifact(self): + self.assertEqual(VERIFY.score_records(records(), truth(), "stream-jsonl")["verdict"], "verified") + artifact = [r for r in records() if r["record_type"] in ("span", "block", "document_manifest")] + self.assertEqual(VERIFY.score_records(artifact, truth(), "artifact-jsonl")["verdict"], "verified") + + def test_inversion_inside_one_block(self): + source = [dict(record_type="document_manifest", page_count=2), + dict(record_type="span", page_index=0, text="Anchor Alpha Anchor Beta"), + dict(record_type="span", page_index=1, text="Anchor Gamma"), + dict(record_type="block", page_index=0, text="Anchor Beta Anchor Alpha"), + dict(record_type="block", page_index=1, text="Anchor Gamma")] + result = VERIFY.score_records(source, truth(), "artifact-jsonl") + self.assertEqual(result["emitted_page_local"]["passed"], 0) + self.assertEqual(result["emitted_document"]["passed"], 1) + self.assertEqual(result["verdict"], "failed") + + def test_missing_duplicate_and_wrong_page(self): + for replacement in ("", "Anchor Alpha Anchor Alpha", "Anchor Beta"): + source = records() + next(r for r in source if r["record_type"] == "span")["text"] = replacement + self.assertEqual(VERIFY.score_records(source, truth(), "stream-jsonl")["verdict"], "failed") + source = records() + next(r for r in source if r["record_type"] == "span")["page_index"] = 1 + self.assertEqual(VERIFY.score_records(source, truth(), "stream-jsonl")["verdict"], "failed") + + def test_cross_page_order_separate_from_local(self): + source = [dict(record_type="document_manifest", page_count=2)] + for kind in ("span", "block"): + source.extend([dict(record_type=kind, page_index=1, text="Anchor Gamma"), + dict(record_type=kind, page_index=0, text="Anchor Alpha"), + dict(record_type=kind, page_index=0, text="Anchor Beta")]) + result = VERIFY.score_records(source, truth(), "artifact-jsonl") + self.assertEqual(result["emitted_page_local"]["passed"], 1) + self.assertEqual(result["emitted_document"]["passed"], 0) + self.assertIn("not inferred", result["graph_diagnostics"]) + + def test_lifecycle_and_cli_errors(self): + variants = [records()[:-1], records()[1:], records() + [dict(record_type="block", text="late", page_index=1)]] + incomplete = records() + incomplete[1]["status"] = "completed" + variants.append(incomplete) + error = records() + error[-1]["errors"] = ["failed extraction"] + variants.append(error) + for source in variants: + self.assertEqual(VERIFY.score_records(source, truth(), "stream-jsonl")["verdict"], "failed") + self.assertEqual(VERIFY.score_records(records(), truth(), "stream-jsonl", 7)["verdict"], "failed") + + def test_malformed_jsonl(self): + for source in (b"not json\n", b"[]\n", b'{"text":"missing record type"}\n', + b'{"record_type":"span","text":null,"page_index":0}\n'): + with self.assertRaises(ValueError): + VERIFY.parse_records(source) + + def test_diagnostic_failures_are_separate_and_visible(self): + row = dict(doc_id="example", metrics=dict(reading_graph_required_recall=0.0, + reading_graph_valid_projection=1.0, reading_graph_cycle_rate=0.0)) + result = VERIFY.score_diagnostic([row], "example") + self.assertEqual(result["verdict"], "failed") + self.assertEqual(len(result["failures"]), 1) + row["metrics"]["reading_graph_required_recall"] = 1.0 + self.assertEqual(VERIFY.score_diagnostic([row], "example")["verdict"], "verified") + self.assertEqual(VERIFY.score_diagnostic([row], "other")["verdict"], "failed") + self.assertEqual(VERIFY.score_diagnostic([], "example")["verdict"], "failed") + + def test_failed_cli_evidence_retained(self): + with tempfile.TemporaryDirectory() as directory: + stem = Path(directory) / "failure" + code, output = VERIFY.execute(["/usr/bin/false"], stem, 2) + self.assertNotEqual(code, 0) + self.assertEqual(output, b"") + self.assertTrue(stem.with_suffix(".stdout.jsonl").is_file()) + self.assertTrue(stem.with_suffix(".stderr.txt").is_file()) + + def test_whitespace_and_anchor_boundaries(self): + self.assertEqual(len(VERIFY.occurrences("Anchor Alpha\nAnchor Alpha", "Anchor Alpha")), 2) + self.assertEqual(VERIFY.occurrences("Anchor Alphabet", "Anchor Alpha"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmark/eval/reading_order_next/truth/order-next-accepted-separator-dev.json b/benchmark/eval/reading_order_next/truth/order-next-accepted-separator-dev.json new file mode 100644 index 0000000..2b93d3b --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-accepted-separator-dev.json @@ -0,0 +1,54 @@ +{ + "document_precedence": [], + "nodes": [ + { + "id": "left_open", + "page_index": 0, + "text_anchor": "DEV left open" + }, + { + "id": "left_close", + "page_index": 0, + "text_anchor": "DEV left close" + }, + { + "id": "right_open", + "page_index": 0, + "text_anchor": "DEV right open" + }, + { + "id": "right_close", + "page_index": 0, + "text_anchor": "DEV right close" + } + ], + "page_local_precedence": [ + [ + "left_open", + "left_close" + ], + [ + "left_close", + "right_open" + ], + [ + "right_open", + "right_close" + ] + ], + "required_precedence": [ + [ + "left_open", + "left_close" + ], + [ + "left_close", + "right_open" + ], + [ + "right_open", + "right_close" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-accepted-separator-holdout.json b/benchmark/eval/reading_order_next/truth/order-next-accepted-separator-holdout.json new file mode 100644 index 0000000..c59d021 --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-accepted-separator-holdout.json @@ -0,0 +1,54 @@ +{ + "document_precedence": [], + "nodes": [ + { + "id": "left_open", + "page_index": 0, + "text_anchor": "SEALED left open" + }, + { + "id": "left_close", + "page_index": 0, + "text_anchor": "SEALED left close" + }, + { + "id": "right_open", + "page_index": 0, + "text_anchor": "SEALED right open" + }, + { + "id": "right_close", + "page_index": 0, + "text_anchor": "SEALED right close" + } + ], + "page_local_precedence": [ + [ + "left_open", + "left_close" + ], + [ + "left_close", + "right_open" + ], + [ + "right_open", + "right_close" + ] + ], + "required_precedence": [ + [ + "left_open", + "left_close" + ], + [ + "left_close", + "right_open" + ], + [ + "right_open", + "right_close" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-multipage-continuation-dev.json b/benchmark/eval/reading_order_next/truth/order-next-multipage-continuation-dev.json new file mode 100644 index 0000000..3118dc9 --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-multipage-continuation-dev.json @@ -0,0 +1,55 @@ +{ + "document_precedence": [ + [ + "page_one_continue", + "page_two_resume" + ] + ], + "nodes": [ + { + "id": "page_one_open", + "page_index": 0, + "text_anchor": "DEV page one open" + }, + { + "id": "page_one_continue", + "page_index": 0, + "text_anchor": "DEV page one continue" + }, + { + "id": "page_two_resume", + "page_index": 1, + "text_anchor": "DEV page two resume" + }, + { + "id": "page_two_close", + "page_index": 1, + "text_anchor": "DEV page two close" + } + ], + "page_local_precedence": [ + [ + "page_one_open", + "page_one_continue" + ], + [ + "page_two_resume", + "page_two_close" + ] + ], + "required_precedence": [ + [ + "page_one_open", + "page_one_continue" + ], + [ + "page_one_continue", + "page_two_resume" + ], + [ + "page_two_resume", + "page_two_close" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-multipage-continuation-holdout.json b/benchmark/eval/reading_order_next/truth/order-next-multipage-continuation-holdout.json new file mode 100644 index 0000000..f6a3c40 --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-multipage-continuation-holdout.json @@ -0,0 +1,55 @@ +{ + "document_precedence": [ + [ + "page_one_continue", + "page_two_resume" + ] + ], + "nodes": [ + { + "id": "page_one_open", + "page_index": 0, + "text_anchor": "SEALED page one open" + }, + { + "id": "page_one_continue", + "page_index": 0, + "text_anchor": "SEALED page one continue" + }, + { + "id": "page_two_resume", + "page_index": 1, + "text_anchor": "SEALED page two resume" + }, + { + "id": "page_two_close", + "page_index": 1, + "text_anchor": "SEALED page two close" + } + ], + "page_local_precedence": [ + [ + "page_one_open", + "page_one_continue" + ], + [ + "page_two_resume", + "page_two_close" + ] + ], + "required_precedence": [ + [ + "page_one_open", + "page_one_continue" + ], + [ + "page_one_continue", + "page_two_resume" + ], + [ + "page_two_resume", + "page_two_close" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-quarter-turn-crop-userunit-dev.json b/benchmark/eval/reading_order_next/truth/order-next-quarter-turn-crop-userunit-dev.json new file mode 100644 index 0000000..3bd7666 --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-quarter-turn-crop-userunit-dev.json @@ -0,0 +1,41 @@ +{ + "document_precedence": [], + "nodes": [ + { + "id": "opening", + "page_index": 0, + "text_anchor": "DEV opening" + }, + { + "id": "middle", + "page_index": 0, + "text_anchor": "DEV middle" + }, + { + "id": "closing", + "page_index": 0, + "text_anchor": "DEV closing" + } + ], + "page_local_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "required_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-quarter-turn-crop-userunit-holdout.json b/benchmark/eval/reading_order_next/truth/order-next-quarter-turn-crop-userunit-holdout.json new file mode 100644 index 0000000..a7f0a6b --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-quarter-turn-crop-userunit-holdout.json @@ -0,0 +1,41 @@ +{ + "document_precedence": [], + "nodes": [ + { + "id": "opening", + "page_index": 0, + "text_anchor": "SEALED opening" + }, + { + "id": "middle", + "page_index": 0, + "text_anchor": "SEALED middle" + }, + { + "id": "closing", + "page_index": 0, + "text_anchor": "SEALED closing" + } + ], + "page_local_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "required_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-rejected-separators-graphic-dev.json b/benchmark/eval/reading_order_next/truth/order-next-rejected-separators-graphic-dev.json new file mode 100644 index 0000000..3bd7666 --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-rejected-separators-graphic-dev.json @@ -0,0 +1,41 @@ +{ + "document_precedence": [], + "nodes": [ + { + "id": "opening", + "page_index": 0, + "text_anchor": "DEV opening" + }, + { + "id": "middle", + "page_index": 0, + "text_anchor": "DEV middle" + }, + { + "id": "closing", + "page_index": 0, + "text_anchor": "DEV closing" + } + ], + "page_local_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "required_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-rejected-separators-graphic-holdout.json b/benchmark/eval/reading_order_next/truth/order-next-rejected-separators-graphic-holdout.json new file mode 100644 index 0000000..a7f0a6b --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-rejected-separators-graphic-holdout.json @@ -0,0 +1,41 @@ +{ + "document_precedence": [], + "nodes": [ + { + "id": "opening", + "page_index": 0, + "text_anchor": "SEALED opening" + }, + { + "id": "middle", + "page_index": 0, + "text_anchor": "SEALED middle" + }, + { + "id": "closing", + "page_index": 0, + "text_anchor": "SEALED closing" + } + ], + "page_local_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "required_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-reverse-stream-dev.json b/benchmark/eval/reading_order_next/truth/order-next-reverse-stream-dev.json new file mode 100644 index 0000000..3bd7666 --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-reverse-stream-dev.json @@ -0,0 +1,41 @@ +{ + "document_precedence": [], + "nodes": [ + { + "id": "opening", + "page_index": 0, + "text_anchor": "DEV opening" + }, + { + "id": "middle", + "page_index": 0, + "text_anchor": "DEV middle" + }, + { + "id": "closing", + "page_index": 0, + "text_anchor": "DEV closing" + } + ], + "page_local_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "required_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-reverse-stream-holdout.json b/benchmark/eval/reading_order_next/truth/order-next-reverse-stream-holdout.json new file mode 100644 index 0000000..a7f0a6b --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-reverse-stream-holdout.json @@ -0,0 +1,41 @@ +{ + "document_precedence": [], + "nodes": [ + { + "id": "opening", + "page_index": 0, + "text_anchor": "SEALED opening" + }, + { + "id": "middle", + "page_index": 0, + "text_anchor": "SEALED middle" + }, + { + "id": "closing", + "page_index": 0, + "text_anchor": "SEALED closing" + } + ], + "page_local_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "required_precedence": [ + [ + "opening", + "middle" + ], + [ + "middle", + "closing" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-ruled-table-prose-dev.json b/benchmark/eval/reading_order_next/truth/order-next-ruled-table-prose-dev.json new file mode 100644 index 0000000..57ba98c --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-ruled-table-prose-dev.json @@ -0,0 +1,106 @@ +{ + "document_precedence": [], + "nodes": [ + { + "id": "introduction", + "page_index": 0, + "text_anchor": "DEV introduction" + }, + { + "id": "header_left", + "page_index": 0, + "text_anchor": "DEV header left" + }, + { + "id": "header_right", + "page_index": 0, + "text_anchor": "DEV header right" + }, + { + "id": "row_one_left", + "page_index": 0, + "text_anchor": "DEV row one left" + }, + { + "id": "row_one_right", + "page_index": 0, + "text_anchor": "DEV row one right" + }, + { + "id": "row_two_left", + "page_index": 0, + "text_anchor": "DEV row two left" + }, + { + "id": "row_two_right", + "page_index": 0, + "text_anchor": "DEV row two right" + }, + { + "id": "conclusion", + "page_index": 0, + "text_anchor": "DEV conclusion" + } + ], + "page_local_precedence": [ + [ + "introduction", + "header_left" + ], + [ + "header_left", + "header_right" + ], + [ + "header_right", + "row_one_left" + ], + [ + "row_one_left", + "row_one_right" + ], + [ + "row_one_right", + "row_two_left" + ], + [ + "row_two_left", + "row_two_right" + ], + [ + "row_two_right", + "conclusion" + ] + ], + "required_precedence": [ + [ + "introduction", + "header_left" + ], + [ + "header_left", + "header_right" + ], + [ + "header_right", + "row_one_left" + ], + [ + "row_one_left", + "row_one_right" + ], + [ + "row_one_right", + "row_two_left" + ], + [ + "row_two_left", + "row_two_right" + ], + [ + "row_two_right", + "conclusion" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-ruled-table-prose-holdout.json b/benchmark/eval/reading_order_next/truth/order-next-ruled-table-prose-holdout.json new file mode 100644 index 0000000..496008a --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-ruled-table-prose-holdout.json @@ -0,0 +1,106 @@ +{ + "document_precedence": [], + "nodes": [ + { + "id": "introduction", + "page_index": 0, + "text_anchor": "SEALED introduction" + }, + { + "id": "header_left", + "page_index": 0, + "text_anchor": "SEALED header left" + }, + { + "id": "header_right", + "page_index": 0, + "text_anchor": "SEALED header right" + }, + { + "id": "row_one_left", + "page_index": 0, + "text_anchor": "SEALED row one left" + }, + { + "id": "row_one_right", + "page_index": 0, + "text_anchor": "SEALED row one right" + }, + { + "id": "row_two_left", + "page_index": 0, + "text_anchor": "SEALED row two left" + }, + { + "id": "row_two_right", + "page_index": 0, + "text_anchor": "SEALED row two right" + }, + { + "id": "conclusion", + "page_index": 0, + "text_anchor": "SEALED conclusion" + } + ], + "page_local_precedence": [ + [ + "introduction", + "header_left" + ], + [ + "header_left", + "header_right" + ], + [ + "header_right", + "row_one_left" + ], + [ + "row_one_left", + "row_one_right" + ], + [ + "row_one_right", + "row_two_left" + ], + [ + "row_two_left", + "row_two_right" + ], + [ + "row_two_right", + "conclusion" + ] + ], + "required_precedence": [ + [ + "introduction", + "header_left" + ], + [ + "header_left", + "header_right" + ], + [ + "header_right", + "row_one_left" + ], + [ + "row_one_left", + "row_one_right" + ], + [ + "row_one_right", + "row_two_left" + ], + [ + "row_two_left", + "row_two_right" + ], + [ + "row_two_right", + "conclusion" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-section-column-reset-dev.json b/benchmark/eval/reading_order_next/truth/order-next-section-column-reset-dev.json new file mode 100644 index 0000000..15182af --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-section-column-reset-dev.json @@ -0,0 +1,132 @@ +{ + "document_precedence": [], + "nodes": [ + { + "id": "section_one", + "page_index": 0, + "text_anchor": "DEV section one" + }, + { + "id": "first_left_open", + "page_index": 0, + "text_anchor": "DEV first left open" + }, + { + "id": "first_left_close", + "page_index": 0, + "text_anchor": "DEV first left close" + }, + { + "id": "first_right_open", + "page_index": 0, + "text_anchor": "DEV first right open" + }, + { + "id": "first_right_close", + "page_index": 0, + "text_anchor": "DEV first right close" + }, + { + "id": "section_two", + "page_index": 0, + "text_anchor": "DEV section two" + }, + { + "id": "second_left_open", + "page_index": 0, + "text_anchor": "DEV second left open" + }, + { + "id": "second_left_close", + "page_index": 0, + "text_anchor": "DEV second left close" + }, + { + "id": "second_right_open", + "page_index": 0, + "text_anchor": "DEV second right open" + }, + { + "id": "second_right_close", + "page_index": 0, + "text_anchor": "DEV second right close" + } + ], + "page_local_precedence": [ + [ + "section_one", + "first_left_open" + ], + [ + "first_left_open", + "first_left_close" + ], + [ + "first_left_close", + "first_right_open" + ], + [ + "first_right_open", + "first_right_close" + ], + [ + "first_right_close", + "section_two" + ], + [ + "section_two", + "second_left_open" + ], + [ + "second_left_open", + "second_left_close" + ], + [ + "second_left_close", + "second_right_open" + ], + [ + "second_right_open", + "second_right_close" + ] + ], + "required_precedence": [ + [ + "section_one", + "first_left_open" + ], + [ + "first_left_open", + "first_left_close" + ], + [ + "first_left_close", + "first_right_open" + ], + [ + "first_right_open", + "first_right_close" + ], + [ + "first_right_close", + "section_two" + ], + [ + "section_two", + "second_left_open" + ], + [ + "second_left_open", + "second_left_close" + ], + [ + "second_left_close", + "second_right_open" + ], + [ + "second_right_open", + "second_right_close" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/truth/order-next-section-column-reset-holdout.json b/benchmark/eval/reading_order_next/truth/order-next-section-column-reset-holdout.json new file mode 100644 index 0000000..b5ffb9a --- /dev/null +++ b/benchmark/eval/reading_order_next/truth/order-next-section-column-reset-holdout.json @@ -0,0 +1,132 @@ +{ + "document_precedence": [], + "nodes": [ + { + "id": "section_one", + "page_index": 0, + "text_anchor": "SEALED section one" + }, + { + "id": "first_left_open", + "page_index": 0, + "text_anchor": "SEALED first left open" + }, + { + "id": "first_left_close", + "page_index": 0, + "text_anchor": "SEALED first left close" + }, + { + "id": "first_right_open", + "page_index": 0, + "text_anchor": "SEALED first right open" + }, + { + "id": "first_right_close", + "page_index": 0, + "text_anchor": "SEALED first right close" + }, + { + "id": "section_two", + "page_index": 0, + "text_anchor": "SEALED section two" + }, + { + "id": "second_left_open", + "page_index": 0, + "text_anchor": "SEALED second left open" + }, + { + "id": "second_left_close", + "page_index": 0, + "text_anchor": "SEALED second left close" + }, + { + "id": "second_right_open", + "page_index": 0, + "text_anchor": "SEALED second right open" + }, + { + "id": "second_right_close", + "page_index": 0, + "text_anchor": "SEALED second right close" + } + ], + "page_local_precedence": [ + [ + "section_one", + "first_left_open" + ], + [ + "first_left_open", + "first_left_close" + ], + [ + "first_left_close", + "first_right_open" + ], + [ + "first_right_open", + "first_right_close" + ], + [ + "first_right_close", + "section_two" + ], + [ + "section_two", + "second_left_open" + ], + [ + "second_left_open", + "second_left_close" + ], + [ + "second_left_close", + "second_right_open" + ], + [ + "second_right_open", + "second_right_close" + ] + ], + "required_precedence": [ + [ + "section_one", + "first_left_open" + ], + [ + "first_left_open", + "first_left_close" + ], + [ + "first_left_close", + "first_right_open" + ], + [ + "first_right_open", + "first_right_close" + ], + [ + "first_right_close", + "section_two" + ], + [ + "section_two", + "second_left_open" + ], + [ + "second_left_open", + "second_left_close" + ], + [ + "second_left_close", + "second_right_open" + ], + [ + "second_right_open", + "second_right_close" + ] + ], + "version": 1 +} diff --git a/benchmark/eval/reading_order_next/verify.py b/benchmark/eval/reading_order_next/verify.py new file mode 100644 index 0000000..5960250 --- /dev/null +++ b/benchmark/eval/reading_order_next/verify.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Score public block order independently from optional internal graph diagnostics.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parent + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def normalize(text): + return " ".join(text.split()) + + +def occurrences(text, anchor): + return list(re.finditer(r"(? list[tuple[int, int, int, int]] | None: + """Validate physical slots independently of flattened text accuracy. + + Empty materialized placeholders under a merged cell are permitted. An + uncovered empty cell remains semantic, and a nonempty collision is invalid. + Truth can use compact rows with implicit columns; emitted grids use explicit + columns. Refuse excessive dimensions before expanding occupancy. + """ + rows = table.get("rows", []) + if not isinstance(rows, list) or not rows or len(rows) > 1000: + return None + columns = table.get("column_count") + if columns is not None and (type(columns) is not int or not 0 < columns <= 1000): + return None + covered = set() + topology = [] + for row_index, row in enumerate(rows): + if isinstance(row, dict) and (type(row.get("row_index", row_index)) is not int or row.get("row_index", row_index) != row_index): + return None + cells = row.get("cells", []) if isinstance(row, dict) else row + if not isinstance(cells, list) or len(cells) > 1000: + return None + next_column = 0 + for cell in cells: + if not isinstance(cell, dict): + return None + if type(cell.get("row", row_index)) is not int or cell.get("row", row_index) != row_index: + return None + explicit = "column" in cell + if not explicit: + while (row_index, next_column) in covered: + next_column += 1 + column = cell.get("column", next_column) + rowspan, colspan = cell.get("rowspan", 1), cell.get("colspan", 1) + if any(type(v) is not int for v in (column, rowspan, colspan)) or column < 0 or rowspan < 1 or colspan < 1: + return None + if (row_index, column) in covered and normalize_cell(cell.get("text", "")) == "" and explicit: + next_column = max(next_column, column + 1) + continue + if row_index + rowspan > len(rows) or column + colspan > (columns or 1000): + return None + if rowspan * colspan > 100_000 or len(covered) + rowspan * colspan > 100_000: + return None + slots = {(r, c) for r in range(row_index, row_index + rowspan) + for c in range(column, column + colspan)} + if slots & covered: + return None + covered.update(slots) + topology.append((row_index, column, rowspan, colspan)) + next_column = column + colspan + return sorted(topology) + + +def topology_accuracy(predicted: list[dict[str, Any]], truth: list[dict[str, Any]]) -> float | None: + if not truth: + return float(not predicted) + if not any(table.get("topology_required") is True for table in truth): + return None + if len(predicted) != len(truth): + return 0.0 + correct = 0 + for actual, expected in zip(predicted, truth): + expected_topology = table_topology(expected) + if expected_topology is None: + raise ValueError("invalid frozen table topology truth") + actual_topology = table_topology(actual) + if actual_topology is None: + continue + # Cell placement alone does not identify the whole grid: a declared + # extra empty column is still a topology error. Compact truth/baseline + # rows without dimensions retain their inferred extent. + expected_columns = expected.get("column_count", max((c + cs for _, c, _, cs in expected_topology), default=0)) + actual_columns = actual.get("column_count", max((c + cs for _, c, _, cs in actual_topology), default=0)) + correct += (actual_topology == expected_topology and actual_columns == expected_columns + and len(actual["rows"]) == len(expected["rows"])) + return correct / len(truth) + + def flatten_cells(tables: list[dict[str, Any]]) -> list[dict[str, Any]]: cells: list[dict[str, Any]] = [] for table in tables: @@ -616,6 +696,8 @@ def failed(entry: Entry, tool: str, reason: str) -> dict[str, Any]: def empty_metrics() -> dict[str, None]: return { + "table_count_accuracy": None, + "topology_accuracy": None, "cell_text_accuracy": None, "role_accuracy": None, "bbox_iou": None, diff --git a/benchmark/eval/table_topology/README.md b/benchmark/eval/table_topology/README.md new file mode 100644 index 0000000..ae4f75d --- /dev/null +++ b/benchmark/eval/table_topology/README.md @@ -0,0 +1,44 @@ +# Table topology controls + +Eight small deterministic PDFs split into four development and four held-out +cases. They cover ruled blank-versus-zero cells, parentheses negatives, +right-to-left content streams with independently positioned cell tokens, +text crossing shared row rules, and graphic-axis no-table controls. +`freeze.json` binds PDF bytes, cell truth, case selection and acceptance before +evaluation. These synthetic reductions do not qualify arbitrary real tables. + +The verifier drives both public JSONL formats with OCR disabled. It scores +physical row/column spans independently of flattened cell text, resolves cell +source IDs to exact emitted text on the same page, rejects duplicate owners +and empty-cell text claims, and checks streaming completion. Crossing lettering +has an exclusive-assignment gate without an invented semantic row label. + +```sh +python3 benchmark/eval/table_topology/generate.py --check +python3 -m unittest discover -s benchmark/eval/table_topology -p 'test_*.py' +python3 benchmark/eval/table_topology/verify.py \ + --cli zig-out/bin/pdf-parser --split dev \ + --output benchmark/eval/outputs/topology-order-2026-10-12/table-dev +``` + +After development and review, assess `--split holdout` into a new directory. +Evidence directories cannot be overwritten. Preserve failures and frozen truth; +do not retune the experiment using held-out results. Existing financial stress +floors remain required; topology comparison is opt-in through +`topology_required` in table truth. Comparator schema `0.4.0` adds separate +`table_count_accuracy` and `topology_accuracy` metrics; it does not change the +parser output schema. + +Use the project-local monitor for every compiler invocation in this work: + +```sh +python3 benchmark/eval/bounded_zig.py build -- --summary all +python3 benchmark/eval/bounded_zig.py test -- --summary all +``` + +It uses one task cache for local and global compiler objects, a 6 GiB observed +allocation budget, a 12 GiB free-space floor and two concurrent build jobs. +Allocation sampling can observe an overshoot; this is not a filesystem quota. +On a breach it terminates only its owned build process group and leaves logs, +receipts and caches intact. Evidence is stored outside compiler caches. Existing +toolchains and earlier verification evidence are retained. diff --git a/benchmark/eval/table_topology/cases.json b/benchmark/eval/table_topology/cases.json new file mode 100644 index 0000000..067697f --- /dev/null +++ b/benchmark/eval/table_topology/cases.json @@ -0,0 +1,58 @@ +[ + { + "id": "blank-zero-dev", + "split": "dev", + "pdf": "corpus/blank-zero-dev.pdf", + "truth": "truth/blank-zero-dev.json", + "family": "blank-zero" + }, + { + "id": "reverse-multispan-dev", + "split": "dev", + "pdf": "corpus/reverse-multispan-dev.pdf", + "truth": "truth/reverse-multispan-dev.json", + "family": "reverse-multispan" + }, + { + "id": "boundary-ownership-dev", + "split": "dev", + "pdf": "corpus/boundary-ownership-dev.pdf", + "truth": "truth/boundary-ownership-dev.json", + "family": "boundary-ownership" + }, + { + "id": "graphic-axis-dev", + "split": "dev", + "pdf": "corpus/graphic-axis-dev.pdf", + "truth": "truth/graphic-axis-dev.json", + "family": "negative" + }, + { + "id": "blank-zero-holdout", + "split": "holdout", + "pdf": "corpus/blank-zero-holdout.pdf", + "truth": "truth/blank-zero-holdout.json", + "family": "blank-zero" + }, + { + "id": "reverse-multispan-holdout", + "split": "holdout", + "pdf": "corpus/reverse-multispan-holdout.pdf", + "truth": "truth/reverse-multispan-holdout.json", + "family": "reverse-multispan" + }, + { + "id": "boundary-ownership-holdout", + "split": "holdout", + "pdf": "corpus/boundary-ownership-holdout.pdf", + "truth": "truth/boundary-ownership-holdout.json", + "family": "boundary-ownership" + }, + { + "id": "graphic-axis-holdout", + "split": "holdout", + "pdf": "corpus/graphic-axis-holdout.pdf", + "truth": "truth/graphic-axis-holdout.json", + "family": "negative" + } +] diff --git a/benchmark/eval/table_topology/corpus/blank-zero-dev.pdf b/benchmark/eval/table_topology/corpus/blank-zero-dev.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3f46df5844f7da466e0153a8dfd70f90d9dc95dc GIT binary patch literal 931 zcmah|!A`mLs41bYz)A6SdZH?(`~AqkNACm-wOd&>wh@;DDJSUJ0KS;LLQH z?}{a5$f-~0v^FGyiZC3Kc1xb!to*IcHs%SvC}cuu3awJ%b_2YLkVIVBH z9{p>OEsORdi?U_W#x14bM_@?X8f1&3vXXxQ$+BQ2Z=*9fis^z9REu~`CFR0Mgb~6) zggC{?7#GVW*ZUOlGp^#_9X#*IBmLKb6#}bpsmiDS!#8yIPbSBeHttw24r$YbHN9wN zjXP<3Ru5;@XJ@`!m}*uTlyF9>)b+`=-o9#+ELE%X`$De;6p|bSM-g%H@+dJ42BE{bLcF8)tUq7Qx(XZHl+*mY*6c{W(JG%Um$-;W%g)i*#P&h4F zH;!)Ic$j`{?S3|+4i@{RYu1%Pg|pB^t9()u1@aYCc=n5#^-vsLrV7TKFcOdDL5{6f0tRXbC;rp}R8 KVlcS8or!k_&EduX literal 0 HcmV?d00001 diff --git a/benchmark/eval/table_topology/corpus/boundary-ownership-dev.pdf b/benchmark/eval/table_topology/corpus/boundary-ownership-dev.pdf new file mode 100644 index 0000000000000000000000000000000000000000..88a28319fbc125521ec6ef26a7d739db1202406f GIT binary patch literal 979 zcma)5!A|2a5WV{=-lY;8mc(x2v=XXBDHYN#T10aL4sPp~pe8|y3oKv1Gf7NKSuNy{ zOy0biH=Y^yt|!wU@+1-xA{&}(p&W$sha*g2HtZfZNgd2=T!WK&=^8d$x zmKXfYmNjnsOuwfr<+gB;dx7lv9LO@2C2Fe^T-jx2#^oz+L*i?xkPOv5IAP(4S0iEt z1UI%Wx7C6ws~XJA>nt4k$srNLSQSZj6K; zKo18!OQXeAnF9TR1(VfD%sUN;hAGZaC|7CJs=_{Yu`8Ue$ z=yyjdiSLemcG}2pcHM6BWp4H&m~H2-*=+_Tg0ZP>>$$X>XX~c V)ZN9}naVYFCPOWHy^G&d@fX8q>h}Nu literal 0 HcmV?d00001 diff --git a/benchmark/eval/table_topology/corpus/boundary-ownership-holdout.pdf b/benchmark/eval/table_topology/corpus/boundary-ownership-holdout.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5ca2a39b3e02a01ce48c1ab224b8cf5890d40636 GIT binary patch literal 982 zcmah|O;6)65WV|Xyh|lGEV1+1NT|{lsE`)4h~@?y+}15YO|nW{zJC4ABr$}t+WItk ze)HaBW<0t%pZ!oLp-_;h7Ej{z6khiB$G|(&R&T2fyc@kS4SXWpVlok?TrrWqzb14% z?^nKRa2GKBfmEtpX`u#*eDgh#ZE8c*&d0kpt6Wd3SKLYB$I3^N`VU~i(h{#h$PVys zO;hdaB~{3!&*-)b6rz?enUMFup3|KDl_@v&5n+%pp|Q2mg^(u^UPKxpiX{qA5RWCc zm^)wv;`?*Rc6W$kfI*!?43G;(~N#lfqIdxsioW)`jP?S zXpF4D(ItK^m)!7UWKX!r{+;0m<6%Z|Frl>w;@;v+H;;$)RacSxFO!9%#MORo7KaOW z^!uYYl=sI@o|dv7U%#F_d^hH9pWD^&=DUS$*PB5FZ>k&9-CUaOv$6S750>S%Hfuqn z7V_gz>!KjS8lPB-_7>$QMg!T$l%ksW40wJ>>%NC2{y@*xI^UYQeU|h5-5{oouPS~~ V{c~)ru5DXqlBQxby7)a4e*q%C>)`+Z literal 0 HcmV?d00001 diff --git a/benchmark/eval/table_topology/corpus/graphic-axis-dev.pdf b/benchmark/eval/table_topology/corpus/graphic-axis-dev.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6a1bfed0252714c8029183b367366fc308cd219c GIT binary patch literal 708 zcmZ`%%TB{E5WM><=28g`G;taSDxn?`YeLcHL9|43<>)9F4 z*z4Z(a&jX3v5*L1pI*fIIfC@_)gZXgUgvg$V5&E!LPUfaMk8U$6%%>?Q_%8)n`~8K zF<|;5onq_KLw1DR`72}_XNcPR1b1eY>9PI5A|!qyBb*MSNAQ8ACtih^9T3b+Wox&j zgdBZB-L{ZW4Pi7Q?~XmocJ{ZX-1uk6WWa>VJEIE`_7&bm5+V*m6re&;kz;dw=AII|X%T!j0DM5%$o z8V78Mri=1`sk1+rl$42AhZn^(++SF%Iy~=mmK)c6!g=;(kkEXs<^Qw&e3jSEH#t?L KM6Y-CFcIJ6C$*{o literal 0 HcmV?d00001 diff --git a/benchmark/eval/table_topology/corpus/graphic-axis-holdout.pdf b/benchmark/eval/table_topology/corpus/graphic-axis-holdout.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7410f782a80e5d5972337a336cccce79e9722105 GIT binary patch literal 708 zcmZ`%%TB{E5WM><=28g`G;taSDxn?=-wc8a?R9R~2&Q^%Dnvwh#AqZ;xnv^me+yb(aFZ=7 z%m++=qEl>LddQBDJBLEHafYa^PjF|JnI78@%tPWQGQ#OFdIBFRq)(aId$ffg<1mlZi0pnnC{k z{zA(IpYnBs$06g-l%?927V_jGpZs}|ZE8bk>l1u2>s(K(H#~;KCsJWFR?pysr6sOG z#104^OjGUa6;;Tk&*`)cBw`0lCggo$&)2;DXH#zN3*y+-*Vx+VLWEZ_eu#uj!w?01 zVx&Usun>yATahEL4pXmkFB74PM_IEyw4*1M0Z7-V~Q_p%qS47+v5%!X7-+{NfdFf@@ zzGMHxe%#v6a8vNi^=9TOI{s$0`X|nX`1C$6twA?1-9qrV`N3*8$PHt#*e|VHC>AAx zscuYr^3m+R8=J56$#}liW+P}KM0j|p3Gpz-21jiScZ%|;QKQZ^DXAuYPB<0Of}Lq; qL_6s}xRz<`&;IK(H!6>$oy=iGbeP?WMaMQ}g zr}5k0%sj7VJi5D?e^M8rP>?8BFXHkNUi$OZz`NE~@5>#$JH0bCd;;z9h?Lq6U~wsdukEr@8iDOtG`ih@vqAwXKZKg}jLHLnINRSRw}n z@kC;erSJpUi5xjKh#kt2Q@gYrr((!O8<8Te%A`Ds6m??T-c@IDmeL0*&|Rckx>CLv z1wWV|8c&esID0&cL-}}qj`Rh$>A!iUk|Td(R(2Sc!4X~zl58VZtqnOyvH?e4>-u>Z zCIb%sfPttru)tN$b2nU?eRK&|>-GP6w&EJN%CrpK*mP^bL+8e;-J@nCz2#wLo3&z5 z!JFyYv?mL*|88uy*1aKptISr=hzWUosCN`Z*y3GF(43-p*T~>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + extra + + " /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>").encode(), + b"<< /Length " + str(len(content)).encode() + b" >>\nstream\n" + content + b"endstream", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"] + output = bytearray(b"%PDF-1.4\n") + offsets = [0] + for index, obj in enumerate(objects, 1): + offsets.append(len(output)) + output.extend(f"{index} 0 obj\n".encode() + obj + b"\nendobj\n") + xref = len(output) + output.extend(f"xref\n0 {len(objects)+1}\n0000000000 65535 f \n".encode()) + for offset in offsets[1:]: + output.extend(f"{offset:010d} 00000 n \n".encode()) + output.extend(f"trailer\n<< /Size {len(objects)+1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()) + return bytes(output) + + +def text(value, x, y): + value = value.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + return f"BT /F1 12 Tf 1 0 0 1 {x} {y} Tm ({value}) Tj ET" + + +def cases(): + for split, left, top, width, prefix in [("dev", 60, 720, 160, "D"), + ("holdout", 95, 680, 135, "H")]: + xs = [left, left+width, left+2*width] + ys = [top, top-40, top-80, top-120] + rules = [f"{x} {ys[-1]} m {x} {top} l S" for x in xs] + rules += [f"{left} {y} m {xs[-1]} {y} l S" for y in ys] + values = [[prefix+" Account", prefix+" Amount"], [prefix+" Cash", "0"], + [prefix+" Debt", "(275)" if split == "dev" else "(640)"]] + for family in ("blank-zero", "reverse-multispan", "boundary-ownership"): + commands = ["0.5 w", *rules] + rows = [] + for row_index, row in enumerate(values): + cells = [] + for column, value in enumerate(row): + if family == "blank-zero" and row_index == 2 and column == 1: + value = "" + y = ys[row_index]-22 + if family == "boundary-ownership" and row_index == 1 and column == 0: + # This text crosses a row rule. Its center belongs to one + # band; tolerant inclusion must never duplicate it. + y = ys[row_index]-4 + if value: + commands.append(text(value, xs[column]+8, y)) + cells.append({"text":value, "row":row_index, "column":column, + "rowspan":1, "colspan":1}) + rows.append({"row_index":row_index, "cells":cells}) + if family == "reverse-multispan": + # Independently positioned tokens on the same line, deliberately + # emitted right-to-left in content-stream order. + commands = ["0.5 w", *rules] + for row_index in reversed(range(3)): + for column in reversed(range(2)): + value = rows[row_index]["cells"][column]["text"] + if column == 0: + first, second = value.split(" ") + commands += [text(second, xs[column]+24, ys[row_index]-22), + text(first, xs[column]+8, ys[row_index]-22)] + else: + commands.append(text(value, xs[column]+8, ys[row_index]-22)) + truth = [{"column_count":2, "rows":rows, "topology_required":True, + "source_ownership_required":True}] + # Boundary ownership is an invariant control, not an arbitrary + # semantic interpretation of lettering crossing a rule. + yield f"{family}-{split}", split, pdf(("\n".join(commands)+"\n").encode()), truth, family + # Graphic axes with narrative prose provide a no-table control. + content = "\n".join(["0.5 w", f"{left} 400 m {left} 500 l S", + f"{left} 400 m {left+200} 400 l S", + text(prefix+" Narrative only", left, 620), + text(prefix+" No numeric columns", left, 590)])+"\n" + yield f"graphic-axis-{split}", split, pdf(content.encode()), [], "negative" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + def retain(path, data): + if path.exists(): + if path.read_bytes() != data:raise ValueError("frozen artifact differs: "+str(path)) + elif args.check: + raise ValueError("frozen artifact missing: "+str(path)) + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + manifest, hashes = [], {} + for name, split, data, truth, family in cases(): + path = ROOT / "corpus" / (name+".pdf") + retain(path, data) + truth_path = ROOT / "truth" / (name+".json") + retain(truth_path, (json.dumps(truth, indent=2)+"\n").encode()) + hashes[str(path.relative_to(ROOT))] = hashlib.sha256(data).hexdigest() + hashes[str(truth_path.relative_to(ROOT))] = hashlib.sha256(truth_path.read_bytes()).hexdigest() + manifest.append({"id":name,"split":split,"pdf":str(path.relative_to(ROOT)), + "truth":str(truth_path.relative_to(ROOT)),"family":family}) + retain(ROOT/"cases.json", (json.dumps(manifest, indent=2)+"\n").encode()) + hashes["cases.json"] = hashlib.sha256((ROOT/"cases.json").read_bytes()).hexdigest() + freeze = {"version":1,"files_sha256":hashes, + "acceptance":{"required":["one source span owner per semantic cell assignment", + "blank is distinct from zero","exact cell coordinates and spans on regular grids", + "reverse content streams retain visual cell text order","no table on graphic-axis controls"], + "boundary_case":"retain every input label exactly once; crossing lettering has no row-truth claim"}, + "scope":"Small synthetic reductions. Holdouts assessed after development; no rule tuning on holdout results."} + retain(ROOT/"freeze.json", (json.dumps(freeze, indent=2)+"\n").encode()) + + +if __name__ == "__main__": + main() diff --git a/benchmark/eval/table_topology/test_verify.py b/benchmark/eval/table_topology/test_verify.py new file mode 100644 index 0000000..ed13da1 --- /dev/null +++ b/benchmark/eval/table_topology/test_verify.py @@ -0,0 +1,110 @@ +import importlib.util +from pathlib import Path +import unittest + +SPEC = importlib.util.spec_from_file_location("topology_verifier", Path(__file__).with_name("verify.py")) +verify = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(verify) + + +def cell(text="Cash", column=0, rowspan=1, colspan=1, ids=None): + return dict(text=text,row=0,column=column,rowspan=rowspan,colspan=colspan,page_index=0, + source_span_ids=["span-0"] if ids is None else ids) + + +class TopologyTests(unittest.TestCase): + def test_boundary_count_uses_exact_financial_tokens(self): + values=["Cash","0","(640)"] + cells=[cell(text,column=i,ids=[f"span-{i}"]) for i,text in enumerate(values)] + records=[dict(record_type="document_manifest",page_count=1)] + records += [dict(record_type="span",span_id=f"span-{i}",text=text,page_index=0) for i,text in enumerate(values)] + records += [dict(record_type="table",column_count=3,rows=[dict(cells=cells)])] + truth=[dict(rows=[dict(cells=cells)])] + self.assertEqual("verified",verify.score(records,truth,"boundary-ownership")["status"]) + records[-1]["rows"][0]["cells"] = cells + [cell("0",column=3,ids=["span-3"])] + records.insert(1,dict(record_type="span",span_id="span-3",text="0",page_index=0)) + self.assertIn("crossing text duplicated or missing: 0",verify.score(records,truth,"boundary-ownership")["failures"]) + + def test_declared_empty_row_changes_physical_topology(self): + rows=[dict(cells=[cell(),cell("0",column=1)])] + truth=[dict(topology_required=True,column_count=2,rows=rows)] + actual=[dict(column_count=2,rows=rows+[dict(row_index=1,cells=[])])] + self.assertEqual(0.0,verify.topology_accuracy(actual,truth)) + + def test_declared_grid_extent_is_scored(self): + rows=[dict(cells=[cell(),cell("0",column=1)])] + truth=[dict(topology_required=True,column_count=2,rows=rows)] + self.assertEqual(0.0,verify.topology_accuracy([dict(column_count=3,rows=rows)],truth)) + self.assertEqual(1.0,verify.topology_accuracy([dict(column_count=2,rows=rows)],truth)) + self.assertEqual(1.0,verify.topology_accuracy([dict(rows=rows)],truth)) + + def test_flattened_text_does_not_hide_wrong_span(self): + actual=[dict(column_count=2,rows=[dict(cells=[cell(colspan=2)])])] + truth=[dict(topology_required=True,column_count=2,rows=[dict(cells=[cell()])])] + self.assertEqual(0.0, verify.topology_accuracy(actual,truth)) + + def test_merged_placeholder_is_not_an_overlap(self): + table=dict(column_count=2,rows=[dict(cells=[cell(colspan=2),cell("",column=1)])]) + self.assertEqual([(0,0,1,2)],verify.table_topology(table)) + table["rows"][0]["cells"][1]["text"]="unexpected" + self.assertIsNone(verify.table_topology(table)) + + def test_uncovered_blank_is_a_real_slot(self): + table=dict(column_count=2,rows=[dict(cells=[cell(),cell("",column=1)])]) + self.assertEqual([(0,0,1,1),(0,1,1,1)],verify.table_topology(table)) + + def test_malformed_or_oversized_spans_are_rejected(self): + for value in [True,0,-1,10**12]: + table=dict(column_count=2,rows=[dict(cells=[cell(colspan=value)])]) + self.assertIsNone(verify.table_topology(table)) + + def test_nonempty_source_claim_must_match_text(self): + records=[dict(record_type="document_manifest"),dict(record_type="span",span_id="span-0",text="Other",page_index=0), + dict(record_type="table",column_count=1,rows=[dict(cells=[cell()])])] + truth=[dict(topology_required=True,column_count=1,rows=[dict(cells=[cell()])])] + self.assertIn("source text disagrees with cell text",verify.score(records,truth,"regular")["failures"]) + + def test_empty_cell_cannot_claim_row_evidence(self): + records=[dict(record_type="document_manifest"),dict(record_type="span",span_id="span-0",text="Cash",page_index=0), + dict(record_type="table",column_count=1,rows=[dict(cells=[cell("")])])] + truth=[dict(topology_required=True,column_count=1,rows=[dict(cells=[cell("")])])] + self.assertIn("empty cell claims source text",verify.score(records,truth,"regular")["failures"]) + + def test_duplicate_source_ownership_is_rejected(self): + cells=[cell(),cell(column=1)] + records=[dict(record_type="document_manifest"),dict(record_type="span",span_id="span-0",text="Cash",page_index=0), + dict(record_type="table",column_count=2,rows=[dict(cells=cells)])] + truth=[dict(topology_required=True,column_count=2,rows=[dict(cells=cells)])] + self.assertIn("source span has multiple cell owners",verify.score(records,truth,"regular")["failures"]) + + def test_negative_control_fails_on_invented_table(self): + records=[dict(record_type="document_manifest"),dict(record_type="table",rows=[])] + self.assertEqual("failed",verify.score(records,[],"negative")["status"]) + + def test_incomplete_or_misordered_stream_is_rejected(self): + records=[dict(record_type="document_manifest"),dict(record_type="document_finished",status="completed")] + self.assertEqual("failed",verify.score(records,[],"negative","stream-jsonl")["status"]) + + def test_stream_page_identity_and_completion_count_are_checked(self): + records=[dict(record_type="document_manifest",page_count=1), + dict(record_type="page_started",page_index=0,status="running"), + dict(record_type="page_finished",page_index=0,status="completed"), + dict(record_type="document_finished",status="completed")] + self.assertEqual("verified",verify.score(records,[],"negative","stream-jsonl")["status"]) + records[2]["page_index"]=99 + self.assertEqual("failed",verify.score(records,[],"negative","stream-jsonl")["status"]) + records[2]["page_index"]=0 + records[0]["page_count"]=2 + self.assertEqual("failed",verify.score(records,[],"negative","stream-jsonl")["status"]) + + def test_stream_artifact_cannot_claim_another_page(self): + records=[dict(record_type="document_manifest",page_count=1), + dict(record_type="page_started",page_index=0,status="running"), + dict(record_type="span",span_id="span-0",text="outside",page_index=9), + dict(record_type="page_finished",page_index=0,status="completed"), + dict(record_type="document_finished",status="completed")] + self.assertIn("artifact outside page lifecycle",verify.score(records,[],"negative","stream-jsonl")["failures"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmark/eval/table_topology/truth/blank-zero-dev.json b/benchmark/eval/table_topology/truth/blank-zero-dev.json new file mode 100644 index 0000000..2e36b5c --- /dev/null +++ b/benchmark/eval/table_topology/truth/blank-zero-dev.json @@ -0,0 +1,66 @@ +[ + { + "column_count": 2, + "rows": [ + { + "row_index": 0, + "cells": [ + { + "text": "D Account", + "row": 0, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "D Amount", + "row": 0, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + }, + { + "row_index": 1, + "cells": [ + { + "text": "D Cash", + "row": 1, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "0", + "row": 1, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + }, + { + "row_index": 2, + "cells": [ + { + "text": "D Debt", + "row": 2, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "", + "row": 2, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + } + ], + "topology_required": true, + "source_ownership_required": true + } +] diff --git a/benchmark/eval/table_topology/truth/blank-zero-holdout.json b/benchmark/eval/table_topology/truth/blank-zero-holdout.json new file mode 100644 index 0000000..00afa9d --- /dev/null +++ b/benchmark/eval/table_topology/truth/blank-zero-holdout.json @@ -0,0 +1,66 @@ +[ + { + "column_count": 2, + "rows": [ + { + "row_index": 0, + "cells": [ + { + "text": "H Account", + "row": 0, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "H Amount", + "row": 0, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + }, + { + "row_index": 1, + "cells": [ + { + "text": "H Cash", + "row": 1, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "0", + "row": 1, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + }, + { + "row_index": 2, + "cells": [ + { + "text": "H Debt", + "row": 2, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "", + "row": 2, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + } + ], + "topology_required": true, + "source_ownership_required": true + } +] diff --git a/benchmark/eval/table_topology/truth/boundary-ownership-dev.json b/benchmark/eval/table_topology/truth/boundary-ownership-dev.json new file mode 100644 index 0000000..48b5465 --- /dev/null +++ b/benchmark/eval/table_topology/truth/boundary-ownership-dev.json @@ -0,0 +1,66 @@ +[ + { + "column_count": 2, + "rows": [ + { + "row_index": 0, + "cells": [ + { + "text": "D Account", + "row": 0, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "D Amount", + "row": 0, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + }, + { + "row_index": 1, + "cells": [ + { + "text": "D Cash", + "row": 1, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "0", + "row": 1, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + }, + { + "row_index": 2, + "cells": [ + { + "text": "D Debt", + "row": 2, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "(275)", + "row": 2, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + } + ], + "topology_required": true, + "source_ownership_required": true + } +] diff --git a/benchmark/eval/table_topology/truth/boundary-ownership-holdout.json b/benchmark/eval/table_topology/truth/boundary-ownership-holdout.json new file mode 100644 index 0000000..0b23f3a --- /dev/null +++ b/benchmark/eval/table_topology/truth/boundary-ownership-holdout.json @@ -0,0 +1,66 @@ +[ + { + "column_count": 2, + "rows": [ + { + "row_index": 0, + "cells": [ + { + "text": "H Account", + "row": 0, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "H Amount", + "row": 0, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + }, + { + "row_index": 1, + "cells": [ + { + "text": "H Cash", + "row": 1, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "0", + "row": 1, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + }, + { + "row_index": 2, + "cells": [ + { + "text": "H Debt", + "row": 2, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "(640)", + "row": 2, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + } + ], + "topology_required": true, + "source_ownership_required": true + } +] diff --git a/benchmark/eval/table_topology/truth/graphic-axis-dev.json b/benchmark/eval/table_topology/truth/graphic-axis-dev.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/benchmark/eval/table_topology/truth/graphic-axis-dev.json @@ -0,0 +1 @@ +[] diff --git a/benchmark/eval/table_topology/truth/graphic-axis-holdout.json b/benchmark/eval/table_topology/truth/graphic-axis-holdout.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/benchmark/eval/table_topology/truth/graphic-axis-holdout.json @@ -0,0 +1 @@ +[] diff --git a/benchmark/eval/table_topology/truth/reverse-multispan-dev.json b/benchmark/eval/table_topology/truth/reverse-multispan-dev.json new file mode 100644 index 0000000..48b5465 --- /dev/null +++ b/benchmark/eval/table_topology/truth/reverse-multispan-dev.json @@ -0,0 +1,66 @@ +[ + { + "column_count": 2, + "rows": [ + { + "row_index": 0, + "cells": [ + { + "text": "D Account", + "row": 0, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "D Amount", + "row": 0, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + }, + { + "row_index": 1, + "cells": [ + { + "text": "D Cash", + "row": 1, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "0", + "row": 1, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + }, + { + "row_index": 2, + "cells": [ + { + "text": "D Debt", + "row": 2, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "(275)", + "row": 2, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + } + ], + "topology_required": true, + "source_ownership_required": true + } +] diff --git a/benchmark/eval/table_topology/truth/reverse-multispan-holdout.json b/benchmark/eval/table_topology/truth/reverse-multispan-holdout.json new file mode 100644 index 0000000..0b23f3a --- /dev/null +++ b/benchmark/eval/table_topology/truth/reverse-multispan-holdout.json @@ -0,0 +1,66 @@ +[ + { + "column_count": 2, + "rows": [ + { + "row_index": 0, + "cells": [ + { + "text": "H Account", + "row": 0, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "H Amount", + "row": 0, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + }, + { + "row_index": 1, + "cells": [ + { + "text": "H Cash", + "row": 1, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "0", + "row": 1, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + }, + { + "row_index": 2, + "cells": [ + { + "text": "H Debt", + "row": 2, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "text": "(640)", + "row": 2, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + } + ], + "topology_required": true, + "source_ownership_required": true + } +] diff --git a/benchmark/eval/table_topology/verify.py b/benchmark/eval/table_topology/verify.py new file mode 100644 index 0000000..9358f08 --- /dev/null +++ b/benchmark/eval/table_topology/verify.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Drive both public JSONL formats against frozen topology and ownership truth.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT.parent)) +from table_compare import table_topology, topology_accuracy + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def score(records, truth, family, output_format="artifact-jsonl"): + failures = [] + if not records or records[0].get("record_type") != "document_manifest": + failures.append("manifest missing") + if sum(r.get("record_type") == "document_manifest" for r in records) != 1: + failures.append("manifest count mismatch") + if records and records[0].get("page_count") != 1: + failures.append("manifest page count mismatch") + if output_format == "stream-jsonl": + started = finished = ended = False + completed_pages = [] + for record in records[1:]: + kind = record.get("record_type") + if ended:failures.append("output after document end") + if kind == "page_started": + if started or record.get("page_index") != 0 or record.get("status") != "running": + failures.append("invalid page start") + started = True + elif kind == "page_finished": + if not started or finished or record.get("page_index") != 0 or record.get("status") != "completed":failures.append("invalid page finish") + finished = True + completed_pages.append(record.get("page_index")) + elif kind == "document_finished": + if not finished or record.get("status") != "completed" or completed_pages != [0]:failures.append("invalid document finish") + ended = True + elif kind in ("span", "block", "table") and (not started or finished or record.get("page_index") != 0): + failures.append("artifact outside page lifecycle") + if not (started and finished and ended):failures.append("incomplete stream") + elif any(r.get("record_type") in ("page_started", "page_finished", "document_finished") for r in records): + failures.append("stream lifecycle in batch output") + tables = [r for r in records if r.get("record_type") == "table"] + spans = {r["span_id"]:r for r in records if r.get("record_type") == "span"} + if len(spans) != sum(r.get("record_type") == "span" for r in records): + failures.append("duplicate span IDs") + if len(tables) != len(truth): + failures.append("table count mismatch") + if family != "boundary-ownership" and topology_accuracy(tables, truth) != 1.0: + failures.append("topology mismatch") + expected_text = [c["text"] for t in truth for row in t["rows"] for c in row["cells"]] + actual_cells = [c for t in tables for row in t["rows"] for c in row["cells"]] + actual_text = [c["text"] for c in actual_cells] + if family == "boundary-ownership": + tokens = " ".join(actual_text).split() + for label in expected_text: + target = label.split() + matches = sum(tokens[index:index+len(target)] == target + for index in range(len(tokens)-len(target)+1)) if target else 0 + if label and matches != 1: + failures.append("crossing text duplicated or missing: "+label) + elif actual_text != expected_text: + failures.append("cell text mismatch (including blank versus zero)") + owned = set() + for cell in actual_cells: + ids = cell.get("source_span_ids", []) + if not cell["text"].strip(): + if ids:failures.append("empty cell claims source text") + continue + if not ids: + failures.append("nonempty cell lacks source spans") + source_text = [] + for sid in ids: + if sid in owned:failures.append("source span has multiple cell owners") + owned.add(sid) + span = spans.get(sid) + if span is None or span.get("page_index") != cell["page_index"]: + failures.append("dangling or wrong-page source span") + else: + source_text.append(span["text"]) + if " ".join(source_text).split() != cell["text"].split(): + failures.append("source text disagrees with cell text") + return dict(status="failed" if failures else "verified", failures=failures, + tables=len(tables), cells=len(actual_cells)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cli", type=Path, required=True) + parser.add_argument("--split", choices=["dev", "holdout"], required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + freeze = json.loads((ROOT/"freeze.json").read_text()) + for name, digest in freeze["files_sha256"].items(): + if sha(ROOT/name) != digest:raise ValueError("frozen corpus drift: "+name) + args.output.mkdir(parents=True, exist_ok=False) + binary = args.cli.resolve(strict=True) + identity = dict(cli=str(binary), cli_sha256=sha(binary), freeze_sha256=sha(ROOT/"freeze.json"), + verifier_sha256=sha(Path(__file__)), head=subprocess.check_output(["git","rev-parse","HEAD"],text=True).strip()) + (args.output/"identity.json").write_text(json.dumps(identity,indent=2)+"\n") + results = [] + for case in json.loads((ROOT/"cases.json").read_text()): + if case["split"] != args.split:continue + truth = json.loads((ROOT/case["truth"]).read_text()) + for fmt in ("artifact-jsonl", "stream-jsonl"): + command = [str(binary),"extract-adaptive","--disable-ocr","--input",str(ROOT/case["pdf"]),"--format",fmt] + result = subprocess.run(command,capture_output=True,timeout=30) + name = case["id"]+"-"+fmt + (args.output/(name+".jsonl")).write_bytes(result.stdout) + (args.output/(name+".stderr")).write_bytes(result.stderr) + records = [] + try: + records = [json.loads(line) for line in result.stdout.splitlines() if line.strip()] + report = score(records, truth, case["family"], fmt) + except (ValueError, KeyError, TypeError) as error: + report = dict(status="failed",failures=[str(error)]) + if result.returncode: + report["status"]="failed";report["failures"].append("CLI failed") + if any(r.get("errors") for r in records): + report["status"]="failed";report["failures"].append("record errors") + report.update(id=case["id"],format=fmt,command=command,exit_code=result.returncode) + results.append(report) + report = dict(identity=identity,results=results,verified=sum(r["status"]=="verified" for r in results),total=len(results)) + (args.output/"report.json").write_text(json.dumps(report,indent=2)+"\n") + print(json.dumps(dict(verified=report["verified"],total=report["total"]))) + return int(report["verified"] != report["total"]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/eval/test_bounded_zig.py b/benchmark/eval/test_bounded_zig.py new file mode 100644 index 0000000..387d5c8 --- /dev/null +++ b/benchmark/eval/test_bounded_zig.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +"""Exercise allocation limits and process ownership without running Zig.""" + +from __future__ import annotations + +import hashlib +import contextlib +import io +import json +import os +import re +import shutil +from pathlib import Path +import subprocess +import sys +import tempfile +import time +import unittest +from unittest import mock +from types import SimpleNamespace + +import bounded_zig + + +class BoundedBuildTests(unittest.TestCase): + def setUp(self): + parent = bounded_zig.REPO_ROOT / "benchmark/eval/outputs/topology-order-2026-10-12" + parent.mkdir(parents=True, exist_ok=True) + # The harness retains even test evidence. No temporary cache is placed in + # a system directory, and no test cleanup deletes previous run results. + self.root = Path(tempfile.mkdtemp(prefix="bounded-runner-test-", dir=parent)) + self.cache = self.root / "cache" + self.log = self.root / "command.log" + self.receipt = self.root / "receipt.json" + + def run_child(self, script: str, **options) -> int: + return bounded_zig.run_command( + [sys.executable, "-c", script], cache_dir=self.cache, + log_path=self.log, receipt_path=self.receipt, + budget_bytes=1024 * 1024, min_free_bytes=0, poll_seconds=0.05, + **options, + ) + + def read_receipt(self): + receipt = json.loads(self.receipt.read_text()) + self.assertEqual(receipt["log_sha256"], hashlib.sha256(self.log.read_bytes()).hexdigest()) + self.assertFalse(receipt["strict_filesystem_quota"]) + self.assertEqual(receipt["deletions"], []) + return receipt + + def test_success_preserves_log_cache_and_identity(self): + result = self.run_child( + "import os,pathlib; " + "p=pathlib.Path(os.environ['ZIG_GLOBAL_CACHE_DIR'])/'item'; " + "p.write_bytes(b'cache'); print('build completed')" + ) + receipt = self.read_receipt() + self.assertEqual(result, 0) + self.assertEqual(receipt["status"], "completed") + self.assertGreater(receipt["peak_allocated_bytes"], 0) + self.assertEqual((self.cache / "global/item").read_bytes(), b"cache") + self.assertIn(b"build completed", self.log.read_bytes()) + self.assertEqual(len(receipt["executable"]["sha256"]), 64) + + def test_limit_stops_owned_parent_and_child_but_not_unrelated_process(self): + child_ready = self.root / "child-ready" + child_stopped = self.root / "child-stopped" + sleeper = ( + "import pathlib,signal,time; " + f"p=pathlib.Path({str(child_stopped)!r}); " + "signal.signal(signal.SIGTERM, lambda *a: (p.write_text('terminated'), exit(0))); " + f"pathlib.Path({str(child_ready)!r}).write_text('ready'); " + "time.sleep(60)" + ) + script = ( + "import os,pathlib,subprocess,sys,time\n" + f"subprocess.Popen([sys.executable,'-c',{sleeper!r}])\n" + f"ready=pathlib.Path({str(child_ready)!r})\n" + "while not ready.exists(): time.sleep(.01)\n" + "p=pathlib.Path(os.environ['ZIG_GLOBAL_CACHE_DIR'])/'large'\n" + "with p.open('wb') as f:\n" + " f.write(b'x'*(2*1024*1024)); f.flush(); os.fsync(f.fileno())\n" + "print('allocated over budget',flush=True)\n" + "time.sleep(60)\n" + ) + unrelated = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"], start_new_session=True) + try: + result = self.run_child(script) + receipt = self.read_receipt() + self.assertEqual(result, 124) + self.assertEqual(receipt["status"], "allocation_budget_exceeded") + self.assertGreater(receipt["observed_budget_overshoot_bytes"], 0) + self.assertIsNotNone(receipt["command_exit_code"]) + self.assertEqual(child_stopped.read_text(), "terminated") + self.assertIsNone(unrelated.poll(), "runner killed an unrelated process") + self.assertTrue((self.cache / "global/large").exists()) + self.assertIn(b"allocated over budget", self.log.read_bytes()) + finally: + unrelated.terminate() + unrelated.wait(timeout=5) + + def test_preexisting_cache_over_budget_never_starts_command(self): + self.cache.mkdir() + cached = self.cache / "preexisting" + cached.write_bytes(b"x" * (2 * 1024 * 1024)) + marker = self.root / "must-not-start" + result = self.run_child(f"import pathlib; pathlib.Path({str(marker)!r}).touch()") + self.assertEqual(result, 125) + receipt = self.read_receipt() + self.assertEqual(receipt["status"], "preflight_allocation_budget_exceeded") + self.assertIsNone(receipt["pid"]) + self.assertFalse(marker.exists()) + self.assertEqual(cached.stat().st_size, 2 * 1024 * 1024) + + def test_existing_evidence_is_not_overwritten(self): + self.receipt.write_text("prior evidence") + with self.assertRaises(FileExistsError): + self.run_child("print('must not start')") + self.assertEqual(self.receipt.read_text(), "prior evidence") + self.assertFalse(self.log.exists()) + + def test_free_space_floor_prevents_start(self): + result = bounded_zig.run_command( + [sys.executable, "-c", "raise RuntimeError('must not start')"], + cache_dir=self.cache, log_path=self.log, receipt_path=self.receipt, + budget_bytes=1024 * 1024, min_free_bytes=2**63, poll_seconds=0.05, + ) + self.assertEqual(result, 125) + receipt = self.read_receipt() + self.assertEqual(receipt["status"], "preflight_free_space_floor_crossed") + self.assertIsNone(receipt["pid"]) + + def test_cli_rejects_cache_and_jobs_override(self): + for argument in ("--cache-dir=/tmp/other", "--global-cache-dir", "-j8"): + with self.subTest(argument=argument), contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + bounded_zig.main(["test", "--", argument]) + self.assertEqual(error.exception.code, 2) + + def test_cli_passes_task_cache_job_limit_and_step_to_selected_tool(self): + tool = self.root / "fake-zig" + tool.write_text( + f"#!{sys.executable}\n" + "import json,os,pathlib,sys\n" + "p=pathlib.Path(os.environ['ZIG_GLOBAL_CACHE_DIR'])/'arguments.json'\n" + "p.write_text(json.dumps(sys.argv[1:]))\n" + "print('selected tool completed')\n" + ) + tool.chmod(0o700) + with contextlib.redirect_stdout(io.StringIO()): + result = bounded_zig.main([ + "test", "--zig", str(tool), "--cache-dir", str(self.cache), + "--log", str(self.log), "--receipt", str(self.receipt), + "--min-free-gib", "0", "--poll-seconds", "0.05", "--", "--summary", "all", + ]) + self.assertEqual(result, 0) + arguments = json.loads((self.cache / "global/arguments.json").read_text()) + self.assertEqual(arguments, [ + "build", "test", "-j2", "--cache-dir", str(self.cache), + "--summary", "all", + ]) + receipt = self.read_receipt() + self.assertEqual(receipt["executable"]["sha256"], hashlib.sha256(tool.read_bytes()).hexdigest()) + self.assertEqual(receipt["cache_environment"], { + "ZIG_LOCAL_CACHE_DIR": str(self.cache), + "ZIG_GLOBAL_CACHE_DIR": str(self.cache / "global"), + }) + + @unittest.skipUnless(shutil.which("zig"), "installed Zig is unavailable") + def test_installed_zig_env_observes_owned_global_cache_without_building(self): + # zig env reports configuration without compiling. Capture its complete + # output privately, checking only the cache variables we supplied. + env = os.environ.copy() + global_cache = self.cache / "global" + env["ZIG_GLOBAL_CACHE_DIR"] = str(global_cache) + env["ZIG_LOCAL_CACHE_DIR"] = str(self.cache) + result = subprocess.run([shutil.which("zig"), "env"], cwd=bounded_zig.REPO_ROOT, env=env, capture_output=True, text=True, check=False) + self.assertEqual(result.returncode, 0, "installed zig env failed") + selected = {} + for name in ("global_cache_dir", "ZIG_GLOBAL_CACHE_DIR", "ZIG_LOCAL_CACHE_DIR"): + match = re.search(r"\." + name + r'\s*=\s*("(?:[^"\\]|\\.)*")', result.stdout) + self.assertIsNotNone(match, f"installed compiler did not report {name}") + selected[name] = json.loads(match.group(1)) + # Zig may report the selected global directory relative to its cwd. + self.assertEqual((bounded_zig.REPO_ROOT / selected["global_cache_dir"]).resolve(), global_cache) + self.assertEqual(selected["ZIG_GLOBAL_CACHE_DIR"], str(global_cache)) + self.assertEqual(selected["ZIG_LOCAL_CACHE_DIR"], str(self.cache)) + # Zig 0.17 opens/creates its selected global directory even for env; + # it must not create compiled artifacts during this configuration probe. + self.assertFalse(any(path.is_file() for path in self.cache.rglob("*"))) + (self.root / "selected-compiler-cache-environment.json").write_text(json.dumps(selected, indent=2) + "\n") + + def test_nonzero_command_exit_is_preserved(self): + self.assertEqual(self.run_child("import sys; print('failed'); sys.exit(7)"), 7) + receipt = self.read_receipt() + self.assertEqual(receipt["status"], "command_failed") + self.assertEqual(receipt["command_exit_code"], 7) + + def test_sigterm_retains_receipt_and_stops_owned_tool(self): + ready = self.root / "ready" + stopped = self.root / "stopped" + tool = self.root / "fake-zig" + tool.write_text( + f"#!{sys.executable}\n" + "import pathlib,signal,time\n" + f"p=pathlib.Path({str(stopped)!r})\n" + "signal.signal(signal.SIGTERM, lambda *a: (p.write_text('terminated'), exit(0)))\n" + f"pathlib.Path({str(ready)!r}).write_text('ready')\n" + "time.sleep(60)\n" + ) + tool.chmod(0o700) + runner = subprocess.Popen([ + sys.executable, str(Path(bounded_zig.__file__)), "test", "--zig", str(tool), + "--cache-dir", str(self.cache), "--log", str(self.log), "--receipt", str(self.receipt), + "--min-free-gib", "0", "--poll-seconds", "0.05", + ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True) + try: + deadline = time.monotonic() + 5 + while not ready.exists() and runner.poll() is None and time.monotonic() < deadline: + time.sleep(0.02) + self.assertTrue(ready.exists(), "tool did not become ready") + runner.terminate() + self.assertEqual(runner.wait(timeout=5), 130) + self.assertEqual(stopped.read_text(), "terminated") + self.assertEqual(self.read_receipt()["status"], "interrupted") + finally: + if runner.poll() is None: + runner.terminate() + runner.wait(timeout=5) + + def test_symlinked_cache_is_rejected(self): + actual = self.root / "actual" + actual.mkdir() + self.cache.symlink_to(actual, target_is_directory=True) + with self.assertRaises(ValueError): + self.run_child("print('must not start')") + self.assertFalse(self.receipt.exists()) + + @contextlib.contextmanager + def child_finishing_during_observation(self, action: str, *, measurement="allocation", exit_code=0, measurement_error=False): + """Complete a real child after a sampled value, before the runner poll. + + Synchronize on child readiness and exit, rather than relying on sampling + call counts or timing a write to land in a narrow race window. + """ + ready = self.root / "race-ready" + released = self.root / "race-released" + finished = self.root / "race-finished" + script = ( + "import os,pathlib,sys,time\n" + f"pathlib.Path({str(ready)!r}).touch()\n" + f"while not pathlib.Path({str(released)!r}).exists(): time.sleep(.005)\n" + + action + "\n" + + f"pathlib.Path({str(finished)!r}).touch()\n" + + f"sys.exit({exit_code})\n" + ) + actual_popen = subprocess.Popen + actual_allocated = bounded_zig.allocated_bytes + owned = {} + + def launch(*args, **kwargs): + proc = actual_popen(*args, **kwargs) + if kwargs.get("start_new_session"): + owned["process"] = proc + return proc + + def complete_after_read(value): + if "process" in owned and not released.exists(): + deadline = time.monotonic() + 5 + while not ready.exists() and time.monotonic() < deadline: + time.sleep(.005) + if not ready.exists(): + raise TimeoutError("race child did not become ready") + released.touch() + owned["process"].wait(timeout=5) + return value + + def allocation(path): + if finished.exists() and measurement_error: + raise PermissionError("controlled final measurement failure") + value = actual_allocated(path) + return complete_after_read(value) if measurement == "allocation" else value + + def free_space(_path): + value = SimpleNamespace(free=0 if finished.exists() else 200) + return complete_after_read(value) if measurement == "free" else value + + with mock.patch.object(bounded_zig.subprocess, "Popen", side_effect=launch), mock.patch.object(bounded_zig, "allocated_bytes", side_effect=allocation): + if measurement == "free": + with mock.patch.object(bounded_zig.shutil, "disk_usage", side_effect=free_space): + yield script + else: + yield script + + def test_final_allocation_breach_rejects_real_child_success(self): + action = ( + "p=pathlib.Path(os.environ['ZIG_GLOBAL_CACHE_DIR'])/'final-large'\n" + "with p.open('wb') as f:\n" + " f.write(b'x'*(2*1024*1024)); f.flush(); os.fsync(f.fileno())" + ) + with self.child_finishing_during_observation(action) as script: + result = self.run_child(script) + receipt = self.read_receipt() + self.assertEqual(result, 124) + self.assertEqual(receipt["command_exit_code"], 0) + self.assertEqual(receipt["status"], "allocation_budget_exceeded") + self.assertEqual(receipt["final_breach"], "allocation_budget_exceeded") + self.assertGreater(receipt["last_allocated_bytes"], receipt["budget_allocated_bytes"]) + + def test_final_free_floor_breach_rejects_real_child_success(self): + with self.child_finishing_during_observation("print('completed')", measurement="free") as script: + result = bounded_zig.run_command( + [sys.executable, "-c", script], cache_dir=self.cache, + log_path=self.log, receipt_path=self.receipt, + budget_bytes=1024 * 1024, min_free_bytes=100, poll_seconds=.05, + ) + receipt = self.read_receipt() + self.assertEqual(result, 124) + self.assertEqual(receipt["command_exit_code"], 0) + self.assertEqual(receipt["status"], "free_space_floor_crossed") + self.assertEqual(receipt["final_breach"], "free_space_floor_crossed") + self.assertLess(receipt["last_free_bytes"], receipt["minimum_free_bytes"]) + + def test_final_breach_keeps_original_command_failure(self): + with self.child_finishing_during_observation("print('command failed')", measurement="free", exit_code=7) as script: + result = bounded_zig.run_command( + [sys.executable, "-c", script], cache_dir=self.cache, + log_path=self.log, receipt_path=self.receipt, + budget_bytes=1024 * 1024, min_free_bytes=100, poll_seconds=.05, + ) + receipt = self.read_receipt() + self.assertEqual(result, 7) + self.assertEqual(receipt["status"], "command_failed") + self.assertEqual(receipt["final_breach"], "free_space_floor_crossed") + + def test_final_measurement_error_rejects_success(self): + with self.child_finishing_during_observation("print('completed')", measurement_error=True) as script: + result = self.run_child(script) + receipt = self.read_receipt() + self.assertEqual(result, 125) + self.assertEqual(receipt["status"], "final_measurement_failed") + self.assertEqual(receipt["command_exit_code"], 0) + self.assertIn("PermissionError", receipt["final_measurement_error"]) + + def test_final_measurement_error_keeps_original_command_failure(self): + with self.child_finishing_during_observation("print('failed')", exit_code=7, measurement_error=True) as script: + result = self.run_child(script) + receipt = self.read_receipt() + self.assertEqual(result, 7) + self.assertEqual(receipt["status"], "command_failed") + self.assertIn("PermissionError", receipt["final_measurement_error"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/output-schema.md b/docs/output-schema.md index 2a18446..bbc42ae 100644 --- a/docs/output-schema.md +++ b/docs/output-schema.md @@ -6,11 +6,10 @@ should depend on the records documented here. ## Version -Current development schema version: `0.19.0` (locally checked, committed on -`codex/coordinate-vector-ocr`). +Current development schema version: `0.19.1` (local working-tree continuation). Current parser build identity: `0.4.0-dev`. The `v0.3.0` release emitted schema -`0.11.0`; the development suffix keeps post-release schema `0.19.0` artifacts +`0.11.0`; the development suffix keeps post-release schema `0.19.1` artifacts distinguishable until the next release is tagged. The project is still pre-`1.0.0`, so incompatible schema changes may happen. @@ -35,6 +34,17 @@ JSON output is a `document_manifest` object with arrays of typed records. but records are produced page by page so host applications can persist partial artifacts and enqueue embeddings before the document finishes. +Schema `0.19.1` fixes table evidence granularity and shared-rule assignment. +Reconstructed table spans retain individual nonempty cell text and bounds, +rather than synthesizing whole-row spans whose centers can falsely identify +an empty cell. Shared rules assign a center to one band; surrounding prose +is retained when a table layer replaces native page text. Fields, types, +coordinate units and source kinds retain their existing meanings. Blank +cells remain in the table grid and do not create text spans. Derived cell +spans retain `table_model` provenance; they are not original native glyphs. +Legacy native ordering also resets columns at corroborated section headings. +This does not promote the separate diagnostic graph model. + Schema `0.19.0` adds required nullable vector evidence on page and region route traces. Bounded path collection proposes geometric OCR candidates without changing the raw-coordinate or source-text contract. Readers must explicitly diff --git a/integrations/javascript/integration.test.mjs b/integrations/javascript/integration.test.mjs index 7a25c6d..55e8ebe 100644 --- a/integrations/javascript/integration.test.mjs +++ b/integrations/javascript/integration.test.mjs @@ -80,7 +80,7 @@ test('actual CLI -> validated rich text -> overlapping chunks -> page/span locat skip: available ? false : 'Build the parser with zig build to run the real CLI integration', }, async () => { const document = await extractPdfDocument(fixturePdf(), { binaryPath }); - assert.equal(document.manifest.schema_version, '0.19.0'); + assert.equal(document.manifest.schema_version, '0.19.1'); assert.equal(document.spans.length, 4); assert.equal(document.blocks.length, 4); assert.equal(document.sourceText, Array(4).fill('Repeated evidence.').join('\n\n')); @@ -141,7 +141,7 @@ test('actual multi-page stream preserves global ownership with page-local locato const receipt = await extractPdfPages(fixturePdf(), { binaryPath, onPage: async page => { pages.push(page); } }); assert.equal(receipt.complete, true); assert.equal(receipt.pageCount, 2); - assert.equal(receipt.schemaVersion, '0.19.0'); + assert.equal(receipt.schemaVersion, '0.19.1'); assert.deepEqual(pages.map(page => page.pageIndex), [0, 1]); assert.equal(new Set(pages.flatMap(page => page.spans.map(span => span.span_id))).size, 4); for (const page of pages) { diff --git a/integrations/javascript/vector-routing.test.mjs b/integrations/javascript/vector-routing.test.mjs index e63e5e6..1714afd 100644 --- a/integrations/javascript/vector-routing.test.mjs +++ b/integrations/javascript/vector-routing.test.mjs @@ -62,7 +62,7 @@ async function cliRecords(t, name, flags = [], format = 'artifact-jsonl') { encoding: 'utf8', timeout: 45000, maxBuffer: 10 * 1024 * 1024, }); const records = raw.trim().split('\n').map(JSON.parse); - assert.equal(records[0].schema_version, '0.19.0'); + assert.equal(records[0].schema_version, '0.19.1'); const document = format === 'artifact-jsonl' ? parseArtifactJsonl(raw, { inputSha256: createHash('sha256').update(bytes).digest('hex') }) : null; return { bytes, raw, records, document }; } @@ -194,7 +194,7 @@ test('deterministic regional vector OCR binds routes attempts source text chunks const pages = []; const receipt = await extractPdfPages(bytes, { binaryPath: fake.binaryPath, enableOcr: true, onPage: page => pages.push(page) }); assert.equal(receipt.complete, true); - assert.equal(receipt.schemaVersion, '0.19.0'); + assert.equal(receipt.schemaVersion, '0.19.1'); assertNative(pages[0], 'mixed-native-outlines'); assertLocated(pages[0], 'VECTOR EVIDENCE'); const cropPaths = (await readFile(fake.log, 'utf8')).trim().split('\n'); diff --git a/src/adaptive.zig b/src/adaptive.zig index 3d7239d..751fc87 100644 --- a/src/adaptive.zig +++ b/src/adaptive.zig @@ -2046,12 +2046,15 @@ fn appendLayoutBlockSpans( block_index: usize, ) !void { const block = page_layout.blocks[block_index]; - if (block.removed or block.lines.len == 0) return; + if (block.lines.len == 0) return; + // The table layer replaces the native page layer. Furniture suppression + // is a text-rendering choice, not authority to erase that source evidence. if (page_layout.tableForBlock(block_index)) |table| { if (tableHasUsefulTextRows(table)) { - try appendBlockLinesOutsideTable(allocator, spans, block, table); - try appendTableRowSpans(allocator, spans, table); + try appendBlockLinesOutsideTable(allocator, spans, block, table, true); + try appendTableCellSpans(allocator, spans, table); + try appendBlockLinesOutsideTable(allocator, spans, block, table, false); return; } } @@ -2060,7 +2063,13 @@ fn appendLayoutBlockSpans( if (tableCoveringBlock(page_layout.tables, block_index)) |table| { skip_covered = tableHasUsefulTextRows(table); } - if (skip_covered) return; + if (skip_covered) { + if (tableCoveringBlock(page_layout.tables, block_index)) |table| { + try appendBlockLinesOutsideTable(allocator, spans, block, table, true); + try appendBlockLinesOutsideTable(allocator, spans, block, table, false); + } + return; + } } for (block.lines) |line| { @@ -2131,29 +2140,31 @@ fn tableHasUsefulTextRows(table: *const layout.TableGrid) bool { return occupied_rows >= 2; } -fn appendTableRowSpans( +fn appendTableCellSpans( allocator: std.mem.Allocator, spans: *std.ArrayList(layout.TextSpan), table: *const layout.TableGrid, ) !void { for (table.rows) |row| { - const text = try tableRowTextOwned(allocator, row); - errdefer allocator.free(text); - if (text.len == 0) { - allocator.free(text); - continue; + for (row.cells) |cell| { + // A full-row bbox cannot identify which cell owns its text: + // its center can even fall in an empty numeric cell. Retain the + // already reconstructed cell granularity through reconciliation. + if (std.mem.trim(u8, cell.text, " \t\r\n").len == 0) continue; + const text = try allocator.dupe(u8, cell.text); + errdefer allocator.free(text); + try spans.append(allocator, layout.TextSpan.init(.{ + .page_index = cell.bounds.page_index, + .bbox = cell.bounds.bbox, + .text = text, + .source = .table_model, + .confidence = cell.confidence, + .font = stableFont(cell.bounds.font), + .block_id = cell.bounds.block_id, + .line_id = cell.bounds.line_id, + .mcid = cell.bounds.mcid, + })); } - try spans.append(allocator, layout.TextSpan.init(.{ - .page_index = row.bounds.page_index, - .bbox = row.bounds.bbox, - .text = text, - .source = .table_model, - .confidence = table.confidence, - .font = stableFont(row.bounds.font), - .block_id = row.bounds.block_id, - .line_id = row.bounds.line_id, - .mcid = row.bounds.mcid, - })); } } @@ -2162,9 +2173,12 @@ fn appendBlockLinesOutsideTable( spans: *std.ArrayList(layout.TextSpan), block: layout.LayoutBlock, table: *const layout.TableGrid, + before: bool, ) !void { for (block.lines) |line| { if (spansIntersect(line.bounds, table.bounds)) continue; + const above = line.bounds.y0 >= table.bounds.y1; + if (above != before) continue; try appendLineSpan(allocator, spans, line, block.confidence); } } @@ -2214,21 +2228,6 @@ fn lineTextOwned(allocator: std.mem.Allocator, line: *const layout.TextLine) ![] return output.toOwnedSlice(allocator); } -fn tableRowTextOwned(allocator: std.mem.Allocator, row: layout.TableRow) ![]u8 { - var output: std.ArrayList(u8) = .empty; - errdefer output.deinit(allocator); - - var wrote_cell = false; - for (row.cells) |cell| { - if (cell.text.len == 0) continue; - if (wrote_cell) try output.append(allocator, ' '); - try output.appendSlice(allocator, cell.text); - wrote_cell = true; - } - - return output.toOwnedSlice(allocator); -} - fn stableFont(font: layout.FontMetadata) layout.FontMetadata { return .{ .size = font.size, @@ -3097,6 +3096,35 @@ test "mixed native OCR filter keeps image OCR and drops page furniture OCR" { try std.testing.expectEqualStrings("scan text", filtered[0].text); } +fn tableCellSpanAllocationProbe(allocator: std.mem.Allocator) !void { + var spans: std.ArrayList(layout.TextSpan) = .empty; + defer { + for (spans.items) |span| allocator.free(@constCast(span.text)); + spans.deinit(allocator); + } + const left = layout.TextSpan.init(.{ .bbox = .{ .x0 = 10, .y0 = 50, .x1 = 40, .y1 = 62 }, .text = "" }); + const right = layout.TextSpan.init(.{ .bbox = .{ .x0 = 100, .y0 = 50, .x1 = 110, .y1 = 62 }, .text = "" }); + var cells = [_]layout.TableCell{ + .{ .bounds = left, .text = "Cash", .row_index = 0, .column_index = 0 }, + .{ .bounds = right, .text = "0", .row_index = 0, .column_index = 1 }, + .{ .bounds = right, .text = "", .row_index = 0, .column_index = 2 }, + }; + var rows = [_]layout.TableRow{.{ .bounds = left, .cells = &cells, .row_index = 0 }}; + const table = layout.TableGrid{ .bounds = left, .block_index = 0, .rows = &rows, .column_count = 3 }; + try appendTableCellSpans(allocator, &spans, &table); + try std.testing.expectEqual(@as(usize, 2), spans.items.len); + try std.testing.expectEqualStrings("Cash", spans.items[0].text); + try std.testing.expectEqualStrings("0", spans.items[1].text); + try std.testing.expectEqualDeep(left.bbox, spans.items[0].bbox); + try std.testing.expectEqualDeep(right.bbox, spans.items[1].bbox); + try std.testing.expectEqual(layout.SourceKind.table_model, spans.items[0].source); +} + +test "table evidence retains cell granularity and excludes blank placeholders" { + try tableCellSpanAllocationProbe(std.testing.allocator); + try std.testing.checkAllAllocationFailures(std.testing.allocator, tableCellSpanAllocationProbe, .{}); +} + test "table layout layer suppresses represented OCR words but keeps residual totals" { const represented = [_]layout.TextSpan{ layout.TextSpan.init(.{ diff --git a/src/integration_test.zig b/src/integration_test.zig index f823d76..b38b2b6 100644 --- a/src/integration_test.zig +++ b/src/integration_test.zig @@ -728,7 +728,7 @@ test "versioned schema renders native document manifest spans blocks chunks and const parsed = try std.json.parseFromSlice(std.json.Value, allocator, json, .{}); defer parsed.deinit(); try std.testing.expectEqualStrings("document_manifest", parsed.value.object.get("schema_name").?.string); - try std.testing.expectEqualStrings("0.19.0", parsed.value.object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.19.1", parsed.value.object.get("schema_version").?.string); try std.testing.expectEqualStrings("0.4.0-dev", parsed.value.object.get("parser_version").?.string); try std.testing.expectEqualStrings("document_manifest", parsed.value.object.get("record_type").?.string); try std.testing.expectEqualStrings("external-clean-native", parsed.value.object.get("source_id").?.string); @@ -774,7 +774,7 @@ test "versioned schema renders native document manifest spans blocks chunks and const debug_assets = parsed.value.object.get("debug_assets").?.array.items; try std.testing.expect(debug_assets.len > 0); try expectProvenanceObject(debug_assets[0]); - try std.testing.expectEqualStrings("0.19.0", debug_assets[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.19.1", debug_assets[0].object.get("schema_version").?.string); try std.testing.expect(debug_assets[0].object.get("asset_kind") != null); try std.testing.expect(debug_assets[0].object.get("path") != null); try std.testing.expectEqual(.null, debug_assets[0].object.get("path").?); @@ -785,7 +785,7 @@ test "versioned schema renders native document manifest spans blocks chunks and try std.testing.expectEqualStrings("debug", debug_assets[0].object.get("provenance").?.object.get("source_kind").?.string); try std.testing.expect(std.mem.indexOf(u8, json, "\"schema_name\":\"document_manifest\"") != null); - try std.testing.expect(std.mem.indexOf(u8, json, "\"schema_version\":\"0.19.0\"") != null); + try std.testing.expect(std.mem.indexOf(u8, json, "\"schema_version\":\"0.19.1\"") != null); try std.testing.expect(std.mem.indexOf(u8, json, "\"record_type\":\"span\"") != null); try std.testing.expect(std.mem.indexOf(u8, json, "\"record_type\":\"block\"") != null); try std.testing.expect(std.mem.indexOf(u8, json, "\"record_type\":\"rag_chunk\"") != null); @@ -909,7 +909,7 @@ test "versioned artifact jsonl starts with manifest then typed records" { try std.testing.expect(manifest.value.object.get("output_artifacts") != null); try std.testing.expect(manifest.value.object.get("extraction_counts") != null); try std.testing.expect(manifest.value.object.get("capability_coverage") != null); - try std.testing.expect(std.mem.indexOf(u8, jsonl[first_newline + 1 ..], "\"schema_version\":\"0.19.0\"") != null); + try std.testing.expect(std.mem.indexOf(u8, jsonl[first_newline + 1 ..], "\"schema_version\":\"0.19.1\"") != null); try std.testing.expect(std.mem.indexOf(u8, jsonl[first_newline + 1 ..], "\"record_type\":\"span\"") != null); try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"record_type\":\"route_trace\"") != null); try expectJsonlLinesParse(allocator, jsonl); @@ -941,7 +941,7 @@ test "specialist protocol emits batch request records for routed table formula r const requests = parsed.value.object.get("specialist_requests").?.array.items; try std.testing.expect(requests.len > 0); try expectProvenanceObject(requests[0]); - try std.testing.expectEqualStrings("0.19.0", requests[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.19.1", requests[0].object.get("schema_version").?.string); try std.testing.expectEqualStrings("specialist_request", requests[0].object.get("record_type").?.string); try std.testing.expectEqualStrings("external-specialist", requests[0].object.get("source_id").?.string); try std.testing.expect(requests[0].object.get("requested_kind") != null); @@ -1035,7 +1035,7 @@ test "configured formula specialist produces owned linked lifecycle records" { \\request_id=$(printf '%s' "$request" | sed -n 's/.*"request_id":"\([^"]*\)".*/\1/p') \\crop=$(printf '%s' "$request" | sed -n 's/.*"crop_image_path":"\([^"]*\)".*/\1/p') \\test -n "$crop" && test -f "$crop" || exit 7 - \\printf '{"schema_version":"0.19.0","record_type":"specialist_response","request_id":"%s","specialist_id":"fixture-formula","specialist_kind":"formula","status":"completed","formulas":[{"text":"x^2+y^2=z^2","format":"latex","confidence":0.95}]}\n' "$request_id" + \\printf '{"schema_version":"0.19.1","record_type":"specialist_response","request_id":"%s","specialist_id":"fixture-formula","specialist_kind":"formula","status":"completed","formulas":[{"text":"x^2+y^2=z^2","format":"latex","confidence":0.95}]}\n' "$request_id" \\ ); runtime.closeFile(script); @@ -1111,7 +1111,7 @@ test "configured formula specialist produces owned linked lifecycle records" { const formulas = response.object.get("formulas").?.array.items; try std.testing.expectEqual(@as(usize, 1), formulas.len); try std.testing.expectEqualStrings("formula", formulas[0].object.get("schema_name").?.string); - try std.testing.expectEqualStrings("0.19.0", formulas[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.19.1", formulas[0].object.get("schema_version").?.string); try std.testing.expectEqualStrings("formula", formulas[0].object.get("record_type").?.string); try std.testing.expectEqualStrings("source-formula", formulas[0].object.get("source_id").?.string); try expectProvenanceObject(formulas[0]); @@ -1512,7 +1512,7 @@ test "versioned schema exposes financial table cell span metadata" { const tables = parsed.value.object.get("tables").?.array.items; try std.testing.expect(tables.len > 0); try expectProvenanceObject(tables[0]); - try std.testing.expectEqualStrings("0.19.0", tables[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.19.1", tables[0].object.get("schema_version").?.string); try std.testing.expect(tables[0].object.get("logical_table_id") != null); try std.testing.expect(tables[0].object.get("table_part_index") != null); try std.testing.expect(tables[0].object.get("continued_from_table_id") != null); @@ -1522,7 +1522,7 @@ test "versioned schema exposes financial table cell span metadata" { const cells = tables[0].object.get("rows").?.array.items[0].object.get("cells").?.array.items; try std.testing.expect(cells.len > 0); try std.testing.expectEqualStrings("table_cell", cells[0].object.get("schema_name").?.string); - try std.testing.expectEqualStrings("0.19.0", cells[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.19.1", cells[0].object.get("schema_version").?.string); try std.testing.expect(cells[0].object.get("cell_id") != null); try std.testing.expectEqualStrings("external-merged-cells", cells[0].object.get("source_id").?.string); try std.testing.expect(cells[0].object.get("raw_text") != null); diff --git a/src/layout.zig b/src/layout.zig index f9efec5..62a5a26 100644 --- a/src/layout.zig +++ b/src/layout.zig @@ -1622,7 +1622,7 @@ fn buildRuledTableGrid( if (!intersectsSpan(span, grid_bbox)) continue; const cx = (span.x0 + span.x1) / 2.0; const cy = (span.y0 + span.y1) / 2.0; - if (cy > top + 1.0 or cy < bottom - 1.0) continue; + if ((rowForY(ys.items, cy) orelse continue) != row_index) continue; const col_index = columnForX(xs.items, cx) orelse continue; try buckets[col_index].spans.append(allocator, span); } @@ -1717,9 +1717,23 @@ fn intersectsSpan(a: TextSpan, b: TextSpan) bool { } fn columnForX(xs: []const f64, x: f64) ?usize { - if (xs.len < 2) return null; + if (xs.len < 2 or !std.math.isFinite(x)) return null; + // Tolerance applies only to the outside edge. Shared rules have one + // owner: the column to their right, never both adjacent columns. + if (x < xs[0] - 1.0 or x > xs[xs.len - 1] + 1.0) return null; for (0..xs.len - 1) |index| { - if (x >= xs[index] - 1.0 and x <= xs[index + 1] + 1.0) return index; + if (x < xs[index + 1] or index == xs.len - 2) return index; + } + return null; +} + +fn rowForY(ys: []const f64, y: f64) ?usize { + if (ys.len < 2 or !std.math.isFinite(y)) return null; + // Coordinates descend in PDF space. A shared horizontal rule belongs + // to the row below it; the bottom outside edge remains inclusive. + if (y > ys[0] + 1.0 or y < ys[ys.len - 1] - 1.0) return null; + for (0..ys.len - 1) |index| { + if (y > ys[index + 1] or index == ys.len - 2) return index; } return null; } @@ -2787,6 +2801,41 @@ test "layout reconstruction clusters lines and dehyphenates paragraphs" { try std.testing.expectEqualStrings("The document works.", text); } +test "ruled table shared boundaries have exactly one band owner" { + const xs = [_]f64{ 60, 220, 380 }; + const ys = [_]f64{ 720, 680, 640 }; + try std.testing.expectEqual(@as(?usize, 1), columnForX(&xs, 220)); + try std.testing.expectEqual(@as(?usize, 0), columnForX(&xs, 219.9)); + try std.testing.expectEqual(@as(?usize, 1), rowForY(&ys, 680)); + try std.testing.expectEqual(@as(?usize, 0), rowForY(&ys, 680.1)); + try std.testing.expectEqual(@as(?usize, 1), rowForY(&ys, 679.9)); + try std.testing.expectEqual(@as(?usize, 1), rowForY(&ys, 640)); + try std.testing.expectEqual(@as(?usize, null), rowForY(&ys, 638)); + try std.testing.expectEqual(@as(?usize, null), columnForX(&xs, std.math.nan(f64))); +} + +test "ruled table text crossing a rule is not duplicated into adjacent rows" { + const spans = [_]TextSpan{ + testSpan("Account", 70, 698, 115, 710), + testSpan("Amount", 230, 698, 275, 710), + testSpan("Boundary", 70, 674, 125, 686), + testSpan("0", 230, 655, 240, 667), + }; + const rulings = [_]RulingLine{ + .{ .orientation = .vertical, .bbox = .{ .x0 = 60, .x1 = 60, .y0 = 640, .y1 = 720 } }, + .{ .orientation = .vertical, .bbox = .{ .x0 = 220, .x1 = 220, .y0 = 640, .y1 = 720 } }, + .{ .orientation = .vertical, .bbox = .{ .x0 = 380, .x1 = 380, .y0 = 640, .y1 = 720 } }, + .{ .orientation = .horizontal, .bbox = .{ .x0 = 60, .x1 = 380, .y0 = 720, .y1 = 720 } }, + .{ .orientation = .horizontal, .bbox = .{ .x0 = 60, .x1 = 380, .y0 = 680, .y1 = 680 } }, + .{ .orientation = .horizontal, .bbox = .{ .x0 = 60, .x1 = 380, .y0 = 640, .y1 = 640 } }, + }; + var result = try analyzeLayoutWithRulings(std.testing.allocator, &spans, 612, &rulings); + defer result.deinit(); + try std.testing.expectEqual(@as(usize, 1), result.tables.len); + try std.testing.expectEqualStrings("Account", result.tables[0].rows[0].cells[0].text); + try std.testing.expectEqualStrings("Boundary", result.tables[0].rows[1].cells[0].text); +} + test "layout reconstruction records two columns in reading order" { const allocator = std.testing.allocator; const spans = [_]TextSpan{ diff --git a/src/main.zig b/src/main.zig index 2c13e91..3b29412 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2072,7 +2072,7 @@ test "adaptive CLI emits specialist request JSONL sidecars" { \\request_id=$(printf '%s' "$request" | sed -n 's/.*"request_id":"\([^"]*\)".*/\1/p') \\crop=$(printf '%s' "$request" | sed -n 's/.*"crop_image_path":"\([^"]*\)".*/\1/p') \\test -n "$crop" && test -f "$crop" || exit 7 - \\printf '{"schema_version":"0.19.0","record_type":"specialist_response","request_id":"%s","specialist_id":"cli-formula","specialist_kind":"formula","status":"completed","formulas":[{"text":"x^2+y^2=z^2","format":"latex","confidence":0.95}]}\n' "$request_id" + \\printf '{"schema_version":"0.19.1","record_type":"specialist_response","request_id":"%s","specialist_id":"cli-formula","specialist_kind":"formula","status":"completed","formulas":[{"text":"x^2+y^2=z^2","format":"latex","confidence":0.95}]}\n' "$request_id" \\ ); runtime.closeFile(formula_file); @@ -2146,7 +2146,7 @@ test "adaptive CLI emits specialist request JSONL sidecars" { const requests = try runtime.readFileAllocAlignedCwd(allocator, requests_path, .fromByteUnits(1)); defer allocator.free(requests); try std.testing.expect(std.mem.indexOf(u8, requests, "\"record_type\":\"specialist_request\"") != null); - try std.testing.expect(std.mem.indexOf(u8, requests, "\"schema_version\":\"0.19.0\"") != null); + try std.testing.expect(std.mem.indexOf(u8, requests, "\"schema_version\":\"0.19.1\"") != null); try std.testing.expect(std.mem.indexOf(u8, requests, "\"source_id\":\"external-specialist-cli\"") != null); try std.testing.expect(std.mem.indexOf(u8, requests, "\"requested_kind\":\"") != null); try std.testing.expect(std.mem.indexOf(u8, requests, "\"requested_outputs\"") != null); diff --git a/src/native_layout.zig b/src/native_layout.zig index 928515e..b2f7d97 100644 --- a/src/native_layout.zig +++ b/src/native_layout.zig @@ -607,6 +607,7 @@ fn orderLinesByRegions( gutter: ?BBox, ) ![]u32 { const order = try allocator.alloc(u32, lines.len); + errdefer allocator.free(order); if (gutter == null) { for (order, 0..) |*entry, index| entry.* = @intCast(index); return order; @@ -614,9 +615,18 @@ fn orderLinesByRegions( var output: std.ArrayList(u32) = .empty; defer output.deinit(allocator); + var weighted_size: f64 = 0; + var glyph_count: usize = 0; + var left_margin: f64 = std.math.inf(f64); + for (lines) |line| { + weighted_size += line.max_font_size * @as(f64, @floatFromInt(line.count)); + glyph_count += line.count; + if (line.region_index == 0) left_margin = @min(left_margin, line.bbox.x0); + } + const body_size = if (glyph_count > 0) weighted_size / @as(f64, @floatFromInt(glyph_count)) else 12; var section_start: usize = 0; for (lines, 0..) |line, line_index| { - if (line.region_index != 2) continue; + if (line.region_index != 2 and !isColumnSectionHeading(lines, line_index, body_size, left_margin)) continue; try appendColumnSection(allocator, &output, lines, section_start, line_index); try output.append(allocator, @intCast(line_index)); section_start = line_index + 1; @@ -626,6 +636,39 @@ fn orderLinesByRegions( return order; } +fn isColumnSectionHeading(lines: []const PendingLine, index: usize, body_size: f64, left_margin: f64) bool { + const heading = lines[index]; + if (heading.region_index != 0 or heading.max_font_size < body_size * 1.18 or + heading.bbox.x0 > left_margin + heading.max_font_size) return false; + // A large left-column label is not by itself a section boundary. Require + // a clear baseline across both columns, then content in both flows below + // it before another heading. This leaves column-local headings local. + const tolerance = @max(1.5, heading.max_font_size * 0.55); + var previous_index = index; + while (previous_index > 0) { + previous_index -= 1; + const line = lines[previous_index]; + if (line.baseline() - heading.baseline() > tolerance) break; + if (line.region_index == 1 and @abs(line.baseline() - heading.baseline()) <= tolerance) return false; + } + for (lines[index + 1 ..]) |line| { + if (heading.baseline() - line.baseline() > tolerance) break; + if (line.region_index == 1 and @abs(line.baseline() - heading.baseline()) <= tolerance) return false; + } + var has_left = false; + var has_right = false; + for (lines[index + 1 ..]) |line| { + if (line.baseline() >= heading.baseline() - tolerance) continue; + if (line.region_index == 2 or + (line.region_index == 0 and line.max_font_size >= body_size * 1.18 and + line.bbox.x0 <= left_margin + line.max_font_size)) break; + if (line.region_index == 0) has_left = true; + if (line.region_index == 1) has_right = true; + if (has_left and has_right) return true; + } + return false; +} + fn appendColumnSection( allocator: std.mem.Allocator, output: *std.ArrayList(u32), @@ -1186,6 +1229,27 @@ test "scaled baseline bands retain mixed-font superscripts on the same line" { try std.testing.expectEqual(@as(usize, 1), result.lines.len); } +test "column order resets at corroborated section headings" { + var lines: [10]PendingLine = undefined; + for (&lines, 0..) |*line, index| { + const local = index % 5; + const heading = local == 0; + const right = local == 2 or local == 4; + const y: f64 = 750 - @as(f64, @floatFromInt(index / 5)) * 190 - + (if (heading) @as(f64, 0) else if (local <= 2) @as(f64, 50) else @as(f64, 100)); + line.* = .{ .start = index, .count = 20, .baseline_sum = y * 20, .baseline_count = 20, .max_font_size = if (heading) 18 else 12, .bbox = .{ .x0 = if (right) 330 else 60, .x1 = if (right) 500 else 200, .y0 = y, .y1 = y + 12 }, .region_index = if (right) 1 else 0 }; + } + const gutter = BBox{ .x0 = 280, .x1 = 310, .y0 = 0, .y1 = 792 }; + const order = try orderLinesByRegions(std.testing.allocator, &lines, gutter); + defer std.testing.allocator.free(order); + try std.testing.expectEqualSlices(u32, &.{ 0, 1, 3, 2, 4, 5, 6, 8, 7, 9 }, order); + // A right-flow label on the same baseline is contrary evidence, even + // when the left label is larger than body text. + lines[6].baseline_sum = lines[5].baseline_sum; + lines[6].region_index = 1; + try std.testing.expect(!isColumnSectionHeading(&lines, 5, 13, 60)); +} + test "column gutter cannot split a continuous transformed word at paint boundaries" { const words = [_][]const u8{ "VISIBLE", "HIDDEN", "CLIPPED", "TRANSFORMED" }; const xs = [_]f64{ 20, 20, 180, 50 }; diff --git a/src/specialist_protocol.zig b/src/specialist_protocol.zig index 8514585..3098af7 100644 --- a/src/specialist_protocol.zig +++ b/src/specialist_protocol.zig @@ -12,7 +12,7 @@ const reconcile = @import("reconcile.zig"); const runtime = @import("runtime.zig"); const page_geometry = @import("page_geometry.zig"); -pub const schema_version = "0.19.0"; +pub const schema_version = "0.19.1"; pub const SpecialistKind = enum { ocr, From 6ca09fc5e32521fb00cda8b43270e6e0ec5f8b35 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Mon, 12 Oct 2026 14:59:28 -0400 Subject: [PATCH 2/2] Scope reconciled table evidence to cell identity --- .../eval/reading_order_next/test_verify.py | 5 ++ benchmark/eval/reading_order_next/verify.py | 6 +- benchmark/eval/table_topology/test_verify.py | 5 ++ benchmark/eval/table_topology/verify.py | 6 +- docs/output-schema.md | 8 +- src/adaptive.zig | 10 ++- src/layout.zig | 9 +++ src/reconcile.zig | 23 ++++++ src/schema.zig | 77 +++++++++++++++++-- 9 files changed, 134 insertions(+), 15 deletions(-) diff --git a/benchmark/eval/reading_order_next/test_verify.py b/benchmark/eval/reading_order_next/test_verify.py index 55c8fcc..c9369ba 100644 --- a/benchmark/eval/reading_order_next/test_verify.py +++ b/benchmark/eval/reading_order_next/test_verify.py @@ -28,6 +28,11 @@ def records(): class VerificationTest(unittest.TestCase): + def test_public_command_uses_supported_ocr_disable_flag(self): + command = VERIFY.extraction_command("parser", "input.pdf", "stream-jsonl") + self.assertIn("--no-ocr", command) + self.assertNotIn("--disable-ocr", command) + def test_valid_stream_and_artifact(self): self.assertEqual(VERIFY.score_records(records(), truth(), "stream-jsonl")["verdict"], "verified") artifact = [r for r in records() if r["record_type"] in ("span", "block", "document_manifest")] diff --git a/benchmark/eval/reading_order_next/verify.py b/benchmark/eval/reading_order_next/verify.py index 5960250..511a73e 100644 --- a/benchmark/eval/reading_order_next/verify.py +++ b/benchmark/eval/reading_order_next/verify.py @@ -157,6 +157,10 @@ def execute(command, stem, timeout): return code, stdout +def extraction_command(cli, pdf, output_format): + return [str(cli), "extract-adaptive", "--input", str(pdf), "--format", output_format, "--no-ocr"] + + def run(cli, output, split, eval_cli=None, timeout=60): freeze_hash = check_freeze() metadata = [json.loads(line) for line in (ROOT / "metadata.jsonl").read_text().splitlines()] @@ -176,7 +180,7 @@ def run(cli, output, split, eval_cli=None, timeout=60): truth = json.loads((ROOT / "truth" / f"{doc_id}.json").read_text()) pdf = ROOT / "corpus" / f"{doc_id}.pdf" for fmt in ("artifact-jsonl", "stream-jsonl"): - command = [str(cli), "extract-adaptive", "--input", str(pdf), "--format", fmt, "--disable-ocr"] + command = extraction_command(cli, pdf, fmt) code, stdout = execute(command, output / f"{doc_id}-{fmt}", timeout) try: score = score_records(parse_records(stdout), truth, fmt, code) diff --git a/benchmark/eval/table_topology/test_verify.py b/benchmark/eval/table_topology/test_verify.py index ed13da1..cea7c98 100644 --- a/benchmark/eval/table_topology/test_verify.py +++ b/benchmark/eval/table_topology/test_verify.py @@ -13,6 +13,11 @@ def cell(text="Cash", column=0, rowspan=1, colspan=1, ids=None): class TopologyTests(unittest.TestCase): + def test_public_command_uses_supported_ocr_disable_flag(self): + command = verify.extraction_command("parser", "input.pdf", "artifact-jsonl") + self.assertIn("--no-ocr", command) + self.assertNotIn("--disable-ocr", command) + def test_boundary_count_uses_exact_financial_tokens(self): values=["Cash","0","(640)"] cells=[cell(text,column=i,ids=[f"span-{i}"]) for i,text in enumerate(values)] diff --git a/benchmark/eval/table_topology/verify.py b/benchmark/eval/table_topology/verify.py index 9358f08..5a4b6a0 100644 --- a/benchmark/eval/table_topology/verify.py +++ b/benchmark/eval/table_topology/verify.py @@ -16,6 +16,10 @@ def sha(path): return hashlib.sha256(path.read_bytes()).hexdigest() +def extraction_command(binary, pdf, output_format): + return [str(binary), "extract-adaptive", "--no-ocr", "--input", str(pdf), "--format", output_format] + + def score(records, truth, family, output_format="artifact-jsonl"): failures = [] if not records or records[0].get("record_type") != "document_manifest": @@ -109,7 +113,7 @@ def main(): if case["split"] != args.split:continue truth = json.loads((ROOT/case["truth"]).read_text()) for fmt in ("artifact-jsonl", "stream-jsonl"): - command = [str(binary),"extract-adaptive","--disable-ocr","--input",str(ROOT/case["pdf"]),"--format",fmt] + command = extraction_command(binary, ROOT/case["pdf"], fmt) result = subprocess.run(command,capture_output=True,timeout=30) name = case["id"]+"-"+fmt (args.output/(name+".jsonl")).write_bytes(result.stdout) diff --git a/docs/output-schema.md b/docs/output-schema.md index bbc42ae..e6a6193 100644 --- a/docs/output-schema.md +++ b/docs/output-schema.md @@ -518,9 +518,11 @@ reconciled block owns it. Its `provenance.block_ids` agrees with that owner. The older numeric `block_id` retains its layout-index meaning. Tables and nested cells have `reconciled_block_ids`. These list the reconciled -owners of their `source_span_ids` on the same page. Source spans are selected -by the existing span-center-inside-bbox rule; a bbox intersection alone does -not establish ownership. A spanning block can therefore belong to multiple +owners of their `source_span_ids` on the same page. Reconstructed cell spans +retain their selected cell identity through reconciliation, including centers +exactly on shared rules. Blank cells claim no text spans. Other spans retain +the span-center-inside-bbox fallback; a bbox intersection alone does not +establish ownership. A spanning block can therefore belong to multiple cells. Legacy table `block_index`, `block_count` and `provenance.block_ids` retain pre-reconciliation indices and must not be used as authoritative links. These additions establish source linkage, not table topology accuracy. diff --git a/src/adaptive.zig b/src/adaptive.zig index 751fc87..b288fc7 100644 --- a/src/adaptive.zig +++ b/src/adaptive.zig @@ -2153,7 +2153,7 @@ fn appendTableCellSpans( if (std.mem.trim(u8, cell.text, " \t\r\n").len == 0) continue; const text = try allocator.dupe(u8, cell.text); errdefer allocator.free(text); - try spans.append(allocator, layout.TextSpan.init(.{ + var span = layout.TextSpan.init(.{ .page_index = cell.bounds.page_index, .bbox = cell.bounds.bbox, .text = text, @@ -2163,7 +2163,13 @@ fn appendTableCellSpans( .block_id = cell.bounds.block_id, .line_id = cell.bounds.line_id, .mcid = cell.bounds.mcid, - })); + }); + span.table_cell_owner = .{ + .layout_block_index = table.block_index, + .row_index = cell.row_index, + .column_index = cell.column_index, + }; + try spans.append(allocator, span); } } } diff --git a/src/layout.zig b/src/layout.zig index 62a5a26..e5ff760 100644 --- a/src/layout.zig +++ b/src/layout.zig @@ -117,6 +117,9 @@ pub const TextSpan = struct { paint: PaintEvidence = .{}, content_scope: ?form_scope.Scope = null, leading_boundary: ?TextBoundary = null, + // Internal ownership for reconstructed cells; geometry alone cannot + // distinguish a shared-rule center from its blank neighboring slot. + table_cell_owner: ?TableCellOwner = null, pub fn init(args: struct { page_index: u32 = 0, @@ -164,6 +167,12 @@ pub const TextSpan = struct { } }; +pub const TableCellOwner = struct { + layout_block_index: u32, + row_index: u32, + column_index: u32, +}; + pub const TextWord = struct { bounds: TextSpan, spans: []const TextSpan, diff --git a/src/reconcile.zig b/src/reconcile.zig index cc9e288..c45d432 100644 --- a/src/reconcile.zig +++ b/src/reconcile.zig @@ -761,6 +761,11 @@ fn findDuplicate(spans: []const ReconciledSpan, duplicate_index: *const Duplicat const same_page_indices = duplicate_index.indicesForPage(candidate.span.page_index) orelse return null; for (same_page_indices) |index| { const existing = spans[index]; + if (existing.span.table_cell_owner) |owner| { + if (candidate.span.table_cell_owner) |other| { + if (!std.meta.eql(owner, other)) continue; + } + } // MCIDs identify marked-content groups, which may contain multiple // distinct spans. Duplicate identity still requires text and placement. if (!std.mem.eql(u8, normalizedText(existing.span.text), normalizedText(candidate.span.text))) continue; @@ -772,6 +777,7 @@ fn findDuplicate(spans: []const ReconciledSpan, duplicate_index: *const Duplicat fn mergeDuplicate(allocator: std.mem.Allocator, existing: *ReconciledSpan, candidate: ReconciledSpan) void { const merged_mask = existing.source_mask | candidate.source_mask; const replace = sourceScore(candidate) > sourceScore(existing.*); + const cell_owner = existing.span.table_cell_owner orelse candidate.span.table_cell_owner; if (replace) { const old_span = existing.span; @@ -783,6 +789,7 @@ fn mergeDuplicate(allocator: std.mem.Allocator, existing: *ReconciledSpan, candi } existing.source_mask = merged_mask; + existing.span.table_cell_owner = cell_owner; existing.source_count = countSourceBits(merged_mask); existing.duplicate_count += 1; existing.confidence = @max(existing.confidence, candidate.confidence); @@ -1391,6 +1398,22 @@ test "fresh OCR independent crop boundaries separate overlapping PDF regions" { for (doc.spans) |span| try std.testing.expectEqual(@as(usize, 0), span.block_text_start); } +test "reconciliation preserves cell identities across overlap and native duplicates" { + var first = testSpan("0", .table_model, 210, 670, 230, 690); + first.table_cell_owner = .{ .layout_block_index = 7, .row_index = 1, .column_index = 1 }; + var second = first; + second.table_cell_owner.?.column_index = 2; + const cells = [_]TextSpan{ first, second }; + var doc = try reconcile(std.testing.allocator, &.{.{ .spans = &cells }}, .{}); + defer doc.deinit(); + try std.testing.expectEqual(@as(usize, 2), doc.spans.len); + const native = testSpan("0", .native_pdf, 210, 670, 230, 690); + var merged = try reconcile(std.testing.allocator, &.{ .{ .spans = &.{native} }, .{ .spans = &.{first} } }, .{}); + defer merged.deinit(); + try std.testing.expectEqual(@as(usize, 1), merged.spans.len); + try std.testing.expectEqualDeep(first.table_cell_owner, merged.spans[0].span.table_cell_owner); +} + test "coordinate serializers include hocr alto and debug overlay" { const native = [_]TextSpan{ testSpan("Hello ", .native_pdf, 10, 700, 80, 712), diff --git a/src/schema.zig b/src/schema.zig index 667cf9f..5413bc4 100644 --- a/src/schema.zig +++ b/src/schema.zig @@ -1307,7 +1307,7 @@ fn writeTableRecord(allocator: std.mem.Allocator, writer: anytype, document_id: try writer.writeAll(",\"cells\":["); for (row.cells, 0..) |cell, cell_index| { if (cell_index > 0) try writer.writeByte(','); - try writeCellJson(allocator, writer, document_id, source_id, input_sha256, result, table_index, cell, offsets, &scratch.cell_span_ids, stream_meta != null); + try writeCellJson(allocator, writer, document_id, source_id, input_sha256, result, table_index, table.block_index, cell, offsets, &scratch.cell_span_ids, stream_meta != null); } try writer.writeAll("]}"); } @@ -1340,12 +1340,13 @@ fn writeCellJson( input_sha256: ?[]const u8, result: anytype, table_index: u32, + layout_block_index: u32, cell: layout.TableCell, offsets: RecordOffsets, cell_span_ids: *std.ArrayList(u32), stream: bool, ) !void { - try collectCellSpanIds(allocator, cell_span_ids, result, cell, offsets); + try collectCellSpanIds(allocator, cell_span_ids, result, cell, layout_block_index, offsets); try writer.writeAll("{\"schema_name\":\"table_cell\",\"schema_version\":\""); try writer.writeAll(schema_version); @@ -1379,7 +1380,7 @@ fn writeCellJson( try writer.writeAll(",\"source_span_ids\":"); try writeSpanIdArray(writer, cell_span_ids.items); try writer.writeAll(",\"reconciled_block_ids\":"); - try writeOwningBlockIds(writer, result, cell.bounds.page_index, cell.bounds.bbox, offsets); + try writeOwningBlockIdsForSpanIds(writer, result, cell_span_ids.items, offsets); try writeProvenancePrefix(writer, .{ .document_id = document_id, .source_id = source_id, @@ -2051,21 +2052,45 @@ fn writePaint(writer: anytype, paint: layout.PaintEvidence) !void { try specialist_protocol.writePaintEvidence(writer, paint); } -fn collectCellSpanIds(allocator: std.mem.Allocator, span_ids: *std.ArrayList(u32), result: anytype, cell: layout.TableCell, offsets: RecordOffsets) !void { +fn writeOwningBlockIdsForSpanIds(writer: anytype, result: anytype, span_ids: []const u32, offsets: RecordOffsets) !void { + try writer.writeByte('['); + var first = true; + for (result.reconciled.blocks) |block| { + var matched = false; + for (span_ids) |span_id| { + const local: usize = span_id - offsets.span_base; + if (local >= block.span_start and local - block.span_start < block.span_count) { + matched = true; + break; + } + } + if (!matched) continue; + if (!first) try writer.writeByte(','); + try writer.print("\"block-{d}\"", .{offsets.block_base + block.id}); + first = false; + } + try writer.writeByte(']'); +} + +fn collectCellSpanIds(allocator: std.mem.Allocator, span_ids: *std.ArrayList(u32), result: anytype, cell: layout.TableCell, layout_block_index: u32, offsets: RecordOffsets) !void { span_ids.clearRetainingCapacity(); + if (std.mem.trim(u8, cell.text, " \t\r\n").len == 0) return; + const owner = layout.TableCellOwner{ .layout_block_index = layout_block_index, .row_index = cell.row_index, .column_index = cell.column_index }; if (offsets.span_lookup) |lookup| { if (lookup.spanRange(cell.bounds.page_index)) |range| { - try collectSpanIdsInRange(allocator, span_ids, result.reconciled.spans[range.start..range.end], range.start, cell.bounds.page_index, cell.bounds.bbox, offsets.span_base); + try collectSpanIdsInRange(allocator, span_ids, result.reconciled.spans[range.start..range.end], range.start, cell.bounds.page_index, cell.bounds.bbox, owner, offsets.span_base); } return; } - try collectSpanIdsInRange(allocator, span_ids, result.reconciled.spans, 0, cell.bounds.page_index, cell.bounds.bbox, offsets.span_base); + try collectSpanIdsInRange(allocator, span_ids, result.reconciled.spans, 0, cell.bounds.page_index, cell.bounds.bbox, owner, offsets.span_base); } -fn collectSpanIdsInRange(allocator: std.mem.Allocator, span_ids: *std.ArrayList(u32), spans: anytype, start_index: usize, page_index: u32, bbox: layout.BBox, span_base: u32) !void { +fn collectSpanIdsInRange(allocator: std.mem.Allocator, span_ids: *std.ArrayList(u32), spans: anytype, start_index: usize, page_index: u32, bbox: layout.BBox, owner: layout.TableCellOwner, span_base: u32) !void { for (spans, start_index..) |span, span_index| { if (span.span.page_index != page_index) continue; - if (!centerInside(span.span.bbox, bbox)) continue; + if (span.span.table_cell_owner) |actual| { + if (!std.meta.eql(actual, owner)) continue; + } else if (!centerInside(span.span.bbox, bbox)) continue; try span_ids.append(allocator, span_base + @as(u32, @intCast(span_index))); } } @@ -2653,6 +2678,42 @@ fn centerInside(inner: layout.BBox, outer: layout.BBox) bool { return x >= outer.x0 and x <= outer.x1 and y >= outer.y0 and y <= outer.y1; } +test "cell ownership rejects shared-rule neighbors in indexed and streaming selections" { + const allocator = std.testing.allocator; + var span = layout.TextSpan.init(.{ .bbox = .{ .x0 = 215, .x1 = 225, .y0 = 675, .y1 = 685 }, .text = "Edge", .source = .table_model }); + span.table_cell_owner = .{ .layout_block_index = 7, .row_index = 1, .column_index = 1 }; + const spans = [_]struct { span: layout.TextSpan }{.{ .span = span }}; + const blocks = [_]struct { id: u32, span_start: u32, span_count: u32 }{.{ .id = 3, .span_start = 0, .span_count = 1 }}; + const result = .{ .reconciled = .{ .spans = &spans, .blocks = &blocks } }; + const bounds = layout.TextSpan.init(.{ .bbox = .{ .x0 = 220, .x1 = 380, .y0 = 640, .y1 = 680 }, .text = "" }); + const cell = layout.TableCell{ .bounds = bounds, .text = "Edge", .row_index = 1, .column_index = 1 }; + var ids: std.ArrayList(u32) = .empty; + defer ids.deinit(allocator); + const ranges = [_]IndexRange{.{ .start = 0, .end = 1 }}; + const lookup = SpanLookup{ .allocator = allocator, .span_ranges = @constCast(&ranges) }; + for ([_]RecordOffsets{ .{ .span_base = 100, .block_base = 200 }, .{ .span_base = 100, .block_base = 200, .span_lookup = &lookup } }) |offsets| { + try collectCellSpanIds(allocator, &ids, result, cell, 7, offsets); + try std.testing.expectEqualSlices(u32, &.{100}, ids.items); + var neighbor = cell; + neighbor.text = ""; + neighbor.column_index = 0; + try collectCellSpanIds(allocator, &ids, result, neighbor, 7, offsets); + try std.testing.expectEqual(@as(usize, 0), ids.items.len); + neighbor.text = "Different"; + try collectCellSpanIds(allocator, &ids, result, neighbor, 7, offsets); + try std.testing.expectEqual(@as(usize, 0), ids.items.len); + try collectCellSpanIds(allocator, &ids, result, cell, 8, offsets); + try std.testing.expectEqual(@as(usize, 0), ids.items.len); + } + var output: std.ArrayList(u8) = .empty; + defer output.deinit(allocator); + try writeOwningBlockIdsForSpanIds(runtime.arrayListWriter(&output, allocator), result, &.{100}, .{ .span_base = 100, .block_base = 200 }); + try std.testing.expectEqualStrings("[\"block-203\"]", output.items); + output.clearRetainingCapacity(); + try writeOwningBlockIdsForSpanIds(runtime.arrayListWriter(&output, allocator), result, &.{}, .{}); + try std.testing.expectEqualStrings("[]", output.items); +} + fn findManualSpanForField(result: anytype, field: anytype) ?usize { const value = field.value orelse return null; for (result.reconciled.spans, 0..) |span, index| {